Improve configuration generation

Signed-off-by: Knut Ahlers <knut@ahlers.me>

README.md

@@ -1,8 +1,21 @@
 # Luzifer-Docker / netdata
 
-This repository contains a dockerized version of the [netdata](https://github.com/firehol/netdata) daemon. It supports adding and overriding configurations and plugins through overrides.
+This repository contains a dockerized version of the [netdata](https://github.com/firehol/netdata) daemon. It supports adding and overriding configurations and plugins through overrides and partially configuration through environment variables.
 
-The expected structure on the `/override` volume mount is as following:
+## Usage
+
+```
+docker run -d --cap-add SYS_PTRACE \
+           -v /proc:/host/proc:ro \
+           -v /sys:/host/sys:ro \
+           -p 19999:19999 quay.io/luzifer/netdata
+```
+
+## Configuration
+
+To configure alerts have a look at the [`health_alarm_notify.conf` template](templates/health_alarm_notify.conf). There you can see all variable names you need to specify as environment variables.
+
+When using the override mount you can add configuration files and plugins. The expected structure on the `/override` volume mount is as following:
 
 ```
 /override
@@ -12,3 +25,4 @@ The expected structure on the `/override` volume mount is as following:
 ├── plugins.d
 └── python.d
 ```
+

build.sh

@@ -42,3 +42,8 @@ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 ln -sf /dev/stdout /var/log/netdata/access.log
 ln -sf /dev/stdout /var/log/netdata/debug.log
 ln -sf /dev/stderr /var/log/netdata/error.log
+
+# Install korvike for template substitution
+curl -sSLfo /usr/local/bin/korvike https://github.com/Luzifer/korvike/releases/download/v0.4.1/korvike_linux_amd64
+chmod +x /usr/local/bin/korvike
+echo 'f791fecdc62b2e2ff07342b41fdf165ee40c2a8a286f1c2c0f48228b982e2953  /usr/local/bin/korvike' | sha256sum -c

run.sh

@@ -25,30 +25,6 @@ root:netdata@$SSMTP_HOSTNAME:$SSMTP_SERVER:$SSMTP_PORT
 EOF
 fi
 
-if [[ $SLACK_WEBHOOK_URL ]]; then
-  sed -i -e "s@SLACK_WEBHOOK_URL=\"\"@SLACK_WEBHOOK_URL=\"${SLACK_WEBHOOK_URL}\"@" /etc/netdata/health_alarm_notify.conf
-fi
-
-if [[ $SLACK_CHANNEL ]]; then
-  sed -i -e "s@DEFAULT_RECIPIENT_SLACK=\"\"@DEFAULT_RECIPIENT_SLACK=\"${SLACK_CHANNEL}\"@" /etc/netdata/health_alarm_notify.conf
-fi
-
-if [[ $TELEGRAM_BOT_TOKEN ]]; then
-  sed -i -e "s@TELEGRAM_BOT_TOKEN=\"\"@TELEGRAM_BOT_TOKEN=\"${TELEGRAM_BOT_TOKEN}\"@" /etc/netdata/health_alarm_notify.conf
-fi
-
-if [[ $TELEGRAM_CHAT_ID ]]; then
-  sed -i -e "s@DEFAULT_RECIPIENT_TELEGRAM=\"\"@DEFAULT_RECIPIENT_TELEGRAM=\"${TELEGRAM_CHAT_ID}\"@" /etc/netdata/health_alarm_notify.conf
-fi
-
-if [[ $PUSHBULLET_ACCESS_TOKEN ]]; then
-  sed -i -e "s@PUSHBULLET_ACCESS_TOKEN=\"\"@PUSHBULLET_ACCESS_TOKEN=\"${PUSHBULLET_ACCESS_TOKEN}\"@" /etc/netdata/health_alarm_notify.conf
-fi
-
-if [[ $PUSHBULLET_DEFAULT_EMAIL ]]; then
-  sed -i -e "s#DEFAULT_RECIPIENT_PUSHBULLET=\"\"#DEFAULT_RECIPIENT_PUSHBULLET=\"${PUSHBULLET_DEFAULT_EMAIL}\"#" /etc/netdata/health_alarm_notify.conf
-fi
-
 if [[ $NETDATA_IP ]]; then
   NETDATA_ARGS="${NETDATA_ARGS} -i ${NETDATA_IP}"
 fi
@@ -72,6 +48,15 @@ if printenv | grep -q 'NETDATA_API_KEY_ENABLE_'; then
   printenv | grep -oe 'NETDATA_API_KEY_ENABLE_[^=]\+' | sed 's/NETDATA_API_KEY_ENABLE_//' | xargs -n1 -I{} echo '['{}$']\n\tenabled = yes' >> /etc/netdata/stream.conf
 fi
 
+# Execute templates
+korvike -i /src/templates/health_alarm_notify.conf -o /etc/netdata/health_alarm_notify.conf
+
+# Pull in overrides and additions for config and plugins
+[ -e /override/conf.d ] && rsync -arv /override/conf.d/ /etc/netdata/
+for dir in charts.d node.d plugins.d python.d; do
+  [ -e "/override/${dir}" ] && rsync -arv "/override/${dir}/" "/usr/libexec/netdata/${dir}/"
+done
+
 # exec custom command
 if [[ $# -gt 0 ]] ; then
   exec "$@"
@@ -87,11 +72,5 @@ if [[ -d "/fakenet/" ]]; then
   sleep 1
 fi
 
-# Pull in overrides and additions for config and plugins
-[ -e /override/conf.d ] && rsync -arv /override/conf.d/ /etc/netdata/
-for dir in charts.d node.d plugins.d python.d; do
-  [ -e "/override/${dir}" ] && rsync -arv "/override/${dir}/" "/usr/libexec/netdata/${dir}/"
-done
-
 # main entrypoint
 exec /usr/sbin/netdata -D -u root -s /host -p ${NETDATA_PORT} ${NETDATA_ARGS}

templates/health_alarm_notify.conf

@@ -0,0 +1,498 @@
+# Configuration for alarm notifications
+#
+# This configuration is used by: alarm-notify.sh
+# changes take effect immediately (the next alarm will use them).
+#
+# alarm-notify.sh can send:
+# - e-mails (using the sendmail command),
+# - push notifications to your mobile phone (pushover.net),
+# - messages to your slack team (slack.com),
+# - messages to your discord guild (discordapp.com),
+# - messages to your telegram chat / group chat (telegram.org)
+# - sms messages to your cell phone or any sms enabled device (twilio.com)
+# - sms messages to your cell phone or any sms enabled device (messagebird.com)
+# - notifications to users on pagerduty.com
+#
+# The 'to' line given at netdata alarms defines a *role*, so that many
+# people can be notified for each role.
+#
+# This file is a BASH script itself.
+#
+#
+#------------------------------------------------------------------------------
+# proxy configuration
+#
+# If you need to send curl based notifications (pushover, pushbullet, slack,
+# discord, telegram) via a proxy, set these to your proxy address:
+#export http_proxy="http://10.0.0.1:3128/"
+#export https_proxy="http://10.0.0.1:3128/"
+
+
+#------------------------------------------------------------------------------
+# notifications images
+#
+# Images in notifications need to be downloaded from an Internet facing site.
+# To allow notification providers fetch the icons/images, by default we set
+# the URL of the global public netdata registry.
+# If you have an Internet facing netdata (or you have copied the images/ folder
+# of netdata to your web server), set its URL here, to fetch the notification
+# images from it.
+#images_base_url="http://my.public.netdata.server:19999"
+
+
+#------------------------------------------------------------------------------
+# external commands
+
+# The full path to the sendmail command.
+# If empty, the system $PATH will be searched for it.
+# If not found, email notifications will be disabled (silently).
+sendmail=""
+
+# The full path of the curl command.
+# If empty, the system $PATH will be searched for it.
+# If not found, most notifications will be silently disabled.
+curl=""
+
+
+#------------------------------------------------------------------------------
+# NOTE ABOUT RECIPIENTS
+#
+# When you define recipients (all types):
+#
+#  - emails addresses
+#  - pushover user tokens
+#  - telegram chat ids
+#  - slack channels
+#  - discord channels
+#  - hipchat rooms
+#  - sms phone numbers
+#  - pagerduty.com (pd) services
+#
+# You can append |critical to limit the notifications to be sent.
+#
+# In these examples, the first recipient receives all the alarms
+# while the second one receives only the critical ones:
+#
+#  email      : "user1@example.com user2@example.com|critical"
+#  pushover   : "2987343...9437837 8756278...2362736|critical"
+#  telegram   : "111827421 112746832|critical"
+#  slack      : "alarms disasters|critical"
+#  discord    : "alarms disasters|critical"
+#  twilio     : "+15555555555 +17777777777|critical"
+#  messagebird: "+15555555555 +17777777777|critical"
+#  pd         : "<pd_service_key_1> <pd_service_key_2>|critical"
+#
+# If a recipient is set to empty string, the default recipient of the given
+# notification method (email, pushover, telegram, slack, etc) will be used.
+# To disable a notification, use the recipient called: disabled
+# This works for all notification methods (including the default recipients).
+
+
+#------------------------------------------------------------------------------
+# email global notification options
+
+# multiple recipients can be given like this:
+#              "admin1@example.com admin2@example.com ..."
+
+# enable/disable sending emails
+SEND_EMAIL="{{ env `SEND_EMAIL` `YES` }}"
+
+# if a role recipient is not configured, an email will be send to:
+DEFAULT_RECIPIENT_EMAIL="{{ env `DEFAULT_RECIPIENT_EMAIL` `root` }}"
+# to receive only critical alarms, set it to "root|critical"
+
+
+#------------------------------------------------------------------------------
+# pushover (pushover.net) global notification options
+
+# multiple recipients can be given like this:
+#                  "USERTOKEN1 USERTOKEN2 ..."
+
+# enable/disable sending pushover notifications
+SEND_PUSHOVER="{{ env `SEND_PUSHOVER` `YES` }}"
+
+# Login to pushover.net to get your pushover app token.
+# You need only one for all your netdata servers (or you can have one for
+# each of your netdata - your call).
+# Without an app token, netdata cannot send pushover notifications.
+PUSHOVER_APP_TOKEN="{{ env `PUSHOVER_APP_TOKEN` }}"
+
+# if a role's recipients are not configured, a notification will be send to
+# this pushover user token (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_PUSHOVER="{{ env `DEFAULT_RECIPIENT_PUSHOVER` }}"
+
+
+#------------------------------------------------------------------------------
+# pushbullet (pushbullet.com) push notification options
+
+# multiple recipients can be given like this:
+#                  "user1@email.com user2@mail.com"
+
+# enable/disable sending pushbullet notifications
+SEND_PUSHBULLET="{{ env `SEND_PUSHBULLET` `YES` }}"
+
+# Signup and Login to pushbullet.com
+# To get your Access Token, go to https://www.pushbullet.com/#settings/account
+# Create a new access token and paste it below.
+# Then just set the recipients' emails.
+# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does
+# not have a pushbullet account, the pushbullet service will send an email
+# to that address instead.
+
+# Without an access token, netdata cannot send pushbullet notifications.
+PUSHBULLET_ACCESS_TOKEN="{{ env `PUSHBULLET_ACCESS_TOKEN` }}"
+DEFAULT_RECIPIENT_PUSHBULLET="{{ env `DEFAULT_RECIPIENT_PUSHBULLET` }}"
+
+
+#------------------------------------------------------------------------------
+# Twilio (twilio.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "+15555555555 +17777777777"
+
+# enable/disable sending twilio SMS
+SEND_TWILIO="{{ env `SEND_TWILIO` `YES` }}"
+
+# Signup for free trial and select a SMS capable Twilio Number
+# To get your Account SID and Token, go to https://www.twilio.com/console
+# Place your sid, token and number below.
+# Then just set the recipients' phone numbers.
+# The trial account is only allowed to use the number specified when set up.
+
+# Without an account sid and token, netdata cannot send Twilio text messages.
+TWILIO_ACCOUNT_SID="{{ env `TWILIO_ACCOUNT_SID` }}"
+TWILIO_ACCOUNT_TOKEN="{{ env `TWILIO_ACCOUNT_TOKEN` }}"
+TWILIO_NUMBER="{{ env `TWILIO_NUMBER` }}"
+DEFAULT_RECIPIENT_TWILIO="{{ env `DEFAULT_RECIPIENT_TWILIO` }}"
+
+
+#------------------------------------------------------------------------------
+# Messagebird (messagebird.com) SMS options
+
+# multiple recipients can be given like this:
+#                  "+15555555555 +17777777777"
+
+# enable/disable sending messagebird SMS
+SEND_MESSAGEBIRD="{{ env `SEND_MESSAGEBIRD` `YES` }}"
+
+# to get an access key, create a free account at https://www.messagebird.com
+# verify and activate the account (no CC info needed)
+# login to your account and enter your phonenumber to get some free credits
+# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' 
+# click 'Add access key' and fill in data (you want a live key to send SMS)
+
+# Without an access key, netdata cannot send Messagebird text messages.
+MESSAGEBIRD_ACCESS_KEY="{{ env `MESSAGEBIRD_ACCESS_KEY` }}"
+MESSAGEBIRD_NUMBER="{{ env `MESSAGEBIRD_NUMBER` }}"
+DEFAULT_RECIPIENT_MESSAGEBIRD="{{ env `DEFAULT_RECIPIENT_MESSAGEBIRD` }}"
+
+
+#------------------------------------------------------------------------------
+# telegram (telegram.org) global notification options
+
+# To get your chat ID send the command /my_id to telegram bot @get_id.
+# Users also need to open a query with the bot (see below).
+
+# note: multiple recipients can be given like this:
+#                  "CHAT_ID_1 CHAT_ID_2 ..."
+
+# enable/disable sending telegram messages
+SEND_TELEGRAM="{{ env `SEND_TELEGRAM` `YES` }}"
+
+# Contact the bot @BotFather to create a new bot and receive a bot token.
+# Without it, netdata cannot send telegram messages.
+TELEGRAM_BOT_TOKEN="{{ env `TELEGRAM_BOT_TOKEN` }}"
+
+# If a role's recipients are not configured, a message will be send to
+# this chat id (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_TELEGRAM="{{ env `DEFAULT_RECIPIENT_TELEGRAM` }}"
+
+
+#------------------------------------------------------------------------------
+# slack (slack.com) global notification options
+
+# multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending slack notifications
+SEND_SLACK="{{ env `SEND_SLACK` `YES` }}"
+
+# Login to slack.com and create an incoming webhook. You need only one for all
+# your netdata servers (or you can have one for each of your netdata).
+# Without it, netdata cannot send slack notifications.
+# Get yours from: https://api.slack.com/incoming-webhooks
+SLACK_WEBHOOK_URL="{{ env `SLACK_WEBHOOK_URL` }}"
+
+# if a role's recipients are not configured, a notification will be send to
+# this slack channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_SLACK="{{ env `DEFAULT_RECIPIENT_SLACK` }}"
+
+
+#------------------------------------------------------------------------------
+# discord (discordapp.com) global notification options
+
+# multiple recipients can be given like this:
+#                  "CHANNEL1 CHANNEL2 ..."
+
+# enable/disable sending discord notifications
+SEND_DISCORD="{{ env `SEND_DISCORD` `YES` }}"
+
+# Create a webhook by following the official documentation -
+# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks
+DISCORD_WEBHOOK_URL="{{ env `DISCORD_WEBHOOK_URL` }}"
+
+# if a role's recipients are not configured, a notification will be send to
+# this discord channel (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_DISCORD="{{ env `DEFAULT_RECIPIENT_DISCORD` }}"
+
+
+#------------------------------------------------------------------------------
+# hipchat global notification options
+
+# multiple recipients can be given like this:
+#                  "ROOM1 ROOM2 ..."
+
+# enable/disable sending hipchat notifications
+SEND_HIPCHAT="{{ env `SEND_HIPCHAT` `YES` }}"
+
+# define hipchat server
+HIPCHAT_SERVER="{{ env `HIPCHAT_SERVER` `api.hipchat.com` }}"
+
+# api.hipchat.com authorization token
+# Without this, netdata cannot send hipchat notifications.
+HIPCHAT_AUTH_TOKEN="{{ env `HIPCHAT_AUTH_TOKEN` }}"
+
+# if a role's recipients are not configured, a notification will be send to
+# this hipchat room (empty = do not send a notification for unconfigured
+# roles):
+DEFAULT_RECIPIENT_HIPCHAT="{{ env `DEFAULT_RECIPIENT_HIPCHAT` }}"
+
+
+#------------------------------------------------------------------------------
+# kafka notification options
+
+# enable/disable sending kafka notifications
+SEND_KAFKA="{{ env `SEND_KAFKA` `YES` }}"
+
+# The URL to POST kafka alarm data to. It should be the full URL.
+KAFKA_URL="{{ env `KAFKA_URL` }}"
+
+# The IP to be used in the kafka message as the sender.
+KAFKA_SENDER_IP="{{ env `KAFKA_SENDER_IP` }}"
+
+
+#------------------------------------------------------------------------------
+# pagerduty.com notification options
+#
+# pagerduty.com notifications require the pagerduty agent to be installed and 
+# a "Generic API" pagerduty service.
+# https://www.pagerduty.com/docs/guides/agent-install-guide/
+
+# multiple recipients can be given like this:
+#              "<pd_service_key_1> <pd_service_key_2> ..."
+
+# enable/disable sending pagerduty notifications
+SEND_PD="{{ env `SEND_PD` `YES` }}"
+
+# if a role's recipients are not configured, a notification will be sent to
+# the "General API" pagerduty.com service that uses this service key.
+# (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_PD="{{ env `DEFAULT_RECIPIENT_PD` }}"
+
+
+#------------------------------------------------------------------------------
+# custom notifications
+#
+
+# enable/disable sending custom notifications
+SEND_CUSTOM="{{ env `SEND_CUSTOM` `YES` }}"
+
+# if a role's recipients are not configured, use the following.
+# (empty = do not send a notification for unconfigured roles)
+DEFAULT_RECIPIENT_CUSTOM="{{ env `DEFAULT_RECIPIENT_CUSTOM` }}"
+
+# The custom_sender() is a custom function to do whatever you need to do
+custom_sender() {
+    # variables you can use:
+    # ${host}               the host generated this event
+    # ${url_host}           same as ${host} but URL encoded
+    # ${unique_id}          the unique id of this event
+    # ${alarm_id}           the unique id of the alarm that generated this event
+    # ${event_id}           the incremental id of the event, for this alarm id
+    # ${when}               the timestamp this event occurred
+    # ${name}               the name of the alarm, as given in netdata health.d entries
+    # ${url_name}           same as ${name} but URL encoded
+    # ${chart}              the name of the chart (type.id)
+    # ${url_chart}          same as ${chart} but URL encoded
+    # ${family}             the family of the chart
+    # ${url_family}         same as ${family} but URL encoded
+    # ${status}             the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+    # ${old_status}         the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
+    # ${value}              the current value of the alarm
+    # ${old_value}          the previous value of the alarm
+    # ${src}                the line number and file the alarm has been configured
+    # ${duration}           the duration in seconds of the previous alarm state
+    # ${duration_txt}       same as ${duration} for humans
+    # ${non_clear_duration} the total duration in seconds this is/was non-clear
+    # ${non_clear_duration_txt} same as ${non_clear_duration} for humans
+    # ${units}              the units of the value
+    # ${info}               a short description of the alarm
+    # ${value_string}       friendly value (with units)
+    # ${old_value_string}   friendly old value (with units)
+    # ${image}              the URL of an image to represent the status of the alarm
+    # ${color}              a color in #AABBCC format for the alarm
+    # ${goto_url}           the URL the user can click to see the netdata dashboard
+
+    # these are more human friendly:
+    # ${alarm}              like "name = value units"
+    # ${status_message}     like "needs attention", "recovered", "is critical"
+    # ${severity}           like "Escalated to CRITICAL", "Recovered from WARNING"
+    # ${raised_for}         like "(alarm was raised for 10 minutes)"
+
+    # example human readable SMS
+    local msg="${host} ${status_message}: ${alarm} ${raised_for}"
+
+    # limit it to 160 characters and encode it for use in a URL
+    urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}"
+
+    # a space separated list of the recipients to send alarms to
+    to="${1}"
+
+    info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured."
+}
+
+
+###############################################################################
+# RECIPIENTS PER ROLE
+
+# -----------------------------------------------------------------------------
+# generic system alarms
+# CPU, disks, network interfaces, entropy, etc
+
+role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+# -----------------------------------------------------------------------------
+# DNS related alarms
+
+role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[domainadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+# -----------------------------------------------------------------------------
+# database servers alarms
+# mysql, redis, memcached, postgres, etc
+
+role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[dba]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+# -----------------------------------------------------------------------------
+# web servers alarms
+# apache, nginx, lighttpd, etc
+
+role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[webmaster]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"
+
+# -----------------------------------------------------------------------------
+# proxy servers alarms
+# squid, etc
+
+role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"
+
+role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"
+
+role_recipients_pushbullet[proxyadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"
+
+role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
+
+role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"
+
+role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}"
+
+role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"
+
+role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"
+
+role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"
+
+role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"
+
+role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"