From a1d4557c2115b4177b45a406dc8dc93425a65773 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Thu, 31 Aug 2017 11:19:25 +0200 Subject: [PATCH] Improve configuration generation Signed-off-by: Knut Ahlers --- README.md | 18 +- build.sh | 5 + run.sh | 39 +-- templates/health_alarm_notify.conf | 498 +++++++++++++++++++++++++++++ 4 files changed, 528 insertions(+), 32 deletions(-) create mode 100644 templates/health_alarm_notify.conf diff --git a/README.md b/README.md index 0952be4..0b70289 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,21 @@ # Luzifer-Docker / netdata -This repository contains a dockerized version of the [netdata](https://github.com/firehol/netdata) daemon. It supports adding and overriding configurations and plugins through overrides. +This repository contains a dockerized version of the [netdata](https://github.com/firehol/netdata) daemon. It supports adding and overriding configurations and plugins through overrides and partially configuration through environment variables. -The expected structure on the `/override` volume mount is as following: +## Usage + +``` +docker run -d --cap-add SYS_PTRACE \ + -v /proc:/host/proc:ro \ + -v /sys:/host/sys:ro \ + -p 19999:19999 quay.io/luzifer/netdata +``` + +## Configuration + +To configure alerts have a look at the [`health_alarm_notify.conf` template](templates/health_alarm_notify.conf). There you can see all variable names you need to specify as environment variables. + +When using the override mount you can add configuration files and plugins. The expected structure on the `/override` volume mount is as following: ``` /override @@ -12,3 +25,4 @@ The expected structure on the `/override` volume mount is as following: ├── plugins.d └── python.d ``` + diff --git a/build.sh b/build.sh index fdacbc3..97d74e9 100755 --- a/build.sh +++ b/build.sh @@ -42,3 +42,8 @@ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ln -sf /dev/stdout /var/log/netdata/access.log ln -sf /dev/stdout /var/log/netdata/debug.log ln -sf /dev/stderr /var/log/netdata/error.log + +# Install korvike for template substitution +curl -sSLfo /usr/local/bin/korvike https://github.com/Luzifer/korvike/releases/download/v0.4.1/korvike_linux_amd64 +chmod +x /usr/local/bin/korvike +echo 'f791fecdc62b2e2ff07342b41fdf165ee40c2a8a286f1c2c0f48228b982e2953 /usr/local/bin/korvike' | sha256sum -c diff --git a/run.sh b/run.sh index 9d2f3e4..c0aca29 100755 --- a/run.sh +++ b/run.sh @@ -25,30 +25,6 @@ root:netdata@$SSMTP_HOSTNAME:$SSMTP_SERVER:$SSMTP_PORT EOF fi -if [[ $SLACK_WEBHOOK_URL ]]; then - sed -i -e "s@SLACK_WEBHOOK_URL=\"\"@SLACK_WEBHOOK_URL=\"${SLACK_WEBHOOK_URL}\"@" /etc/netdata/health_alarm_notify.conf -fi - -if [[ $SLACK_CHANNEL ]]; then - sed -i -e "s@DEFAULT_RECIPIENT_SLACK=\"\"@DEFAULT_RECIPIENT_SLACK=\"${SLACK_CHANNEL}\"@" /etc/netdata/health_alarm_notify.conf -fi - -if [[ $TELEGRAM_BOT_TOKEN ]]; then - sed -i -e "s@TELEGRAM_BOT_TOKEN=\"\"@TELEGRAM_BOT_TOKEN=\"${TELEGRAM_BOT_TOKEN}\"@" /etc/netdata/health_alarm_notify.conf -fi - -if [[ $TELEGRAM_CHAT_ID ]]; then - sed -i -e "s@DEFAULT_RECIPIENT_TELEGRAM=\"\"@DEFAULT_RECIPIENT_TELEGRAM=\"${TELEGRAM_CHAT_ID}\"@" /etc/netdata/health_alarm_notify.conf -fi - -if [[ $PUSHBULLET_ACCESS_TOKEN ]]; then - sed -i -e "s@PUSHBULLET_ACCESS_TOKEN=\"\"@PUSHBULLET_ACCESS_TOKEN=\"${PUSHBULLET_ACCESS_TOKEN}\"@" /etc/netdata/health_alarm_notify.conf -fi - -if [[ $PUSHBULLET_DEFAULT_EMAIL ]]; then - sed -i -e "s#DEFAULT_RECIPIENT_PUSHBULLET=\"\"#DEFAULT_RECIPIENT_PUSHBULLET=\"${PUSHBULLET_DEFAULT_EMAIL}\"#" /etc/netdata/health_alarm_notify.conf -fi - if [[ $NETDATA_IP ]]; then NETDATA_ARGS="${NETDATA_ARGS} -i ${NETDATA_IP}" fi @@ -72,6 +48,15 @@ if printenv | grep -q 'NETDATA_API_KEY_ENABLE_'; then printenv | grep -oe 'NETDATA_API_KEY_ENABLE_[^=]\+' | sed 's/NETDATA_API_KEY_ENABLE_//' | xargs -n1 -I{} echo '['{}$']\n\tenabled = yes' >> /etc/netdata/stream.conf fi +# Execute templates +korvike -i /src/templates/health_alarm_notify.conf -o /etc/netdata/health_alarm_notify.conf + +# Pull in overrides and additions for config and plugins +[ -e /override/conf.d ] && rsync -arv /override/conf.d/ /etc/netdata/ +for dir in charts.d node.d plugins.d python.d; do + [ -e "/override/${dir}" ] && rsync -arv "/override/${dir}/" "/usr/libexec/netdata/${dir}/" +done + # exec custom command if [[ $# -gt 0 ]] ; then exec "$@" @@ -87,11 +72,5 @@ if [[ -d "/fakenet/" ]]; then sleep 1 fi -# Pull in overrides and additions for config and plugins -[ -e /override/conf.d ] && rsync -arv /override/conf.d/ /etc/netdata/ -for dir in charts.d node.d plugins.d python.d; do - [ -e "/override/${dir}" ] && rsync -arv "/override/${dir}/" "/usr/libexec/netdata/${dir}/" -done - # main entrypoint exec /usr/sbin/netdata -D -u root -s /host -p ${NETDATA_PORT} ${NETDATA_ARGS} diff --git a/templates/health_alarm_notify.conf b/templates/health_alarm_notify.conf new file mode 100644 index 0000000..36e1c1a --- /dev/null +++ b/templates/health_alarm_notify.conf @@ -0,0 +1,498 @@ +# Configuration for alarm notifications +# +# This configuration is used by: alarm-notify.sh +# changes take effect immediately (the next alarm will use them). +# +# alarm-notify.sh can send: +# - e-mails (using the sendmail command), +# - push notifications to your mobile phone (pushover.net), +# - messages to your slack team (slack.com), +# - messages to your discord guild (discordapp.com), +# - messages to your telegram chat / group chat (telegram.org) +# - sms messages to your cell phone or any sms enabled device (twilio.com) +# - sms messages to your cell phone or any sms enabled device (messagebird.com) +# - notifications to users on pagerduty.com +# +# The 'to' line given at netdata alarms defines a *role*, so that many +# people can be notified for each role. +# +# This file is a BASH script itself. +# +# +#------------------------------------------------------------------------------ +# proxy configuration +# +# If you need to send curl based notifications (pushover, pushbullet, slack, +# discord, telegram) via a proxy, set these to your proxy address: +#export http_proxy="http://10.0.0.1:3128/" +#export https_proxy="http://10.0.0.1:3128/" + + +#------------------------------------------------------------------------------ +# notifications images +# +# Images in notifications need to be downloaded from an Internet facing site. +# To allow notification providers fetch the icons/images, by default we set +# the URL of the global public netdata registry. +# If you have an Internet facing netdata (or you have copied the images/ folder +# of netdata to your web server), set its URL here, to fetch the notification +# images from it. +#images_base_url="http://my.public.netdata.server:19999" + + +#------------------------------------------------------------------------------ +# external commands + +# The full path to the sendmail command. +# If empty, the system $PATH will be searched for it. +# If not found, email notifications will be disabled (silently). +sendmail="" + +# The full path of the curl command. +# If empty, the system $PATH will be searched for it. +# If not found, most notifications will be silently disabled. +curl="" + + +#------------------------------------------------------------------------------ +# NOTE ABOUT RECIPIENTS +# +# When you define recipients (all types): +# +# - emails addresses +# - pushover user tokens +# - telegram chat ids +# - slack channels +# - discord channels +# - hipchat rooms +# - sms phone numbers +# - pagerduty.com (pd) services +# +# You can append |critical to limit the notifications to be sent. +# +# In these examples, the first recipient receives all the alarms +# while the second one receives only the critical ones: +# +# email : "user1@example.com user2@example.com|critical" +# pushover : "2987343...9437837 8756278...2362736|critical" +# telegram : "111827421 112746832|critical" +# slack : "alarms disasters|critical" +# discord : "alarms disasters|critical" +# twilio : "+15555555555 +17777777777|critical" +# messagebird: "+15555555555 +17777777777|critical" +# pd : " |critical" +# +# If a recipient is set to empty string, the default recipient of the given +# notification method (email, pushover, telegram, slack, etc) will be used. +# To disable a notification, use the recipient called: disabled +# This works for all notification methods (including the default recipients). + + +#------------------------------------------------------------------------------ +# email global notification options + +# multiple recipients can be given like this: +# "admin1@example.com admin2@example.com ..." + +# enable/disable sending emails +SEND_EMAIL="{{ env `SEND_EMAIL` `YES` }}" + +# if a role recipient is not configured, an email will be send to: +DEFAULT_RECIPIENT_EMAIL="{{ env `DEFAULT_RECIPIENT_EMAIL` `root` }}" +# to receive only critical alarms, set it to "root|critical" + + +#------------------------------------------------------------------------------ +# pushover (pushover.net) global notification options + +# multiple recipients can be given like this: +# "USERTOKEN1 USERTOKEN2 ..." + +# enable/disable sending pushover notifications +SEND_PUSHOVER="{{ env `SEND_PUSHOVER` `YES` }}" + +# Login to pushover.net to get your pushover app token. +# You need only one for all your netdata servers (or you can have one for +# each of your netdata - your call). +# Without an app token, netdata cannot send pushover notifications. +PUSHOVER_APP_TOKEN="{{ env `PUSHOVER_APP_TOKEN` }}" + +# if a role's recipients are not configured, a notification will be send to +# this pushover user token (empty = do not send a notification for unconfigured +# roles): +DEFAULT_RECIPIENT_PUSHOVER="{{ env `DEFAULT_RECIPIENT_PUSHOVER` }}" + + +#------------------------------------------------------------------------------ +# pushbullet (pushbullet.com) push notification options + +# multiple recipients can be given like this: +# "user1@email.com user2@mail.com" + +# enable/disable sending pushbullet notifications +SEND_PUSHBULLET="{{ env `SEND_PUSHBULLET` `YES` }}" + +# Signup and Login to pushbullet.com +# To get your Access Token, go to https://www.pushbullet.com/#settings/account +# Create a new access token and paste it below. +# Then just set the recipients' emails. +# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does +# not have a pushbullet account, the pushbullet service will send an email +# to that address instead. + +# Without an access token, netdata cannot send pushbullet notifications. +PUSHBULLET_ACCESS_TOKEN="{{ env `PUSHBULLET_ACCESS_TOKEN` }}" +DEFAULT_RECIPIENT_PUSHBULLET="{{ env `DEFAULT_RECIPIENT_PUSHBULLET` }}" + + +#------------------------------------------------------------------------------ +# Twilio (twilio.com) SMS options + +# multiple recipients can be given like this: +# "+15555555555 +17777777777" + +# enable/disable sending twilio SMS +SEND_TWILIO="{{ env `SEND_TWILIO` `YES` }}" + +# Signup for free trial and select a SMS capable Twilio Number +# To get your Account SID and Token, go to https://www.twilio.com/console +# Place your sid, token and number below. +# Then just set the recipients' phone numbers. +# The trial account is only allowed to use the number specified when set up. + +# Without an account sid and token, netdata cannot send Twilio text messages. +TWILIO_ACCOUNT_SID="{{ env `TWILIO_ACCOUNT_SID` }}" +TWILIO_ACCOUNT_TOKEN="{{ env `TWILIO_ACCOUNT_TOKEN` }}" +TWILIO_NUMBER="{{ env `TWILIO_NUMBER` }}" +DEFAULT_RECIPIENT_TWILIO="{{ env `DEFAULT_RECIPIENT_TWILIO` }}" + + +#------------------------------------------------------------------------------ +# Messagebird (messagebird.com) SMS options + +# multiple recipients can be given like this: +# "+15555555555 +17777777777" + +# enable/disable sending messagebird SMS +SEND_MESSAGEBIRD="{{ env `SEND_MESSAGEBIRD` `YES` }}" + +# to get an access key, create a free account at https://www.messagebird.com +# verify and activate the account (no CC info needed) +# login to your account and enter your phonenumber to get some free credits +# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' +# click 'Add access key' and fill in data (you want a live key to send SMS) + +# Without an access key, netdata cannot send Messagebird text messages. +MESSAGEBIRD_ACCESS_KEY="{{ env `MESSAGEBIRD_ACCESS_KEY` }}" +MESSAGEBIRD_NUMBER="{{ env `MESSAGEBIRD_NUMBER` }}" +DEFAULT_RECIPIENT_MESSAGEBIRD="{{ env `DEFAULT_RECIPIENT_MESSAGEBIRD` }}" + + +#------------------------------------------------------------------------------ +# telegram (telegram.org) global notification options + +# To get your chat ID send the command /my_id to telegram bot @get_id. +# Users also need to open a query with the bot (see below). + +# note: multiple recipients can be given like this: +# "CHAT_ID_1 CHAT_ID_2 ..." + +# enable/disable sending telegram messages +SEND_TELEGRAM="{{ env `SEND_TELEGRAM` `YES` }}" + +# Contact the bot @BotFather to create a new bot and receive a bot token. +# Without it, netdata cannot send telegram messages. +TELEGRAM_BOT_TOKEN="{{ env `TELEGRAM_BOT_TOKEN` }}" + +# If a role's recipients are not configured, a message will be send to +# this chat id (empty = do not send a notification for unconfigured roles): +DEFAULT_RECIPIENT_TELEGRAM="{{ env `DEFAULT_RECIPIENT_TELEGRAM` }}" + + +#------------------------------------------------------------------------------ +# slack (slack.com) global notification options + +# multiple recipients can be given like this: +# "CHANNEL1 CHANNEL2 ..." + +# enable/disable sending slack notifications +SEND_SLACK="{{ env `SEND_SLACK` `YES` }}" + +# Login to slack.com and create an incoming webhook. You need only one for all +# your netdata servers (or you can have one for each of your netdata). +# Without it, netdata cannot send slack notifications. +# Get yours from: https://api.slack.com/incoming-webhooks +SLACK_WEBHOOK_URL="{{ env `SLACK_WEBHOOK_URL` }}" + +# if a role's recipients are not configured, a notification will be send to +# this slack channel (empty = do not send a notification for unconfigured +# roles): +DEFAULT_RECIPIENT_SLACK="{{ env `DEFAULT_RECIPIENT_SLACK` }}" + + +#------------------------------------------------------------------------------ +# discord (discordapp.com) global notification options + +# multiple recipients can be given like this: +# "CHANNEL1 CHANNEL2 ..." + +# enable/disable sending discord notifications +SEND_DISCORD="{{ env `SEND_DISCORD` `YES` }}" + +# Create a webhook by following the official documentation - +# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks +DISCORD_WEBHOOK_URL="{{ env `DISCORD_WEBHOOK_URL` }}" + +# if a role's recipients are not configured, a notification will be send to +# this discord channel (empty = do not send a notification for unconfigured +# roles): +DEFAULT_RECIPIENT_DISCORD="{{ env `DEFAULT_RECIPIENT_DISCORD` }}" + + +#------------------------------------------------------------------------------ +# hipchat global notification options + +# multiple recipients can be given like this: +# "ROOM1 ROOM2 ..." + +# enable/disable sending hipchat notifications +SEND_HIPCHAT="{{ env `SEND_HIPCHAT` `YES` }}" + +# define hipchat server +HIPCHAT_SERVER="{{ env `HIPCHAT_SERVER` `api.hipchat.com` }}" + +# api.hipchat.com authorization token +# Without this, netdata cannot send hipchat notifications. +HIPCHAT_AUTH_TOKEN="{{ env `HIPCHAT_AUTH_TOKEN` }}" + +# if a role's recipients are not configured, a notification will be send to +# this hipchat room (empty = do not send a notification for unconfigured +# roles): +DEFAULT_RECIPIENT_HIPCHAT="{{ env `DEFAULT_RECIPIENT_HIPCHAT` }}" + + +#------------------------------------------------------------------------------ +# kafka notification options + +# enable/disable sending kafka notifications +SEND_KAFKA="{{ env `SEND_KAFKA` `YES` }}" + +# The URL to POST kafka alarm data to. It should be the full URL. +KAFKA_URL="{{ env `KAFKA_URL` }}" + +# The IP to be used in the kafka message as the sender. +KAFKA_SENDER_IP="{{ env `KAFKA_SENDER_IP` }}" + + +#------------------------------------------------------------------------------ +# pagerduty.com notification options +# +# pagerduty.com notifications require the pagerduty agent to be installed and +# a "Generic API" pagerduty service. +# https://www.pagerduty.com/docs/guides/agent-install-guide/ + +# multiple recipients can be given like this: +# " ..." + +# enable/disable sending pagerduty notifications +SEND_PD="{{ env `SEND_PD` `YES` }}" + +# if a role's recipients are not configured, a notification will be sent to +# the "General API" pagerduty.com service that uses this service key. +# (empty = do not send a notification for unconfigured roles): +DEFAULT_RECIPIENT_PD="{{ env `DEFAULT_RECIPIENT_PD` }}" + + +#------------------------------------------------------------------------------ +# custom notifications +# + +# enable/disable sending custom notifications +SEND_CUSTOM="{{ env `SEND_CUSTOM` `YES` }}" + +# if a role's recipients are not configured, use the following. +# (empty = do not send a notification for unconfigured roles) +DEFAULT_RECIPIENT_CUSTOM="{{ env `DEFAULT_RECIPIENT_CUSTOM` }}" + +# The custom_sender() is a custom function to do whatever you need to do +custom_sender() { + # variables you can use: + # ${host} the host generated this event + # ${url_host} same as ${host} but URL encoded + # ${unique_id} the unique id of this event + # ${alarm_id} the unique id of the alarm that generated this event + # ${event_id} the incremental id of the event, for this alarm id + # ${when} the timestamp this event occurred + # ${name} the name of the alarm, as given in netdata health.d entries + # ${url_name} same as ${name} but URL encoded + # ${chart} the name of the chart (type.id) + # ${url_chart} same as ${chart} but URL encoded + # ${family} the family of the chart + # ${url_family} same as ${family} but URL encoded + # ${status} the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + # ${old_status} the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL + # ${value} the current value of the alarm + # ${old_value} the previous value of the alarm + # ${src} the line number and file the alarm has been configured + # ${duration} the duration in seconds of the previous alarm state + # ${duration_txt} same as ${duration} for humans + # ${non_clear_duration} the total duration in seconds this is/was non-clear + # ${non_clear_duration_txt} same as ${non_clear_duration} for humans + # ${units} the units of the value + # ${info} a short description of the alarm + # ${value_string} friendly value (with units) + # ${old_value_string} friendly old value (with units) + # ${image} the URL of an image to represent the status of the alarm + # ${color} a color in #AABBCC format for the alarm + # ${goto_url} the URL the user can click to see the netdata dashboard + + # these are more human friendly: + # ${alarm} like "name = value units" + # ${status_message} like "needs attention", "recovered", "is critical" + # ${severity} like "Escalated to CRITICAL", "Recovered from WARNING" + # ${raised_for} like "(alarm was raised for 10 minutes)" + + # example human readable SMS + local msg="${host} ${status_message}: ${alarm} ${raised_for}" + + # limit it to 160 characters and encode it for use in a URL + urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}" + + # a space separated list of the recipients to send alarms to + to="${1}" + + info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured." +} + + +############################################################################### +# RECIPIENTS PER ROLE + +# ----------------------------------------------------------------------------- +# generic system alarms +# CPU, disks, network interfaces, entropy, etc + +role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}" + +role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}" + +role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}" + +role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" + +role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}" + +role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}" + +role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" + +role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}" + +role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" + +role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}" + +role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}" + +# ----------------------------------------------------------------------------- +# DNS related alarms + +role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}" + +role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}" + +role_recipients_pushbullet[domainadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}" + +role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" + +role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}" + +role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}" + +role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" + +role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}" + +role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" + +role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}" + +role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}" + +# ----------------------------------------------------------------------------- +# database servers alarms +# mysql, redis, memcached, postgres, etc + +role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}" + +role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}" + +role_recipients_pushbullet[dba]="${DEFAULT_RECIPIENT_PUSHBULLET}" + +role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}" + +role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}" + +role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}" + +role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}" + +role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}" + +role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" + +role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}" + +role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}" + +# ----------------------------------------------------------------------------- +# web servers alarms +# apache, nginx, lighttpd, etc + +role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}" + +role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}" + +role_recipients_pushbullet[webmaster]="${DEFAULT_RECIPIENT_PUSHBULLET}" + +role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}" + +role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}" + +role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}" + +role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}" + +role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}" + +role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" + +role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}" + +role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}" + +# ----------------------------------------------------------------------------- +# proxy servers alarms +# squid, etc + +role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}" + +role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}" + +role_recipients_pushbullet[proxyadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}" + +role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}" + +role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}" + +role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}" + +role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}" + +role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}" + +role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}" + +role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}" + +role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"