netdata/templates/health_alarm_notify.conf

# Configuration for alarm notifications
#
# This configuration is used by: alarm-notify.sh
# changes take effect immediately (the next alarm will use them).
#
# alarm-notify.sh can send:
# - e-mails (using the sendmail command),
# - push notifications to your mobile phone (pushover.net),
# - messages to your slack team (slack.com),
# - messages to your discord guild (discordapp.com),
# - messages to your telegram chat / group chat (telegram.org)
# - sms messages to your cell phone or any sms enabled device (twilio.com)
# - sms messages to your cell phone or any sms enabled device (messagebird.com)
# - notifications to users on pagerduty.com
#
# The 'to' line given at netdata alarms defines a *role*, so that many
# people can be notified for each role.
#
# This file is a BASH script itself.
#
#
#------------------------------------------------------------------------------
# proxy configuration
#
# If you need to send curl based notifications (pushover, pushbullet, slack,
# discord, telegram) via a proxy, set these to your proxy address:
#export http_proxy="http://10.0.0.1:3128/"
#export https_proxy="http://10.0.0.1:3128/"


#------------------------------------------------------------------------------
# notifications images
#
# Images in notifications need to be downloaded from an Internet facing site.
# To allow notification providers fetch the icons/images, by default we set
# the URL of the global public netdata registry.
# If you have an Internet facing netdata (or you have copied the images/ folder
# of netdata to your web server), set its URL here, to fetch the notification
# images from it.
#images_base_url="http://my.public.netdata.server:19999"


#------------------------------------------------------------------------------
# external commands

# The full path to the sendmail command.
# If empty, the system $PATH will be searched for it.
# If not found, email notifications will be disabled (silently).
sendmail=""

# The full path of the curl command.
# If empty, the system $PATH will be searched for it.
# If not found, most notifications will be silently disabled.
curl=""


#------------------------------------------------------------------------------
# NOTE ABOUT RECIPIENTS
#
# When you define recipients (all types):
#
#  - emails addresses
#  - pushover user tokens
#  - telegram chat ids
#  - slack channels
#  - discord channels
#  - hipchat rooms
#  - sms phone numbers
#  - pagerduty.com (pd) services
#
# You can append |critical to limit the notifications to be sent.
#
# In these examples, the first recipient receives all the alarms
# while the second one receives only the critical ones:
#
#  email      : "user1@example.com user2@example.com|critical"
#  pushover   : "2987343...9437837 8756278...2362736|critical"
#  telegram   : "111827421 112746832|critical"
#  slack      : "alarms disasters|critical"
#  discord    : "alarms disasters|critical"
#  twilio     : "+15555555555 +17777777777|critical"
#  messagebird: "+15555555555 +17777777777|critical"
#  pd         : "<pd_service_key_1> <pd_service_key_2>|critical"
#
# If a recipient is set to empty string, the default recipient of the given
# notification method (email, pushover, telegram, slack, etc) will be used.
# To disable a notification, use the recipient called: disabled
# This works for all notification methods (including the default recipients).


#------------------------------------------------------------------------------
# email global notification options

# multiple recipients can be given like this:
#              "admin1@example.com admin2@example.com ..."

# the email address sending email notifications
# the default is the system user netdata runs as (usually: netdata)
# The following formats are supported:
# EMAIL_SENDER="user@domain"
# EMAIL_SENDER="User Name <user@domain>"
# EMAIL_SENDER="'User Name' <user@domain>"
# EMAIL_SENDER="\"User Name\" <user@domain>"
EMAIL_SENDER="{{ env `EMAIL_SENDER` }}"

# enable/disable sending emails
SEND_EMAIL="{{ env `SEND_EMAIL` `YES` }}"

# if a role recipient is not configured, an email will be send to:
DEFAULT_RECIPIENT_EMAIL="{{ env `DEFAULT_RECIPIENT_EMAIL` `root` }}"
# to receive only critical alarms, set it to "root|critical"

# Optionally specify the encoding to list in the Content-Type header.
# This doesn't change what encoding the e-mail is sent with, just what
# the headers say it was encoded as.
# This shouldn't need to be changed as it will almost always be
# autodetected from the environment.
#EMAIL_CHARSET="UTF-8"


#------------------------------------------------------------------------------
# pushover (pushover.net) global notification options

# multiple recipients can be given like this:
#                  "USERTOKEN1 USERTOKEN2 ..."

# enable/disable sending pushover notifications
SEND_PUSHOVER="{{ env `SEND_PUSHOVER` `YES` }}"

# Login to pushover.net to get your pushover app token.
# You need only one for all your netdata servers (or you can have one for
# each of your netdata - your call).
# Without an app token, netdata cannot send pushover notifications.
PUSHOVER_APP_TOKEN="{{ env `PUSHOVER_APP_TOKEN` }}"

# if a role's recipients are not configured, a notification will be send to
# this pushover user token (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_PUSHOVER="{{ env `DEFAULT_RECIPIENT_PUSHOVER` }}"


#------------------------------------------------------------------------------
# pushbullet (pushbullet.com) push notification options

# multiple recipients can be given like this:
#                  "user1@email.com user2@mail.com"

# enable/disable sending pushbullet notifications
SEND_PUSHBULLET="{{ env `SEND_PUSHBULLET` `YES` }}"

# Signup and Login to pushbullet.com
# To get your Access Token, go to https://www.pushbullet.com/#settings/account
# Create a new access token and paste it below.
# Then just set the recipients' emails.
# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does
# not have a pushbullet account, the pushbullet service will send an email
# to that address instead.

# Without an access token, netdata cannot send pushbullet notifications.
PUSHBULLET_ACCESS_TOKEN="{{ env `PUSHBULLET_ACCESS_TOKEN` }}"
DEFAULT_RECIPIENT_PUSHBULLET="{{ env `DEFAULT_RECIPIENT_PUSHBULLET` }}"


#------------------------------------------------------------------------------
# Twilio (twilio.com) SMS options

# multiple recipients can be given like this:
#                  "+15555555555 +17777777777"

# enable/disable sending twilio SMS
SEND_TWILIO="{{ env `SEND_TWILIO` `YES` }}"

# Signup for free trial and select a SMS capable Twilio Number
# To get your Account SID and Token, go to https://www.twilio.com/console
# Place your sid, token and number below.
# Then just set the recipients' phone numbers.
# The trial account is only allowed to use the number specified when set up.

# Without an account sid and token, netdata cannot send Twilio text messages.
TWILIO_ACCOUNT_SID="{{ env `TWILIO_ACCOUNT_SID` }}"
TWILIO_ACCOUNT_TOKEN="{{ env `TWILIO_ACCOUNT_TOKEN` }}"
TWILIO_NUMBER="{{ env `TWILIO_NUMBER` }}"
DEFAULT_RECIPIENT_TWILIO="{{ env `DEFAULT_RECIPIENT_TWILIO` }}"


#------------------------------------------------------------------------------
# Messagebird (messagebird.com) SMS options

# multiple recipients can be given like this:
#                  "+15555555555 +17777777777"

# enable/disable sending messagebird SMS
SEND_MESSAGEBIRD="{{ env `SEND_MESSAGEBIRD` `YES` }}"

# to get an access key, create a free account at https://www.messagebird.com
# verify and activate the account (no CC info needed)
# login to your account and enter your phonenumber to get some free credits
# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)' 
# click 'Add access key' and fill in data (you want a live key to send SMS)

# Without an access key, netdata cannot send Messagebird text messages.
MESSAGEBIRD_ACCESS_KEY="{{ env `MESSAGEBIRD_ACCESS_KEY` }}"
MESSAGEBIRD_NUMBER="{{ env `MESSAGEBIRD_NUMBER` }}"
DEFAULT_RECIPIENT_MESSAGEBIRD="{{ env `DEFAULT_RECIPIENT_MESSAGEBIRD` }}"


#------------------------------------------------------------------------------
# telegram (telegram.org) global notification options

# To get your chat ID send the command /my_id to telegram bot @get_id.
# Users also need to open a query with the bot (see below).

# note: multiple recipients can be given like this:
#                  "CHAT_ID_1 CHAT_ID_2 ..."

# enable/disable sending telegram messages
SEND_TELEGRAM="{{ env `SEND_TELEGRAM` `YES` }}"

# Contact the bot @BotFather to create a new bot and receive a bot token.
# Without it, netdata cannot send telegram messages.
TELEGRAM_BOT_TOKEN="{{ env `TELEGRAM_BOT_TOKEN` }}"

# If a role's recipients are not configured, a message will be send to
# this chat id (empty = do not send a notification for unconfigured roles):
DEFAULT_RECIPIENT_TELEGRAM="{{ env `DEFAULT_RECIPIENT_TELEGRAM` }}"


#------------------------------------------------------------------------------
# slack (slack.com) global notification options

# multiple recipients can be given like this:
#                  "CHANNEL1 CHANNEL2 ..."

# enable/disable sending slack notifications
SEND_SLACK="{{ env `SEND_SLACK` `YES` }}"

# Login to slack.com and create an incoming webhook. You need only one for all
# your netdata servers (or you can have one for each of your netdata).
# Without it, netdata cannot send slack notifications.
# Get yours from: https://api.slack.com/incoming-webhooks
SLACK_WEBHOOK_URL="{{ env `SLACK_WEBHOOK_URL` }}"

# if a role's recipients are not configured, a notification will be send to
# this slack channel (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_SLACK="{{ env `DEFAULT_RECIPIENT_SLACK` }}"


#------------------------------------------------------------------------------
# discord (discordapp.com) global notification options

# multiple recipients can be given like this:
#                  "CHANNEL1 CHANNEL2 ..."

# enable/disable sending discord notifications
SEND_DISCORD="{{ env `SEND_DISCORD` `YES` }}"

# Create a webhook by following the official documentation -
# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks
DISCORD_WEBHOOK_URL="{{ env `DISCORD_WEBHOOK_URL` }}"

# if a role's recipients are not configured, a notification will be send to
# this discord channel (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_DISCORD="{{ env `DEFAULT_RECIPIENT_DISCORD` }}"


#------------------------------------------------------------------------------
# hipchat global notification options

# multiple recipients can be given like this:
#                  "ROOM1 ROOM2 ..."

# enable/disable sending hipchat notifications
SEND_HIPCHAT="{{ env `SEND_HIPCHAT` `YES` }}"

# define hipchat server
HIPCHAT_SERVER="{{ env `HIPCHAT_SERVER` `api.hipchat.com` }}"

# api.hipchat.com authorization token
# Without this, netdata cannot send hipchat notifications.
HIPCHAT_AUTH_TOKEN="{{ env `HIPCHAT_AUTH_TOKEN` }}"

# if a role's recipients are not configured, a notification will be send to
# this hipchat room (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_HIPCHAT="{{ env `DEFAULT_RECIPIENT_HIPCHAT` }}"


#------------------------------------------------------------------------------
# kafka notification options

# enable/disable sending kafka notifications
SEND_KAFKA="{{ env `SEND_KAFKA` `YES` }}"

# The URL to POST kafka alarm data to. It should be the full URL.
KAFKA_URL="{{ env `KAFKA_URL` }}"

# The IP to be used in the kafka message as the sender.
KAFKA_SENDER_IP="{{ env `KAFKA_SENDER_IP` }}"


#------------------------------------------------------------------------------
# pagerduty.com notification options
#
# pagerduty.com notifications require the pagerduty agent to be installed and 
# a "Generic API" pagerduty service.
# https://www.pagerduty.com/docs/guides/agent-install-guide/

# multiple recipients can be given like this:
#              "<pd_service_key_1> <pd_service_key_2> ..."

# enable/disable sending pagerduty notifications
SEND_PD="{{ env `SEND_PD` `YES` }}"

# if a role's recipients are not configured, a notification will be sent to
# the "General API" pagerduty.com service that uses this service key.
# (empty = do not send a notification for unconfigured roles):
DEFAULT_RECIPIENT_PD="{{ env `DEFAULT_RECIPIENT_PD` }}"


#------------------------------------------------------------------------------
# custom notifications
#

# enable/disable sending custom notifications
SEND_CUSTOM="{{ env `SEND_CUSTOM` `YES` }}"

# if a role's recipients are not configured, use the following.
# (empty = do not send a notification for unconfigured roles)
DEFAULT_RECIPIENT_CUSTOM="{{ env `DEFAULT_RECIPIENT_CUSTOM` }}"

# The custom_sender() is a custom function to do whatever you need to do
custom_sender() {
    # variables you can use:
    # ${host}               the host generated this event
    # ${url_host}           same as ${host} but URL encoded
    # ${unique_id}          the unique id of this event
    # ${alarm_id}           the unique id of the alarm that generated this event
    # ${event_id}           the incremental id of the event, for this alarm id
    # ${when}               the timestamp this event occurred
    # ${name}               the name of the alarm, as given in netdata health.d entries
    # ${url_name}           same as ${name} but URL encoded
    # ${chart}              the name of the chart (type.id)
    # ${url_chart}          same as ${chart} but URL encoded
    # ${family}             the family of the chart
    # ${url_family}         same as ${family} but URL encoded
    # ${status}             the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
    # ${old_status}         the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
    # ${value}              the current value of the alarm
    # ${old_value}          the previous value of the alarm
    # ${src}                the line number and file the alarm has been configured
    # ${duration}           the duration in seconds of the previous alarm state
    # ${duration_txt}       same as ${duration} for humans
    # ${non_clear_duration} the total duration in seconds this is/was non-clear
    # ${non_clear_duration_txt} same as ${non_clear_duration} for humans
    # ${units}              the units of the value
    # ${info}               a short description of the alarm
    # ${value_string}       friendly value (with units)
    # ${old_value_string}   friendly old value (with units)
    # ${image}              the URL of an image to represent the status of the alarm
    # ${color}              a color in #AABBCC format for the alarm
    # ${goto_url}           the URL the user can click to see the netdata dashboard

    # these are more human friendly:
    # ${alarm}              like "name = value units"
    # ${status_message}     like "needs attention", "recovered", "is critical"
    # ${severity}           like "Escalated to CRITICAL", "Recovered from WARNING"
    # ${raised_for}         like "(alarm was raised for 10 minutes)"

    # example human readable SMS
    local msg="${host} ${status_message}: ${alarm} ${raised_for}"

    # limit it to 160 characters and encode it for use in a URL
    urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}"

    # a space separated list of the recipients to send alarms to
    to="${1}"

    info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured."
}


###############################################################################
# RECIPIENTS PER ROLE

# -----------------------------------------------------------------------------
# generic system alarms
# CPU, disks, network interfaces, entropy, etc

role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# DNS related alarms

role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[domainadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# database servers alarms
# mysql, redis, memcached, postgres, etc

role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[dba]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# web servers alarms
# apache, nginx, lighttpd, etc

role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[webmaster]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# proxy servers alarms
# squid, etc

role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[proxyadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"