Enable Unified Alerting and wire up with alertmanager

Grafana v8.2.0 added support for Prometheus's Alertmanager[1] which can be used with the Unified Alerting added in v8.0.0[2].

[1] https://github.com/grafana/grafana/releases/tag/v8.2.0 [2] https://grafana.com/blog/2021/06/08/grafana-8.0-unified-grafana-and-prometheus-alerts-live-streaming-new-visualizations-and-more/

roles/grafana/templates/grafana.ini.j2

@@ -67,6 +67,13 @@ enable_gzip = true
 # Unix socket path
 ;socket =
 
+# CDN Url
+;cdn_url =
+
+# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections.
+# `0` means there is no timeout for reading the request.
+;read_timeout = 0
+
 #################################### Database ####################################
 [database]
 # You can configure the database connection by specifying type, host, name, user and password
@@ -87,6 +94,12 @@ enable_gzip = true
 # For "postgres" only, either "disable", "require" or "verify-full"
 ;ssl_mode = disable
 
+# Database drivers may support different transaction isolation levels.
+# Currently, only "mysql" driver supports isolation levels.
+# If the value is empty - driver's default isolation level is applied.
+# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE".
+;isolation_level =
+
 ;ca_cert_path =
 ;client_key_path =
 ;client_cert_path =
@@ -151,18 +164,26 @@ enable_gzip = true
 # waiting for the server to approve.
 ;expect_continue_timeout_seconds = 1
 
+# Optionally limits the total number of connections per host, including connections in the dialing,
+# active, and idle states. On limit violation, dials will block.
+# A value of zero (0) means no limit.
+;max_conns_per_host = 0
+
 # The maximum number of idle connections that Grafana will keep alive.
 ;max_idle_connections = 100
 
-# The maximum number of idle connections per host that Grafana will keep alive.
-;max_idle_connections_per_host = 2
-
 # How many seconds the data proxy keeps an idle connection open before timing out.
 ;idle_conn_timeout_seconds = 90
 
 # If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
 ;send_user_header = false
 
+# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests.
+;response_limit = 0
+
+# Limits the number of rows that Grafana will process from SQL data sources.
+;row_limit = 1000000
+
 #################################### Analytics ####################################
 [analytics]
 # Server reporting, sends usage counters to stats.grafana.org every 24 hours.
@@ -236,12 +257,12 @@ strict_transport_security_max_age_seconds = 86400
 
 # Set to true to enable the X-Content-Type-Options response header.
 # The X-Content-Type-Options response HTTP header is a marker used by the server to indicate that the MIME types advertised
-# in the Content-Type headers should not be changed and be followed. The default will change to true in the next minor release, 6.3.
-x_content_type_options = true
+# in the Content-Type headers should not be changed and be followed.
+;x_content_type_options = true
 
 # Set to true to enable the X-XSS-Protection header, which tells browsers to stop pages from loading
-# when they detect reflected cross-site scripting (XSS) attacks. The default will change to true in the next minor release, 6.3.
-x_xss_protection = true
+# when they detect reflected cross-site scripting (XSS) attacks.
+;x_xss_protection = true
 
 # Enable adding the Content-Security-Policy header to your requests.
 # CSP allows to control resources the user agent is allowed to load and helps prevent XSS attacks.
@@ -378,7 +399,7 @@ org_role = Viewer
 # mask the Grafana version number for unauthenticated users
 ;hide_version = false
 
-#################################### Github Auth ##########################
+#################################### GitHub Auth ##########################
 [auth.github]
 ;enabled = false
 ;allow_sign_up = true
@@ -474,58 +495,20 @@ email_attribute_path = email
 auth_url = https://accounts.archlinux.org/auth/realms/archlinux/protocol/openid-connect/auth
 token_url = https://accounts.archlinux.org/auth/realms/archlinux/protocol/openid-connect/token
 api_url = https://accounts.archlinux.org/auth/realms/archlinux/protocol/openid-connect/userinfo
+;teams_url =
 ;allowed_domains =
 ;team_ids =
 ;allowed_organizations =
 role_attribute_path: contains(roles[*], 'DevOps') && 'Admin'
 role_attribute_strict = true
+;groups_attribute_path =
+;team_ids_attribute_path =
 ;tls_skip_verify_insecure = false
 ;tls_client_cert =
 ;tls_client_key =
 ;tls_client_ca =
 {% endif %}
 
-#################################### SAML Auth ###########################
-[auth.saml] # Enterprise only
-# Defaults to false. If true, the feature is enabled.
-;enabled = false
-
-# Base64-encoded public X.509 certificate. Used to sign requests to the IdP
-;certificate =
-
-# Path to the public X.509 certificate. Used to sign requests to the IdP
-;certificate_path =
-
-# Base64-encoded private key. Used to decrypt assertions from the IdP
-;private_key =
-
-;# Path to the private key. Used to decrypt assertions from the IdP
-;private_key_path =
-
-# Base64-encoded IdP SAML metadata XML. Used to verify and obtain binding locations from the IdP
-;idp_metadata =
-
-# Path to the SAML metadata XML. Used to verify and obtain binding locations from the IdP
-;idp_metadata_path =
-
-# URL to fetch SAML IdP metadata. Used to verify and obtain binding locations from the IdP
-;idp_metadata_url =
-
-# Duration, since the IdP issued a response and the SP is allowed to process it. Defaults to 90 seconds.
-;max_issue_delay = 90s
-
-# Duration, for how long the SP's metadata should be valid. Defaults to 48 hours.
-;metadata_valid_duration = 48h
-
-# Friendly name or name of the attribute within the SAML assertion to use as the user's name
-;assertion_attribute_name = displayName
-
-# Friendly name or name of the attribute within the SAML assertion to use as the user's login handle
-;assertion_attribute_login = mail
-
-# Friendly name or name of the attribute within the SAML assertion to use as the user's email
-;assertion_attribute_email = mail
-
 #################################### Basic Auth ##########################
 [auth.basic]
 ;enabled = true
@@ -610,7 +593,8 @@ role_attribute_strict = true
 
 [emails]
 ;welcome_email_on_sign_up = false
-;templates_pattern = emails/*.html
+;templates_pattern = emails/*.html, emails/*.txt
+;content_types = text/html
 
 #################################### Logging ##########################
 [log]
@@ -729,11 +713,67 @@ mode = syslog
 # global limit of alerts
 ;global_alert_rule = -1
 
+#################################### Unified Alerting ####################
+[unified_alerting]
+#Enable the Unified Alerting sub-system and interface. When enabled we'll migrate all of your alert rules and notification channels to the new system. New alert rules will be created and your notification channels will be converted into an Alertmanager configuration. Previous data is preserved to enable backwards compatibility but new data is removed.```
+;enabled = false
+
+# Comma-separated list of organization IDs for which to disable unified alerting. Only supported if unified alerting is enabled.
+;disabled_orgs = 
+
+# Specify the frequency of polling for admin config changes.
+# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;admin_config_poll_interval = 60s
+
+# Specify the frequency of polling for Alertmanager config changes.
+# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;alertmanager_config_poll_interval = 60s
+
+# Listen address/hostname and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. The default value is `0.0.0.0:9094`.
+;ha_listen_address = "0.0.0.0:9094"
+
+# Listen address/hostname and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. The default value is `0.0.0.0:9094`.
+;ha_advertise_address = ""
+
+# Comma-separated list of initial instances (in a format of host:port) that will form the HA cluster. Configuring this setting will enable High Availability mode for alerting.
+;ha_peers = ""
+
+# Time to wait for an instance to send a notification via the Alertmanager. In HA, each Grafana instance will
+# be assigned a position (e.g. 0, 1). We then multiply this position with the timeout to indicate how long should
+# each instance wait before sending the notification to take into account replication lag.
+# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;ha_peer_timeout = "15s"
+
+# The interval between sending gossip messages. By lowering this value (more frequent) gossip messages are propagated
+# across cluster more quickly at the expense of increased bandwidth usage.
+# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;ha_gossip_interval = "200ms"
+
+# The interval between gossip full state syncs. Setting this interval lower (more frequent) will increase convergence speeds
+# across larger clusters at the expense of increased bandwidth usage.
+# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;ha_push_pull_interval = "60s"
+
+# Enable or disable alerting rule execution. The alerting UI remains visible. This option has a legacy version in the `[alerting]` section that takes precedence.
+;execute_alerts = true
+
+# Alert evaluation timeout when fetching data from the datasource. This option has a legacy version in the `[alerting]` section that takes precedence.
+# The timeout string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;evaluation_timeout = 30s
+
+# Number of times we'll attempt to evaluate an alert rule before giving up on that evaluation. This option has a legacy version in the `[alerting]` section that takes precedence.
+;max_attempts = 3
+
+# Minimum interval to enforce between rule evaluations. Rules will be adjusted if they are less than this value  or if they are not multiple of the scheduler interval (10s). Higher values can help with resource management as we'll schedule fewer evaluations over time. This option has a legacy version in the `[alerting]` section that takes precedence.
+# The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
+;min_interval = 10s
+
 #################################### Alerting ############################
 [alerting]
-# Disable alerting engine & UI features
+# Disable legacy alerting engine & UI features
 enabled = false
-# Makes it possible to turn off alert rule execution but alerting UI is visible
+
+# Makes it possible to turn off alert execution but alerting UI is visible
 ;execute_alerts = true
 
 # Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
@@ -746,7 +786,6 @@ enabled = false
 # This limit will protect the server from render overloading and make sure notifications are sent out quickly
 ;concurrent_render_limit = 5
 
-
 # Default setting for alert calculation timeout. Default value is 30
 ;evaluation_timeout_seconds = 30
 
@@ -758,6 +797,7 @@ enabled = false
 
 # Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
 ;min_interval_seconds = 1
+
 # Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
 # This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
 ;max_annotation_age =
@@ -794,7 +834,7 @@ enabled = false
 #################################### Explore #############################
 [explore]
 # Enable the Explore section
-;enabled = false
+;enabled = true
 
 #################################### Internal Grafana Metrics ##########################
 # Metrics available at HTTP API Url /metrics
@@ -902,13 +942,34 @@ enabled = false
 [plugins]
 ;enable_alpha = false
 ;app_tls_skip_verify_insecure = false
-# Enter a comma-separated list of plugin identifiers to identify plugins that are allowed to be loaded even if they lack a valid signature.
+# Enter a comma-separated list of plugin identifiers to identify plugins to load even if they are unsigned. Plugins with modified signatures are never loaded.
 ;allow_loading_unsigned_plugins =
 # Enable or disable installing plugins directly from within Grafana.
 ;plugin_admin_enabled = false
 ;plugin_admin_external_manage_enabled = false
 ;plugin_catalog_url = https://grafana.com/grafana/plugins/
 
+#################################### Grafana Live ##########################################
+[live]
+# max_connections to Grafana Live WebSocket endpoint per Grafana server instance. See Grafana Live docs
+# if you are planning to make it higher than default 100 since this can require some OS and infrastructure
+# tuning. 0 disables Live, -1 means unlimited connections.
+;max_connections = 100
+
+# allowed_origins is a comma-separated list of origins that can establish connection with Grafana Live.
+# If not set then origin will be matched over root_url. Supports wildcard symbol "*".
+;allowed_origins =
+
+# engine defines an HA (high availability) engine to use for Grafana Live. By default no engine used - in
+# this case Live features work only on a single Grafana server. Available options: "redis".
+# Setting ha_engine is an EXPERIMENTAL feature.
+;ha_engine =
+
+# ha_engine_address sets a connection address for Live HA engine. Depending on engine type address format can differ.
+# For now we only support Redis connection address in "host:port" format.
+# This option is EXPERIMENTAL.
+;ha_engine_address = "127.0.0.1:6379"
+
 #################################### Grafana Image Renderer Plugin ##########################
 [plugin.grafana-image-renderer]
 # Instruct headless browser instance to use a default timezone when not provided by Grafana, e.g. when rendering panel image of alert.