prometheus: Add receive only mode and remote_write metrics to dashboards.al.org

playbooks/dashboards.archlinux.org.yml

@@ -9,7 +9,7 @@
     - { role: root_ssh }
     - { role: hardening }
     - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
-    - { role: prometheus }
+    - { role: prometheus, prometheus_receive_only: true }
     - { role: prometheus_exporters }
     - { role: promtail }
     - { role: certbot }

roles/prometheus/defaults/main.yml

 monitoring_domain: monitoring.archlinux.org
 gitlab_runner_exporter_port: '9252'
+prometheus_domain: dashboards.archlinux.org
 prometheus_mysqld_exporter_port: '9104'
+prometheus_receive_only: false
 
 # for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/  - https:\/\//"
 blackbox_targets:
@@ -57,6 +59,7 @@ blackbox_targets:
     - mail.archlinux.org:465
     - mail.archlinux.org:993
     - mail.archlinux.org:995
+    - dashboards.archlinux.org:9090
   smtp_starttls:
     - mail.archlinux.org:25
     - mail.archlinux.org:587

roles/prometheus/files/prometheus.conf

-PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"

roles/prometheus/tasks/main.yml

@@ -3,17 +3,40 @@
 - name: install prometheus server
   pacman: name=prometheus state=present
 
+- name: install cert renewal hook
+  template: src=letsencrypt.hook.d.j2 dest=/etc/letsencrypt/renewal-hooks/deploy/prometheus owner=root group=root mode=0755
+  when: prometheus_receive_only
+
+- name: create ssl cert
+  include_role:
+    name: certificate
+  vars:
+    domains: ["{{ prometheus_domain }}"]
+  when: prometheus_receive_only
+
 - name: install prometheus configuration
-  template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=root mode=644
+  template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=prometheus mode=640
   notify: reload prometheus
 
 - name: install prometheus cli configuration
-  copy: src=prometheus.conf dest=/etc/conf.d/prometheus owner=root group=root mode=600
+  template: src=prometheus.conf.j2 dest=/etc/conf.d/prometheus owner=root group=root mode=600
   notify: reload prometheus
 
+- name: install prometheus web-config configuration
+  template: src=web-config.yml.j2 dest=/etc/prometheus/web-config.yml owner=root group=prometheus mode=640
+  notify: reload prometheus
+  when: prometheus_receive_only
+
 - name: install prometheus alert configuration
   copy: src=node.rules.yml dest=/etc/prometheus/node.rules.yml owner=root group=root mode=644
   notify: reload prometheus
+  when: not prometheus_receive_only
 
 - name: enable prometheus server service
   systemd: name=prometheus enabled=yes daemon_reload=yes state=started
+
+- name: open firewall holes for prometheus
+  ansible.posix.firewalld: service=prometheus permanent=true state=enabled immediate=yes
+  when: configure_firewall and prometheus_receive_only
+  tags:
+    - firewall

roles/prometheus/templates/letsencrypt.hook.d.j2

+#!/bin/bash
+set -o errexit -o nounset
+
+for domain in ${RENEWED_DOMAINS}; do
+  if [[ "{{ prometheus_domain }}" = "${domain}" ]]; then
+    umask 077
+    cp --dereference "${RENEWED_LINEAGE}/fullchain.pem" /etc/prometheus/server.crt.new
+    cp --dereference "${RENEWED_LINEAGE}/privkey.pem" /etc/prometheus/server.key.new
+    chown root:prometheus /etc/prometheus/server.{crt,key}.new
+    chmod 640 /etc/prometheus/server.{crt,key}.new
+    rename ".new" "" /etc/prometheus/server.{crt,key}.new
+    break
+  fi
+done

roles/prometheus/templates/prometheus.conf.j2

+{% if prometheus_receive_only %}
+PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml"
+{% else %}
+PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
+{% endif %}

roles/prometheus/templates/prometheus.yml.j2

+{% if not prometheus_receive_only %}
 global:
   scrape_interval: 60s
 
@@ -12,6 +13,16 @@ alerting:
     - targets:
        - localhost:9093
 
+remote_write:
+- url: https://{{ prometheus_domain }}:9090/api/v1/write
+  write_relabel_configs:
+  - source_labels: [__name__]
+    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes"
+    action: keep
+  basic_auth:
+    username: {{ vault_prometheus_user }}
+    password: {{ vault_prometheus_passwd }}
+
 scrape_configs:
   - job_name: loki
     static_configs:
@@ -111,3 +122,4 @@ scrape_configs:
       - target_label: __address__
         replacement: 127.0.0.1:9115
 {% endfor %}
+{% endif %}

roles/prometheus/templates/web-config.yml.j2

+tls_server_config:
+  cert_file: server.crt
+  key_file: server.key
+
+# Usernames and passwords required to connect to Prometheus.
+# Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/46630604b0f1c5d64fbd3eb3010d91af38dc798b/docs/web-configuration.md#about-bcrypt
+basic_auth_users:
+  {{ vault_prometheus_user }}: {{ vault_prometheus_passwd_hashed }}