prometheus: Scrap debuginfod and send to dashboards.al.org

A dashboard will be added at a later date.

roles/debuginfod/tasks/main.yml

@@ -38,5 +38,11 @@
   when: debuginfod_domain
   tags: ['nginx']
 
+- name: open debuginfod ipv4 port for monitoring.archlinux.org
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=8002 accept"
+  tags:
+    - firewall
+
 - name: start and enable debuginfod
   service: name=debuginfod enabled=yes state=started

roles/prometheus/defaults/main.yml

@@ -3,6 +3,11 @@ gitlab_runner_exporter_port: '9252'
 prometheus_domain: "{{ hostvars['dashboards.archlinux.org']['wireguard_address'] }}"
 prometheus_mysqld_exporter_port: '9104'
 prometheus_receive_only: false
+prometheus_remote_write_relabel_configs:
+  - label: job
+    regex: debuginfod
+  - label: __name__
+    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
 
 # for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/  - https:\/\//"
 blackbox_targets:

roles/prometheus/templates/prometheus.yml.j2

@@ -14,14 +14,16 @@ alerting:
        - localhost:9093
 
 remote_write:
+{% for relabel_config in prometheus_remote_write_relabel_configs %}
 - url: http://{{ prometheus_domain }}:9090/api/v1/write
   write_relabel_configs:
-  - source_labels: [__name__]
-    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
+  - source_labels: [{{ relabel_config.label }}]
+    regex: "{{ relabel_config.regex }}"
     action: keep
   basic_auth:
     username: {{ vault_prometheus_user }}
     password: {{ vault_prometheus_passwd }}
+{% endfor %}
 
 scrape_configs:
   - job_name: prometheus
@@ -67,6 +69,12 @@ scrape_configs:
 
     {% endfor %}
 
+  - job_name: 'debuginfod'
+    static_configs:
+    - targets: ['{{ hostvars['debuginfod.archlinux.org']['wireguard_address'] }}:8002']
+      labels:
+        instance: "debuginfod.archlinux.org"
+
   - job_name: 'gitlab_runner_exporter'
     static_configs:
     {% for host in groups['gitlab_runners'] %}