Send prometheus metrics and scrap its metrics over WireGuard

host_vars/dashboards.archlinux.org/misc

 ---
 filesystem: btrfs
 ipv4_address: 157.90.255.107
-prometheus_domain: dashboards.archlinux.org
 wireguard_address: 10.0.0.33
 wireguard_public_key: lLZtvFIrmtUXRXmw+qQC8LZ00NzN1wlvcI4grNWt2lE=

roles/grafana/templates/datasources.yaml.j2

@@ -9,7 +9,7 @@ datasources:
   basicAuthUser: {{ vault_prometheus_user }}
   secureJsonData:
     basicAuthPassword: {{ vault_prometheus_passwd }}
-  url: https://{{ prometheus_domain }}:9090
+  url: http://{{ prometheus_domain }}:9090
 {% else %}
 - name: Prometheus
   type: prometheus

roles/prometheus/defaults/main.yml

 monitoring_domain: monitoring.archlinux.org
 gitlab_runner_exporter_port: '9252'
-prometheus_domain: dashboards.archlinux.org
+prometheus_domain: "{{ hostvars['dashboards.archlinux.org']['wireguard_address'] }}"
 prometheus_mysqld_exporter_port: '9104'
 prometheus_receive_only: false
 

roles/prometheus/tasks/main.yml

@@ -3,17 +3,6 @@
 - name: install prometheus server
   pacman: name=prometheus,python-passlib,python-bcrypt state=present
 
-- name: install cert renewal hook
-  template: src=letsencrypt.hook.d.j2 dest=/etc/letsencrypt/renewal-hooks/deploy/prometheus owner=root group=root mode=0755
-  when: prometheus_receive_only
-
-- name: create ssl cert
-  include_role:
-    name: certificate
-  vars:
-    domains: ["{{ prometheus_domain }}"]
-  when: prometheus_receive_only
-
 - name: install prometheus configuration
   template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=prometheus mode=640
   notify: reload prometheus
@@ -35,8 +24,9 @@
 - name: enable prometheus server service
   systemd: name=prometheus enabled=yes daemon_reload=yes state=started
 
-- name: open firewall holes for prometheus
-  ansible.posix.firewalld: service=prometheus permanent=true state=enabled immediate=yes
+- name: open prometheus port for monitoring.archlinux.org
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=9090 accept"
   when: configure_firewall and prometheus_receive_only
   tags:
     - firewall

roles/prometheus/templates/letsencrypt.hook.d.j2

-#!/bin/bash
-set -o errexit -o nounset
-
-for domain in ${RENEWED_DOMAINS}; do
-  if [[ "{{ prometheus_domain }}" = "${domain}" ]]; then
-    umask 077
-    cp --dereference "${RENEWED_LINEAGE}/fullchain.pem" /etc/prometheus/server.crt.new
-    cp --dereference "${RENEWED_LINEAGE}/privkey.pem" /etc/prometheus/server.key.new
-    chown root:prometheus /etc/prometheus/server.{crt,key}.new
-    chmod 640 /etc/prometheus/server.{crt,key}.new
-    rename ".new" "" /etc/prometheus/server.{crt,key}.new
-    break
-  fi
-done

roles/prometheus/templates/prometheus.conf.j2

 {% if prometheus_receive_only %}
-PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml"
+PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml --web.listen-address={{ wireguard_address }}:9090"
 {% else %}
 PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
 {% endif %}

roles/prometheus/templates/prometheus.yml.j2

@@ -14,7 +14,7 @@ alerting:
        - localhost:9093
 
 remote_write:
-- url: https://{{ prometheus_domain }}:9090/api/v1/write
+- url: http://{{ prometheus_domain }}:9090/api/v1/write
   write_relabel_configs:
   - source_labels: [__name__]
     regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
@@ -31,7 +31,7 @@ scrape_configs:
         instance: "{{ ansible_fqdn }}"
 
   - job_name: prometheus-domain
-    scheme: https
+    scheme: http
     basic_auth:
       username: {{ vault_prometheus_user }}
       password: {{ vault_prometheus_passwd }}
@@ -39,7 +39,7 @@ scrape_configs:
     - targets: ['{{ prometheus_domain }}:9090']
       labels:
         job: prometheus
-        instance: "{{ prometheus_domain }}"
+        instance: "dashboards.archlinux.org"
 
   - job_name: loki
     static_configs:

roles/prometheus/templates/web-config.yml.j2

-tls_server_config:
-  cert_file: server.crt
-  key_file: server.key
-
 # Usernames and passwords required to connect to Prometheus.
 # Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/46630604b0f1c5d64fbd3eb3010d91af38dc798b/docs/web-configuration.md#about-bcrypt
 basic_auth_users:

roles/prometheus_exporters/defaults/main.yml

 ---
 
-prometheus_domain: monitoring.archlinux.org
-
 prometheus_textfile_dir: /var/lib/node_exporter
 
 gitlab_runner_exporter_port: '9252'