From a9ee7e5d8f15fd6d294e7bd336e51d32e80739fa Mon Sep 17 00:00:00 2001
From: Kristian Klausen <kristian@klausen.dk>
Date: Wed, 7 Jul 2021 00:56:14 +0200
Subject: [PATCH] Send prometheus metrics and scrap its metrics over WireGuard
---
host_vars/dashboards.archlinux.org/misc | 1 -
roles/grafana/templates/datasources.yaml.j2 | 2 +-
roles/prometheus/defaults/main.yml | 2 +-
roles/prometheus/tasks/main.yml | 16 +++-------------
roles/prometheus/templates/letsencrypt.hook.d.j2 | 14 --------------
roles/prometheus/templates/prometheus.conf.j2 | 2 +-
roles/prometheus/templates/prometheus.yml.j2 | 6 +++---
roles/prometheus/templates/web-config.yml.j2 | 4 ----
roles/prometheus_exporters/defaults/main.yml | 2 --
9 files changed, 9 insertions(+), 40 deletions(-)
delete mode 100644 roles/prometheus/templates/letsencrypt.hook.d.j2
diff --git a/host_vars/dashboards.archlinux.org/misc b/host_vars/dashboards.archlinux.org/misc
index 628bfa1fe..59ce6cb73 100644
--- a/host_vars/dashboards.archlinux.org/misc
+++ b/host_vars/dashboards.archlinux.org/misc
@@ -1,6 +1,5 @@
---
filesystem: btrfs
ipv4_address: 157.90.255.107
-prometheus_domain: dashboards.archlinux.org
wireguard_address: 10.0.0.33
wireguard_public_key: lLZtvFIrmtUXRXmw+qQC8LZ00NzN1wlvcI4grNWt2lE=
diff --git a/roles/grafana/templates/datasources.yaml.j2 b/roles/grafana/templates/datasources.yaml.j2
index 1abb6cf61..c71602cb0 100644
--- a/roles/grafana/templates/datasources.yaml.j2
+++ b/roles/grafana/templates/datasources.yaml.j2
@@ -9,7 +9,7 @@ datasources:
basicAuthUser: {{ vault_prometheus_user }}
secureJsonData:
basicAuthPassword: {{ vault_prometheus_passwd }}
- url: https://{{ prometheus_domain }}:9090
+ url: http://{{ prometheus_domain }}:9090
{% else %}
- name: Prometheus
type: prometheus
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
index e630bd84e..fa3854f91 100644
--- a/roles/prometheus/defaults/main.yml
+++ b/roles/prometheus/defaults/main.yml
@@ -1,6 +1,6 @@
monitoring_domain: monitoring.archlinux.org
gitlab_runner_exporter_port: '9252'
-prometheus_domain: dashboards.archlinux.org
+prometheus_domain: "{{ hostvars['dashboards.archlinux.org']['wireguard_address'] }}"
prometheus_mysqld_exporter_port: '9104'
prometheus_receive_only: false
diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml
index 0cbf67b19..3c1aa21f9 100644
--- a/roles/prometheus/tasks/main.yml
+++ b/roles/prometheus/tasks/main.yml
@@ -3,17 +3,6 @@
- name: install prometheus server
pacman: name=prometheus,python-passlib,python-bcrypt state=present
-- name: install cert renewal hook
- template: src=letsencrypt.hook.d.j2 dest=/etc/letsencrypt/renewal-hooks/deploy/prometheus owner=root group=root mode=0755
- when: prometheus_receive_only
-
-- name: create ssl cert
- include_role:
- name: certificate
- vars:
- domains: ["{{ prometheus_domain }}"]
- when: prometheus_receive_only
-
- name: install prometheus configuration
template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=prometheus mode=640
notify: reload prometheus
@@ -35,8 +24,9 @@
- name: enable prometheus server service
systemd: name=prometheus enabled=yes daemon_reload=yes state=started
-- name: open firewall holes for prometheus
- ansible.posix.firewalld: service=prometheus permanent=true state=enabled immediate=yes
+- name: open prometheus port for monitoring.archlinux.org
+ ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+ rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=9090 accept"
when: configure_firewall and prometheus_receive_only
tags:
- firewall
diff --git a/roles/prometheus/templates/letsencrypt.hook.d.j2 b/roles/prometheus/templates/letsencrypt.hook.d.j2
deleted file mode 100644
index d493a71e2..000000000
--- a/roles/prometheus/templates/letsencrypt.hook.d.j2
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-set -o errexit -o nounset
-
-for domain in ${RENEWED_DOMAINS}; do
- if [[ "{{ prometheus_domain }}" = "${domain}" ]]; then
- umask 077
- cp --dereference "${RENEWED_LINEAGE}/fullchain.pem" /etc/prometheus/server.crt.new
- cp --dereference "${RENEWED_LINEAGE}/privkey.pem" /etc/prometheus/server.key.new
- chown root:prometheus /etc/prometheus/server.{crt,key}.new
- chmod 640 /etc/prometheus/server.{crt,key}.new
- rename ".new" "" /etc/prometheus/server.{crt,key}.new
- break
- fi
-done
diff --git a/roles/prometheus/templates/prometheus.conf.j2 b/roles/prometheus/templates/prometheus.conf.j2
index dbff0daee..50dd731a1 100644
--- a/roles/prometheus/templates/prometheus.conf.j2
+++ b/roles/prometheus/templates/prometheus.conf.j2
@@ -1,5 +1,5 @@
{% if prometheus_receive_only %}
-PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml"
+PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml --web.listen-address={{ wireguard_address }}:9090"
{% else %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
{% endif %}
diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2
index d301b3b8d..a0549850d 100644
--- a/roles/prometheus/templates/prometheus.yml.j2
+++ b/roles/prometheus/templates/prometheus.yml.j2
@@ -14,7 +14,7 @@ alerting:
- localhost:9093
remote_write:
-- url: https://{{ prometheus_domain }}:9090/api/v1/write
+- url: http://{{ prometheus_domain }}:9090/api/v1/write
write_relabel_configs:
- source_labels: [__name__]
regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
@@ -31,7 +31,7 @@ scrape_configs:
instance: "{{ ansible_fqdn }}"
- job_name: prometheus-domain
- scheme: https
+ scheme: http
basic_auth:
username: {{ vault_prometheus_user }}
password: {{ vault_prometheus_passwd }}
@@ -39,7 +39,7 @@ scrape_configs:
- targets: ['{{ prometheus_domain }}:9090']
labels:
job: prometheus
- instance: "{{ prometheus_domain }}"
+ instance: "dashboards.archlinux.org"
- job_name: loki
static_configs:
diff --git a/roles/prometheus/templates/web-config.yml.j2 b/roles/prometheus/templates/web-config.yml.j2
index 1c304aa46..13e8ad8e8 100644
--- a/roles/prometheus/templates/web-config.yml.j2
+++ b/roles/prometheus/templates/web-config.yml.j2
@@ -1,7 +1,3 @@
-tls_server_config:
- cert_file: server.crt
- key_file: server.key
-
# Usernames and passwords required to connect to Prometheus.
# Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/46630604b0f1c5d64fbd3eb3010d91af38dc798b/docs/web-configuration.md#about-bcrypt
basic_auth_users:
diff --git a/roles/prometheus_exporters/defaults/main.yml b/roles/prometheus_exporters/defaults/main.yml
index b66b5c460..0fe0db587 100644
--- a/roles/prometheus_exporters/defaults/main.yml
+++ b/roles/prometheus_exporters/defaults/main.yml
@@ -1,7 +1,5 @@
---
-prometheus_domain: monitoring.archlinux.org
-
prometheus_textfile_dir: /var/lib/node_exporter
gitlab_runner_exporter_port: '9252'
--
GitLab