Verified Commit a9ee7e5d authored by Kristian Klausen's avatar Kristian Klausen 🎉
Browse files

Send prometheus metrics and scrap its metrics over WireGuard

parent 130e19d4
Pipeline #9484 passed with stage
in 58 seconds
---
filesystem: btrfs
ipv4_address: 157.90.255.107
prometheus_domain: dashboards.archlinux.org
wireguard_address: 10.0.0.33
wireguard_public_key: lLZtvFIrmtUXRXmw+qQC8LZ00NzN1wlvcI4grNWt2lE=
......@@ -9,7 +9,7 @@ datasources:
basicAuthUser: {{ vault_prometheus_user }}
secureJsonData:
basicAuthPassword: {{ vault_prometheus_passwd }}
url: https://{{ prometheus_domain }}:9090
url: http://{{ prometheus_domain }}:9090
{% else %}
- name: Prometheus
type: prometheus
......
monitoring_domain: monitoring.archlinux.org
gitlab_runner_exporter_port: '9252'
prometheus_domain: dashboards.archlinux.org
prometheus_domain: "{{ hostvars['dashboards.archlinux.org']['wireguard_address'] }}"
prometheus_mysqld_exporter_port: '9104'
prometheus_receive_only: false
......
......@@ -3,17 +3,6 @@
- name: install prometheus server
pacman: name=prometheus,python-passlib,python-bcrypt state=present
- name: install cert renewal hook
template: src=letsencrypt.hook.d.j2 dest=/etc/letsencrypt/renewal-hooks/deploy/prometheus owner=root group=root mode=0755
when: prometheus_receive_only
- name: create ssl cert
include_role:
name: certificate
vars:
domains: ["{{ prometheus_domain }}"]
when: prometheus_receive_only
- name: install prometheus configuration
template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=prometheus mode=640
notify: reload prometheus
......@@ -35,8 +24,9 @@
- name: enable prometheus server service
systemd: name=prometheus enabled=yes daemon_reload=yes state=started
- name: open firewall holes for prometheus
ansible.posix.firewalld: service=prometheus permanent=true state=enabled immediate=yes
- name: open prometheus port for monitoring.archlinux.org
ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=9090 accept"
when: configure_firewall and prometheus_receive_only
tags:
- firewall
#!/bin/bash
set -o errexit -o nounset
for domain in ${RENEWED_DOMAINS}; do
if [[ "{{ prometheus_domain }}" = "${domain}" ]]; then
umask 077
cp --dereference "${RENEWED_LINEAGE}/fullchain.pem" /etc/prometheus/server.crt.new
cp --dereference "${RENEWED_LINEAGE}/privkey.pem" /etc/prometheus/server.key.new
chown root:prometheus /etc/prometheus/server.{crt,key}.new
chmod 640 /etc/prometheus/server.{crt,key}.new
rename ".new" "" /etc/prometheus/server.{crt,key}.new
break
fi
done
{% if prometheus_receive_only %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml"
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml --web.listen-address={{ wireguard_address }}:9090"
{% else %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
{% endif %}
......@@ -14,7 +14,7 @@ alerting:
- localhost:9093
remote_write:
- url: https://{{ prometheus_domain }}:9090/api/v1/write
- url: http://{{ prometheus_domain }}:9090/api/v1/write
write_relabel_configs:
- source_labels: [__name__]
regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
......@@ -31,7 +31,7 @@ scrape_configs:
instance: "{{ ansible_fqdn }}"
- job_name: prometheus-domain
scheme: https
scheme: http
basic_auth:
username: {{ vault_prometheus_user }}
password: {{ vault_prometheus_passwd }}
......@@ -39,7 +39,7 @@ scrape_configs:
- targets: ['{{ prometheus_domain }}:9090']
labels:
job: prometheus
instance: "{{ prometheus_domain }}"
instance: "dashboards.archlinux.org"
- job_name: loki
static_configs:
......
tls_server_config:
cert_file: server.crt
key_file: server.key
# Usernames and passwords required to connect to Prometheus.
# Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/46630604b0f1c5d64fbd3eb3010d91af38dc798b/docs/web-configuration.md#about-bcrypt
basic_auth_users:
......
---
prometheus_domain: monitoring.archlinux.org
prometheus_textfile_dir: /var/lib/node_exporter
gitlab_runner_exporter_port: '9252'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment