Commit e9f7c970 authored by Kristian Klausen's avatar Kristian Klausen 🎉
Browse files

prometheus: Add receive only mode and remote_write metrics to dashboards.al.org

parent 103bbdec
......@@ -9,7 +9,7 @@
- { role: root_ssh }
- { role: hardening }
- { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
- { role: prometheus }
- { role: prometheus, prometheus_receive_only: true }
- { role: prometheus_exporters }
- { role: promtail }
- { role: certbot }
......
monitoring_domain: monitoring.archlinux.org
gitlab_runner_exporter_port: '9252'
prometheus_domain: dashboards.archlinux.org
prometheus_mysqld_exporter_port: '9104'
prometheus_receive_only: false
# for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/ - https:\/\//"
blackbox_targets:
......@@ -57,6 +59,7 @@ blackbox_targets:
- mail.archlinux.org:465
- mail.archlinux.org:993
- mail.archlinux.org:995
- dashboards.archlinux.org:9090
smtp_starttls:
- mail.archlinux.org:25
- mail.archlinux.org:587
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
......@@ -3,17 +3,40 @@
- name: install prometheus server
pacman: name=prometheus state=present
- name: install cert renewal hook
template: src=letsencrypt.hook.d.j2 dest=/etc/letsencrypt/renewal-hooks/deploy/prometheus owner=root group=root mode=0755
when: prometheus_receive_only
- name: create ssl cert
include_role:
name: certificate
vars:
domains: ["{{ prometheus_domain }}"]
when: prometheus_receive_only
- name: install prometheus configuration
template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=root mode=644
template: src=prometheus.yml.j2 dest=/etc/prometheus/prometheus.yml owner=root group=prometheus mode=640
notify: reload prometheus
- name: install prometheus cli configuration
copy: src=prometheus.conf dest=/etc/conf.d/prometheus owner=root group=root mode=600
template: src=prometheus.conf.j2 dest=/etc/conf.d/prometheus owner=root group=root mode=600
notify: reload prometheus
- name: install prometheus web-config configuration
template: src=web-config.yml.j2 dest=/etc/prometheus/web-config.yml owner=root group=prometheus mode=640
notify: reload prometheus
when: prometheus_receive_only
- name: install prometheus alert configuration
copy: src=node.rules.yml dest=/etc/prometheus/node.rules.yml owner=root group=root mode=644
notify: reload prometheus
when: not prometheus_receive_only
- name: enable prometheus server service
systemd: name=prometheus enabled=yes daemon_reload=yes state=started
- name: open firewall holes for prometheus
ansible.posix.firewalld: service=prometheus permanent=true state=enabled immediate=yes
when: configure_firewall and prometheus_receive_only
tags:
- firewall
#!/bin/bash
set -o errexit -o nounset
for domain in ${RENEWED_DOMAINS}; do
if [[ "{{ prometheus_domain }}" = "${domain}" ]]; then
umask 077
cp --dereference "${RENEWED_LINEAGE}/fullchain.pem" /etc/prometheus/server.crt.new
cp --dereference "${RENEWED_LINEAGE}/privkey.pem" /etc/prometheus/server.key.new
chown root:prometheus /etc/prometheus/server.{crt,key}.new
chmod 640 /etc/prometheus/server.{crt,key}.new
rename ".new" "" /etc/prometheus/server.{crt,key}.new
break
fi
done
{% if prometheus_receive_only %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --enable-feature=remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml"
{% else %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
{% endif %}
{% if not prometheus_receive_only %}
global:
scrape_interval: 60s
......@@ -12,6 +13,16 @@ alerting:
- targets:
- localhost:9093
remote_write:
- url: https://{{ prometheus_domain }}:9090/api/v1/write
write_relabel_configs:
- source_labels: [__name__]
regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes"
action: keep
basic_auth:
username: {{ vault_prometheus_user }}
password: {{ vault_prometheus_passwd }}
scrape_configs:
- job_name: loki
static_configs:
......@@ -111,3 +122,4 @@ scrape_configs:
- target_label: __address__
replacement: 127.0.0.1:9115
{% endfor %}
{% endif %}
tls_server_config:
cert_file: server.crt
key_file: server.key
# Usernames and passwords required to connect to Prometheus.
# Passwords are hashed with bcrypt: https://github.com/prometheus/exporter-toolkit/blob/46630604b0f1c5d64fbd3eb3010d91af38dc798b/docs/web-configuration.md#about-bcrypt
basic_auth_users:
{{ vault_prometheus_user }}: {{ vault_prometheus_passwd_hashed }}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment