Verified Commit 3fd36ddb authored by Jelle van der Waa's avatar Jelle van der Waa 🚧
Browse files

Add blackbox exporter for https status checking

Run the blackbox exporter on monitoring.archlinux.org to monitor other
machines http status for public services we provide. Also has an alert
for when a certificate is about to expire in 3 days.
parent 23564b29
monitoring_domain: monitoring.archlinux.org
gitlab_runner_exporter_port: '9252'
blackbox_targets:
- https://archlinux.org
- https://wiki.archlinux.org
- https://gitlab.archlinux.org
- https://bbs.archlinux.org
- https://bugs.archlinux.org
- https://aur.archlinux.org
- https://archive.archlinux.org
- https://mirror.pkgbuild.com
- https://pkgbuild.com
......@@ -366,3 +366,31 @@ groups:
annotations:
summary: Workhorse has high error rates
description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 10% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes.
- name: blackbox
interval: 15s
rules:
- alert: BlackboxProbeFailed
expr: probe_success == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Blackbox probe failed (instance {{ $labels.instance }})"
description: "Probe failed\n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
- alert: BlackboxProbeHttpFailure
expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400
for: 5m
labels:
severity: critical
annotations:
summary: "Blackbox probe HTTP failure (instance {{ $labels.instance }})"
description: "HTTP status code is not 200-399\n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
- alert: BlackboxSslCertificateWillExpireSoon
expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3
for: 5m
labels:
severity: critical
annotations:
summary: "Blackbox SSL certificate will expire soon (instance {{ $labels.instance }})"
description: "SSL certificate expires in 3 days\n VALUE = {{ $value }}\n LABELS: {{ $labels }}"
modules:
http_prometheus:
prober: http
timeout: 5s
http:
......@@ -3,6 +3,10 @@
- name: install prometheus-node-exporter
pacman: name=prometheus-node-exporter,arch-audit,pacman-contrib state=present
- name: install prometheus-blackbox-exporter
pacman: name=prometheus-blackbox-exporter state=present
when: "'prometheus' in group_names"
- name: install prometheus-memcached-exporter
pacman: name=prometheus-memcached-exporter state=present
when: "'memcached' in group_names"
......@@ -65,9 +69,17 @@
systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "'borg_clients' in group_names"
- name: install blackbox exporter configuration
copy: src=blackbox.yml dest=/etc/prometheus/blackbox.yml owner=root group=root mode=0755
when: "'prometheus' in group_names"
- name: enable prometheus-node-exporter service
systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started
- name: enable prometheus-blackbox-exporter service
systemd: name=prometheus-blackbox-exporter enabled=yes daemon_reload=yes state=started
when: "'prometheus' in group_names"
- name: enable prometheus-memcached-exporter service
systemd: name=prometheus-memcached-exporter enabled=yes daemon_reload=yes state=started
when: "'memcached' in group_names"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment