diff --git a/docs/monitoring.md b/docs/monitoring.md index a09b304f63749077229f9622ddaf0f7a10f94c4c..3e9cc44cf8aa63528c321fa8815c2e24c1e1c8e5 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -60,3 +60,8 @@ Gitlab runners export a [prometheus endpoint](https://docs.gitlab.com/runner/mon ### Network monitoring For http(s)/icmp monitoring [prometheus-black-exporter](https://github.com/prometheus/blackbox_exporter) is used, which currently has alerts configured for https and SSL certificate expiry monitoring. The web endpoints to monitor are configured in `roles/prometheus/defaults/main.yml`. + + +### Archive monitoring + +The [Archive](https://archive.archlinux.org) and its mirrors defined in `archive_mirrors` are monitored using a textcollector which monitors the archive size in bytes. diff --git a/roles/prometheus_exporters/files/archive-textcollector.sh b/roles/prometheus_exporters/files/archive-textcollector.sh new file mode 100644 index 0000000000000000000000000000000000000000..8216fe5dc6edbdd1039aaeeb46caee0350742d0f --- /dev/null +++ b/roles/prometheus_exporters/files/archive-textcollector.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +if (( $# != 1 )); then + echo "Missing textcollector directory argument" + exit 1 +fi + +TEXTFILE_COLLECTOR_DIR=${1} +ARCHIVE_DIR=/srv/archive +PROM_FILE=$TEXTFILE_COLLECTOR_DIR/archive.prom + +TMP_FILE=$PROM_FILE.$$ +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +trap "rm -f $TMP_FILE" EXIT + +directory_size=$(du -sb ${ARCHIVE_DIR} | awk '{ print $1 }') +archived_packages=$(find ${ARCHIVE_DIR}/packages/ -type f -name '*.pkg.tar.xz' -o -name '*.pkg.tar.zst' | wc -l) + +echo "# HELP archive_directory_size_bytes archive directory size in bytes" >> $TMP_FILE +echo "# TYPE archive_directory_size_bytes gauge" >> $TMP_FILE +echo "archive_directory_size_bytes $directory_size" >> $TMP_FILE + +echo "# HELP archive_total_packages total amount of archived packages" >> $TMP_FILE +echo "# TYPE archive_total_packages gauge" >> $TMP_FILE +echo "archive_total_packages $archived_packages" >> $TMP_FILE + +mv -f $TMP_FILE $PROM_FILE diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml index ad5b946fc210042ef45f2d6d3e2df617dd6b6cd8..1dd7bff4d3c91083b36d66245e2e01ff0150ef02 100644 --- a/roles/prometheus_exporters/tasks/main.yml +++ b/roles/prometheus_exporters/tasks/main.yml @@ -58,6 +58,7 @@ - borg-textcollector.sh - rebuilderd-textcollector.sh - rebuilderd-status-textcollector.py + - archive-textcollector.sh - name: install arch textcollector service template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600 @@ -96,6 +97,14 @@ systemd: name=prometheus-rebuilderd-textcollector.timer enabled=yes daemon_reload=yes state=started when: "'rebuilderd' in group_names" +- name: install rebuilderd textcollector service + template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=600 + when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'" + +- name: enable and start prometheus archive textcollector timer + systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started + when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'" + - name: enable prometheus-node-exporter service systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started diff --git a/roles/prometheus_exporters/templates/prometheus-archive-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-archive-textcollector.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..b7a491a8817e61d0cb9aaa6524e7f5a29d2afde4 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-archive-textcollector.service.j2 @@ -0,0 +1,37 @@ +[Unit] +Description=Prometheus Archive Exporter +After=network.target + +[Service] +Type=oneshot +User=node_exporter +ExecStart=/usr/local/bin/archive-textcollector.sh {{ prometheus_textfile_dir }} + +NoNewPrivileges=true +LockPersonality=true +CapabilityBoundingSet= +UMask=077 + +PrivateDevices=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths={{ prometheus_textfile_dir }} + +MemoryDenyWriteExecute=true +RemoveIPC=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictSUIDSGID=true + +RestrictAddressFamilies=~AF_NETLINK +RestrictAddressFamilies=~AF_PACKET + +ProtectHostname=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectClock=true + +SystemCallArchitectures=native diff --git a/roles/prometheus_exporters/templates/prometheus-archive-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-archive-textcollector.timer.j2 new file mode 100644 index 0000000000000000000000000000000000000000..594ce57cd08f7663543371b4d6897f6c8dc5e14a --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-archive-textcollector.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Prometheus Archive Exporter TextCollector Timer + +[Timer] +# 24 hours +OnUnitActiveSec=1440m +OnBootSec=15min +RandomizedDelaySec=1min + +[Install] +WantedBy=timers.target