Add archive specific monitoring

To monitor our archive mirrors and the archive size itself a new
textcollector has been added. This will allow us to monitor the archive
growth and the sync rate to mirrors.

docs/monitoring.md

@@ -60,3 +60,8 @@ Gitlab runners export a [prometheus endpoint](https://docs.gitlab.com/runner/mon
 ### Network monitoring
 
 For http(s)/icmp monitoring [prometheus-black-exporter](https://github.com/prometheus/blackbox_exporter) is used, which currently has alerts configured for https and SSL certificate expiry monitoring. The web endpoints to monitor are configured in `roles/prometheus/defaults/main.yml`.
+
+
+### Archive monitoring
+
+The [Archive](https://archive.archlinux.org) and its mirrors defined in `archive_mirrors` are monitored using a textcollector which monitors the archive size in bytes.

roles/prometheus_exporters/files/archive-textcollector.sh

+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+TEXTFILE_COLLECTOR_DIR=${1}
+ARCHIVE_DIR=/srv/archive
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/archive.prom
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+directory_size=$(du -sb ${ARCHIVE_DIR} | awk '{ print $1 }')
+archived_packages=$(find ${ARCHIVE_DIR}/packages/ -type f -name '*.pkg.tar.xz' -o -name '*.pkg.tar.zst' | wc -l)
+
+echo "# HELP archive_directory_size_bytes archive directory size in bytes" >> $TMP_FILE
+echo "# TYPE archive_directory_size_bytes gauge" >> $TMP_FILE
+echo "archive_directory_size_bytes $directory_size" >> $TMP_FILE
+
+echo "# HELP archive_total_packages total amount of archived packages" >> $TMP_FILE
+echo "# TYPE archive_total_packages gauge" >> $TMP_FILE
+echo "archive_total_packages $archived_packages" >> $TMP_FILE
+
+mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/tasks/main.yml

@@ -58,6 +58,7 @@
     - borg-textcollector.sh
     - rebuilderd-textcollector.sh
     - rebuilderd-status-textcollector.py
+    - archive-textcollector.sh
 
 - name: install arch textcollector service
   template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600
@@ -96,6 +97,14 @@
   systemd: name=prometheus-rebuilderd-textcollector.timer enabled=yes daemon_reload=yes state=started
   when: "'rebuilderd' in group_names"
 
+- name: install rebuilderd textcollector service
+  template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=600
+  when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
+
+- name: enable and start prometheus archive textcollector timer
+  systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
+
 - name: enable prometheus-node-exporter service
   systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started
 

roles/prometheus_exporters/templates/prometheus-archive-textcollector.service.j2

+[Unit]
+Description=Prometheus Archive Exporter
+After=network.target
+
+[Service]
+Type=oneshot
+User=node_exporter
+ExecStart=/usr/local/bin/archive-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+CapabilityBoundingSet=
+UMask=077
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native

roles/prometheus_exporters/templates/prometheus-archive-textcollector.timer.j2

+[Unit]
+Description=Prometheus Archive Exporter TextCollector Timer
+
+[Timer]
+# 24 hours
+OnUnitActiveSec=1440m
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target