Verified Commit 4658d36d authored by Jelle van der Waa's avatar Jelle van der Waa 🚧
Browse files

Add archive specific monitoring

To monitor our archive mirrors and the archive size itself a new
textcollector has been added. This will allow us to monitor the archive
growth and the sync rate to mirrors.
parent 56fd045d
Pipeline #3584 passed with stage
in 46 seconds
......@@ -60,3 +60,8 @@ Gitlab runners export a [prometheus endpoint](https://docs.gitlab.com/runner/mon
### Network monitoring
For http(s)/icmp monitoring [prometheus-black-exporter](https://github.com/prometheus/blackbox_exporter) is used, which currently has alerts configured for https and SSL certificate expiry monitoring. The web endpoints to monitor are configured in `roles/prometheus/defaults/main.yml`.
### Archive monitoring
The [Archive](https://archive.archlinux.org) and its mirrors defined in `archive_mirrors` are monitored using a textcollector which monitors the archive size in bytes.
#!/bin/bash
set -o errexit
set -o nounset
if (( $# != 1 )); then
echo "Missing textcollector directory argument"
exit 1
fi
TEXTFILE_COLLECTOR_DIR=${1}
ARCHIVE_DIR=/srv/archive
PROM_FILE=$TEXTFILE_COLLECTOR_DIR/archive.prom
TMP_FILE=$PROM_FILE.$$
[ -e $TMP_FILE ] && rm -f $TMP_FILE
trap "rm -f $TMP_FILE" EXIT
directory_size=$(du -sb ${ARCHIVE_DIR} | awk '{ print $1 }')
archived_packages=$(find ${ARCHIVE_DIR}/packages/ -type f -name '*.pkg.tar.xz' -o -name '*.pkg.tar.zst' | wc -l)
echo "# HELP archive_directory_size_bytes archive directory size in bytes" >> $TMP_FILE
echo "# TYPE archive_directory_size_bytes gauge" >> $TMP_FILE
echo "archive_directory_size_bytes $directory_size" >> $TMP_FILE
echo "# HELP archive_total_packages total amount of archived packages" >> $TMP_FILE
echo "# TYPE archive_total_packages gauge" >> $TMP_FILE
echo "archive_total_packages $archived_packages" >> $TMP_FILE
mv -f $TMP_FILE $PROM_FILE
......@@ -58,6 +58,7 @@
- borg-textcollector.sh
- rebuilderd-textcollector.sh
- rebuilderd-status-textcollector.py
- archive-textcollector.sh
- name: install arch textcollector service
template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600
......@@ -96,6 +97,14 @@
systemd: name=prometheus-rebuilderd-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "'rebuilderd' in group_names"
- name: install rebuilderd textcollector service
template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=600
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
- name: enable and start prometheus archive textcollector timer
systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
- name: enable prometheus-node-exporter service
systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started
......
[Unit]
Description=Prometheus Archive Exporter
After=network.target
[Service]
Type=oneshot
User=node_exporter
ExecStart=/usr/local/bin/archive-textcollector.sh {{ prometheus_textfile_dir }}
NoNewPrivileges=true
LockPersonality=true
CapabilityBoundingSet=
UMask=077
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths={{ prometheus_textfile_dir }}
MemoryDenyWriteExecute=true
RemoveIPC=true
RestrictRealtime=true
RestrictNamespaces=true
RestrictSUIDSGID=true
RestrictAddressFamilies=~AF_NETLINK
RestrictAddressFamilies=~AF_PACKET
ProtectHostname=true
ProtectControlGroups=true
ProtectKernelLogs=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectClock=true
SystemCallArchitectures=native
[Unit]
Description=Prometheus Archive Exporter TextCollector Timer
[Timer]
# 24 hours
OnUnitActiveSec=1440m
OnBootSec=15min
RandomizedDelaySec=1min
[Install]
WantedBy=timers.target
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment