Skip to content
Snippets Groups Projects
Verified Commit 4658d36d authored by Jelle van der Waa's avatar Jelle van der Waa :construction:
Browse files

Add archive specific monitoring

To monitor our archive mirrors and the archive size itself a new
textcollector has been added. This will allow us to monitor the archive
growth and the sync rate to mirrors.
parent 56fd045d
No related branches found
No related tags found
No related merge requests found
......@@ -60,3 +60,8 @@ Gitlab runners export a [prometheus endpoint](https://docs.gitlab.com/runner/mon
### Network monitoring
For http(s)/icmp monitoring [prometheus-black-exporter](https://github.com/prometheus/blackbox_exporter) is used, which currently has alerts configured for https and SSL certificate expiry monitoring. The web endpoints to monitor are configured in `roles/prometheus/defaults/main.yml`.
### Archive monitoring
The [Archive](https://archive.archlinux.org) and its mirrors defined in `archive_mirrors` are monitored using a textcollector which monitors the archive size in bytes.
#!/bin/bash
set -o errexit
set -o nounset
if (( $# != 1 )); then
echo "Missing textcollector directory argument"
exit 1
fi
TEXTFILE_COLLECTOR_DIR=${1}
ARCHIVE_DIR=/srv/archive
PROM_FILE=$TEXTFILE_COLLECTOR_DIR/archive.prom
TMP_FILE=$PROM_FILE.$$
[ -e $TMP_FILE ] && rm -f $TMP_FILE
trap "rm -f $TMP_FILE" EXIT
directory_size=$(du -sb ${ARCHIVE_DIR} | awk '{ print $1 }')
archived_packages=$(find ${ARCHIVE_DIR}/packages/ -type f -name '*.pkg.tar.xz' -o -name '*.pkg.tar.zst' | wc -l)
echo "# HELP archive_directory_size_bytes archive directory size in bytes" >> $TMP_FILE
echo "# TYPE archive_directory_size_bytes gauge" >> $TMP_FILE
echo "archive_directory_size_bytes $directory_size" >> $TMP_FILE
echo "# HELP archive_total_packages total amount of archived packages" >> $TMP_FILE
echo "# TYPE archive_total_packages gauge" >> $TMP_FILE
echo "archive_total_packages $archived_packages" >> $TMP_FILE
mv -f $TMP_FILE $PROM_FILE
......@@ -58,6 +58,7 @@
- borg-textcollector.sh
- rebuilderd-textcollector.sh
- rebuilderd-status-textcollector.py
- archive-textcollector.sh
- name: install arch textcollector service
template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600
......@@ -96,6 +97,14 @@
systemd: name=prometheus-rebuilderd-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "'rebuilderd' in group_names"
- name: install rebuilderd textcollector service
template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=600
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
- name: enable and start prometheus archive textcollector timer
systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
- name: enable prometheus-node-exporter service
systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started
......
[Unit]
Description=Prometheus Archive Exporter
After=network.target
[Service]
Type=oneshot
User=node_exporter
ExecStart=/usr/local/bin/archive-textcollector.sh {{ prometheus_textfile_dir }}
NoNewPrivileges=true
LockPersonality=true
CapabilityBoundingSet=
UMask=077
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths={{ prometheus_textfile_dir }}
MemoryDenyWriteExecute=true
RemoveIPC=true
RestrictRealtime=true
RestrictNamespaces=true
RestrictSUIDSGID=true
RestrictAddressFamilies=~AF_NETLINK
RestrictAddressFamilies=~AF_PACKET
ProtectHostname=true
ProtectControlGroups=true
ProtectKernelLogs=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectClock=true
SystemCallArchitectures=native
[Unit]
Description=Prometheus Archive Exporter TextCollector Timer
[Timer]
# 24 hours
OnUnitActiveSec=1440m
OnBootSec=15min
RandomizedDelaySec=1min
[Install]
WantedBy=timers.target
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment