Verified Commit c238ebda authored by Jelle van der Waa's avatar Jelle van der Waa 🚧
Browse files

Add prometheus textcollector for Arch repositories

Export the repository size of our repositories so they can be monitored
and we have some useful data for repository growth in the future.
parent 39a34e7d
Pipeline #5438 failed with stage
in 42 seconds
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Arch Repository Sizes",
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 32,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"description": "The size of all our repositories",
"fieldConfig": {
"defaults": {
"custom": {},
"unit": "decbytes"
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "repository_directory_size_bytes",
"interval": "",
"legendFormat": "{{ name }}",
"queryType": "randomWalk",
"refId": "A"
},
{
"expr": "sum(repository_directory_size_bytes)",
"hide": false,
"interval": "",
"legendFormat": "total",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Repository sizes",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:68",
"format": "decbytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:69",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6M",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Arch Repositories",
"uid": "hXmUEbsGk",
"version": 1
}
\ No newline at end of file
#!/bin/bash
set -o errexit
set -o nounset
if (( $# != 1 )); then
echo "Missing textcollector directory argument"
exit 1
fi
TEXTFILE_COLLECTOR_DIR=${1}
REPOS_DIR=/srv/ftp
PROM_FILE=$TEXTFILE_COLLECTOR_DIR/repository.prom
TMP_FILE=$PROM_FILE.$$
[ -e $TMP_FILE ] && rm -f $TMP_FILE
trap "rm -f $TMP_FILE" EXIT
echo "# HELP repository_directory_size_bytes repository directory size in bytes" >> $TMP_FILE
echo "# TYPE repository_directory_size_bytes gauge" >> $TMP_FILE
for dir in $REPOS_DIR/*; do
# All Arch repositories should have an os dir, this excludes the archive and other non repo dirs.
if [ -d "$dir/os" ]; then
reponame=$(basename $dir)
directory_size=$(du -Lsb ${dir} | awk '{ print $1 }')
echo "repository_directory_size_bytes{name=\"${reponame}\"} $directory_size" >> $TMP_FILE
fi
done
mv -f $TMP_FILE $PROM_FILE
...@@ -55,6 +55,7 @@ ...@@ -55,6 +55,7 @@
- rebuilderd-textcollector.sh - rebuilderd-textcollector.sh
- rebuilderd-status-textcollector.py - rebuilderd-status-textcollector.py
- archive-textcollector.sh - archive-textcollector.sh
- repository-textcollector.sh
- btrfs-textcollector.sh - btrfs-textcollector.sh
- name: install arch textcollector service - name: install arch textcollector service
...@@ -98,6 +99,10 @@ ...@@ -98,6 +99,10 @@
template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=644 template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=644
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'" when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
- name: install rebuilderd textcollector service
template: src=prometheus-repository-textcollector.service.j2 dest=/etc/systemd/system/prometheus-repository-textcollector.service owner=root group=root mode=644
when: "inventory_hostname == 'gemini.archlinux.org'"
- name: install rebuilderd textcollector timer - name: install rebuilderd textcollector timer
template: src=prometheus-archive-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.timer owner=root group=root mode=644 template: src=prometheus-archive-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.timer owner=root group=root mode=644
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'" when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
...@@ -106,6 +111,14 @@ ...@@ -106,6 +111,14 @@
systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'" when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
- name: install rebuilderd textcollector timer
template: src=prometheus-repository-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-repository-textcollector.timer owner=root group=root mode=644
when: "inventory_hostname == 'gemini.archlinux.org'"
- name: enable and start prometheus repository textcollector timer
systemd: name=prometheus-repository-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "inventory_hostname == 'gemini.archlinux.org'"
- name: install sudoers for btrfs - name: install sudoers for btrfs
copy: src=sudoers dest=/etc/sudoers.d/node_exporter owner=root group=root mode=0440 copy: src=sudoers dest=/etc/sudoers.d/node_exporter owner=root group=root mode=0440
when: filesystem == "btrfs" when: filesystem == "btrfs"
......
[Unit]
Description=Prometheus Arch Repository Exporter
After=network.target
[Service]
Type=oneshot
User=node_exporter
ExecStart=/usr/local/bin/repository-textcollector.sh {{ prometheus_textfile_dir }}
NoNewPrivileges=true
LockPersonality=true
CapabilityBoundingSet=
UMask=077
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths={{ prometheus_textfile_dir }}
MemoryDenyWriteExecute=true
RemoveIPC=true
RestrictRealtime=true
RestrictNamespaces=true
RestrictSUIDSGID=true
RestrictAddressFamilies=~AF_NETLINK
RestrictAddressFamilies=~AF_PACKET
ProtectHostname=true
ProtectControlGroups=true
ProtectKernelLogs=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectClock=true
SystemCallArchitectures=native
[Unit]
Description=Prometheus Arch Repository Exporter TextCollector Timer
[Timer]
# 24 hours
OnUnitActiveSec=1440m
OnBootSec=15min
RandomizedDelaySec=1min
[Install]
WantedBy=timers.target
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment