Verified Commit 7abc2500 authored by Jelle van der Waa's avatar Jelle van der Waa 🚧
Browse files

Add rebuilderd_results Prometheus metric

To monitor if reproducible builds are going in the right direction,
record the good/bad/unknown metrics from rebuilderd with a Prometheus
textcollector for a Grafana dashboard to display a long term trend.

A Python script is required to handle data collection as obtaining the
status with jq/bash is non trivial and cannot easily dnyamically collect
suites and statuses.

Closes: #146
parent a33ade02
Pipeline #2057 passed with stage
in 43 seconds
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": null,
"iteration": 1601407761906,
"links": [],
"panels": [
{
"aliasColors": {
"bad": "red",
"good": "green"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"description": "rebuilderd results for [$suite] repository",
"fieldConfig": {
"defaults": {
"custom": {
"align": null
},
"displayName": "Core results",
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pluginVersion": "7.1.4",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rebuilderd_results{suite=\"$suite\"}",
"interval": "",
"legendFormat": "{{ status }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$suite",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"description": "Number of active rebuidlerd-workers",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 11,
"w": 11,
"x": 0,
"y": 12
},
"hiddenSeries": false,
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pluginVersion": "7.1.4",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rebuilderd_workers",
"interval": "",
"legendFormat": "active workers",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Active rebuilderd workers",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"description": "The rebuilderd queue length of to be rebuild packages",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 11,
"w": 13,
"x": 11,
"y": 12
},
"hiddenSeries": false,
"id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pluginVersion": "7.1.4",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rebuilderd_queue_length",
"interval": "",
"legendFormat": "queue length",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Queue length",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"schemaVersion": 26,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {
"selected": true,
"text": "extra",
"value": "extra"
},
"datasource": "Prometheus",
"definition": "label_values(rebuilderd_results,suite)",
"hide": 0,
"includeAll": false,
"label": "suite",
"multi": false,
"name": "suite",
"options": [
{
"selected": false,
"text": "community",
"value": "community"
},
{
"selected": false,
"text": "core",
"value": "core"
},
{
"selected": true,
"text": "extra",
"value": "extra"
},
{
"selected": false,
"text": "testing",
"value": "testing"
}
],
"query": "label_values(rebuilderd_results,suite)",
"refresh": 0,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "Rebuilderd",
"uid": null,
"version": 0
}
#!/usr/bin/python
import sys
import shutil
import tempfile
from collections import defaultdict
import requests
# The rebuilderd instance
API_URL = 'https://reproducible.archlinux.org'
def get_metric_header():
return '# HELP rebuilderd_results number of packages\n# TYPE rebuilderd_results gauge\n'
def format_metric(suite, status, total):
status = status.lower()
return f'rebuilderd_results{{suite="{suite}",status="{status}"}} {total}\n'
def get_rebuilderd_data():
req = requests.get(f'{API_URL}/api/v0/pkgs/list')
if req.status_code != 200:
print(f'Failed to obtain rebuilderd data, http status code: {req.status_code}', file=sys.stderr)
sys.exit(1)
data = req.json()
dataset = defaultdict(dict)
for entry in data:
suite = entry['suite']
status = entry['status']
if suite in dataset:
if status in dataset[suite]:
dataset[suite][status] += 1
else:
dataset[suite][status] = 1
else:
dataset[suite] = defaultdict(dict)
dataset[suite][status] = 1
return dataset
def main(directory):
dataset = get_rebuilderd_data()
with tempfile.TemporaryDirectory() as tmpfp:
filename = f'{tmpfp}/rebuilderd_status.prom'
print(filename)
with open(filename, 'w') as fp:
fp.write(get_metric_header())
for suite, data in dataset.items():
for status, total in data.items():
fp.write(format_metric(suite, status, total))
shutil.move(filename, f'{directory}/rebuilderd-status.prom')
if __name__ == "__main__":
if len(sys.argv) != 2:
print('Missing textcollector directory argument')
main(sys.argv[1])
......@@ -57,6 +57,7 @@
- arch-textcollector.sh
- borg-textcollector.sh
- rebuilderd-textcollector.sh
- rebuilderd-status-textcollector.py
- name: install arch textcollector service
template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600
......
......@@ -6,6 +6,7 @@ After=network.target
Type=oneshot
User=node_exporter
ExecStart=/usr/local/bin/rebuilderd-textcollector.sh {{ prometheus_textfile_dir }}
ExecStart=/usr/local/bin/rebuilderd-status-textcollector.py {{ prometheus_textfile_dir }}
NoNewPrivileges=true
LockPersonality=true
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment