Add rebuilderd_results Prometheus metric

To monitor if reproducible builds are going in the right direction,
record the good/bad/unknown metrics from rebuilderd with a Prometheus
textcollector for a Grafana dashboard to display a long term trend.

A Python script is required to handle data collection as obtaining the
status with jq/bash is non trivial and cannot easily dnyamically collect
suites and statuses.

Closes: #146

roles/grafana/files/dashboards/rebuilderd.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1601407761906,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {
+        "bad": "red",
+        "good": "green"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "rebuilderd results for [$suite] repository",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": null
+          },
+          "displayName": "Core results",
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 12,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rebuilderd_results{suite=\"$suite\"}",
+          "interval": "",
+          "legendFormat": "{{ status }}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "$suite",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "Number of active rebuidlerd-workers",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 11,
+        "w": 11,
+        "x": 0,
+        "y": 12
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rebuilderd_workers",
+          "interval": "",
+          "legendFormat": "active workers",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Active rebuilderd workers",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "The rebuilderd queue length of to be rebuild packages",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 11,
+        "w": 13,
+        "x": 11,
+        "y": 12
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rebuilderd_queue_length",
+          "interval": "",
+          "legendFormat": "queue length",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Queue length",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "selected": true,
+          "text": "extra",
+          "value": "extra"
+        },
+        "datasource": "Prometheus",
+        "definition": "label_values(rebuilderd_results,suite)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "suite",
+        "multi": false,
+        "name": "suite",
+        "options": [
+          {
+            "selected": false,
+            "text": "community",
+            "value": "community"
+          },
+          {
+            "selected": false,
+            "text": "core",
+            "value": "core"
+          },
+          {
+            "selected": true,
+            "text": "extra",
+            "value": "extra"
+          },
+          {
+            "selected": false,
+            "text": "testing",
+            "value": "testing"
+          }
+        ],
+        "query": "label_values(rebuilderd_results,suite)",
+        "refresh": 0,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Rebuilderd",
+  "uid": null,
+  "version": 0
+}

roles/prometheus_exporters/files/rebuilderd-status-textcollector.py

+#!/usr/bin/python
+
+import sys
+import shutil
+import tempfile
+
+from collections import defaultdict
+
+import requests
+
+# The rebuilderd instance
+API_URL = 'https://reproducible.archlinux.org'
+
+
+def get_metric_header():
+    return '# HELP rebuilderd_results number of packages\n# TYPE rebuilderd_results gauge\n'
+
+
+def format_metric(suite, status, total):
+    status = status.lower()
+    return f'rebuilderd_results{{suite="{suite}",status="{status}"}} {total}\n'
+
+
+def get_rebuilderd_data():
+    req = requests.get(f'{API_URL}/api/v0/pkgs/list')
+    if req.status_code != 200:
+        print(f'Failed to obtain rebuilderd data, http status code: {req.status_code}', file=sys.stderr)
+        sys.exit(1)
+
+    data = req.json()
+    dataset = defaultdict(dict)
+
+    for entry in data:
+        suite = entry['suite']
+        status = entry['status']
+
+        if suite in dataset:
+            if status in dataset[suite]:
+                dataset[suite][status] += 1
+            else:
+                dataset[suite][status] = 1
+        else:
+            dataset[suite] = defaultdict(dict)
+            dataset[suite][status] = 1
+
+    return dataset
+
+
+def main(directory):
+    dataset = get_rebuilderd_data()
+
+    with tempfile.TemporaryDirectory() as tmpfp:
+        filename = f'{tmpfp}/rebuilderd_status.prom'
+        print(filename)
+
+        with open(filename, 'w') as fp:
+            fp.write(get_metric_header())
+            for suite, data in dataset.items():
+                for status, total in data.items():
+                    fp.write(format_metric(suite, status, total))
+
+        shutil.move(filename, f'{directory}/rebuilderd-status.prom')
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print('Missing textcollector directory argument')
+
+    main(sys.argv[1])

roles/prometheus_exporters/tasks/main.yml

@@ -57,6 +57,7 @@
     - arch-textcollector.sh
     - borg-textcollector.sh
     - rebuilderd-textcollector.sh
+    - rebuilderd-status-textcollector.py
 
 - name: install arch textcollector service
   template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600

roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.service.j2

@@ -6,6 +6,7 @@ After=network.target
 Type=oneshot
 User=node_exporter
 ExecStart=/usr/local/bin/rebuilderd-textcollector.sh {{ prometheus_textfile_dir }}
+ExecStart=/usr/local/bin/rebuilderd-status-textcollector.py {{ prometheus_textfile_dir }}
 
 NoNewPrivileges=true
 LockPersonality=true