Merge branch 'flyspray-textcollector' into 'master'

Collect flyspray open issues

See merge request !725

roles/grafana/files/public-dashboards/flyspray.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 71,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "description": "Currently open bugs on https://bugs.archlinux.org",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 22,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": "Prometheus",
+          "editorMode": "builder",
+          "expr": "flyspray_issues",
+          "legendFormat": "{{ project }}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Open bugs",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Flyspray",
+  "uid": "c9a9b812-a067-4534-96bf-c21ba7a0c47d",
+  "version": 2,
+  "weekStart": ""
+}

roles/prometheus/defaults/main.yml

@@ -11,7 +11,7 @@ prometheus_remote_write_relabel_configs:
   - label: job
     regex: goaurrpc
   - label: __name__
-    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
+    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+|flyspray_issues"
 
 blackbox_targets:
   http_prometheus:

roles/prometheus_exporters/files/flyspray-textcollector.py

+#!/usr/bin/python
+
+import sys
+import shutil
+import tempfile
+
+import sqlalchemy
+
+engine = sqlalchemy.create_engine('mysql://localhost/flyspray', connect_args={'read_default_file': '/root/.my.cnf'})
+
+
+def get_flyspray_header():
+    return '# HELP flyspray_issues number of open issues\n# TYPE flyspray_issues gauge\n'
+
+
+def format_metric(project, total):
+    return f'flyspray_issues{{project="{project}"}} {total}\n'
+
+
+def get_metric(project_id):
+	with engine.connect() as conn:
+		result = conn.execute(f"SELECT count(task_id) from flyspray_tasks where project_id = {project_id} and is_closed = 0")
+		return next(result)[0]
+
+
+def main(directory):
+    with tempfile.TemporaryDirectory() as tmpfp:
+        filename = f'{tmpfp}/flyspray-status.prom'
+        print(filename)
+
+        with open(filename, 'w') as fp:
+            fp.write(get_flyspray_header())
+            fp.write(format_metric("Arch Linux", get_metric(1)))
+            fp.write(format_metric("Community", get_metric(5)))
+
+        shutil.move(filename, f'{directory}/flyspray-status.prom')
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print('Missing textcollector directory argument')
+
+    main(sys.argv[1])

roles/prometheus_exporters/tasks/main.yml

@@ -58,6 +58,7 @@
     - hetzner-textcollector.sh
     - rebuilderd-textcollector.sh
     - rebuilderd-status-textcollector.py
+    - flyspray-textcollector.py
     - archive-textcollector.sh
     - repository-textcollector.sh
     - btrfs-textcollector.sh
@@ -181,6 +182,18 @@
   systemd: name=prometheus-repository-textcollector.timer enabled=yes daemon_reload=yes state=started
   when: "inventory_hostname == 'gemini.archlinux.org'"
 
+- name: Install flyspray textcollector service
+  template: src=prometheus-flyspray-textcollector.service.j2 dest=/etc/systemd/system/prometheus-flyspray-textcollector.service owner=root group=root mode=644
+  when: "inventory_hostname == 'bugs.archlinux.org'"
+
+- name: Install flyspray textcollector timer
+  template: src=prometheus-flyspray-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-flyspray-textcollector.timer owner=root group=root mode=644
+  when: "inventory_hostname == 'bugs.archlinux.org'"
+
+- name: Enable and start prometheus flyspray textcollector timer
+  systemd: name=prometheus-flyspray-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "inventory_hostname == 'bugs.archlinux.org'"
+
 - name: Install sudoers for btrfs
   copy: src=sudoers dest=/etc/sudoers.d/node_exporter owner=root group=root mode=0440
   when: filesystem == "btrfs"

roles/prometheus_exporters/templates/prometheus-flyspray-textcollector.service.j2

+[Unit]
+Description=Prometheus Flyspray Exporter
+After=mysqld.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/flyspray-textcollector.py {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native

roles/prometheus_exporters/templates/prometheus-flyspray-textcollector.timer.j2

+[Unit]
+Description=Prometheus Flyspray Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=5m
+OnBootSec=2min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target