From 87b2eddfae76a623ac06aab882d3adae7d83b082 Mon Sep 17 00:00:00 2001
From: Leonidas Spyropoulos <artafinde@archlinux.org>
Date: Tue, 18 Oct 2022 20:17:43 +0300
Subject: [PATCH] aurweb: enable goaurrpc metrics and dashboard
Signed-off-by: Leonidas Spyropoulos <artafinde@archlinux.org>
Co-authored-by: Kristian Klausen <kristian@klausen.dk>
Co-authored-by: moson-mo <mo-son@mailbox.org>
---
group_vars/all/vault_goaurrpc.yml | 8 +
roles/aurweb/tasks/main.yml | 2 +-
roles/aurweb/templates/goaurrpc.conf.j2 | 3 +-
roles/aurweb/templates/nginx.d.conf.j2 | 9 +
roles/grafana/files/dashboards/goaurrpc.json | 792 ++++++++++++++++++
.../files/public-dashboards/goaurrpc.json | 1 +
roles/prometheus/defaults/main.yml | 2 +
roles/prometheus/templates/prometheus.yml.j2 | 8 +
8 files changed, 823 insertions(+), 2 deletions(-)
create mode 100644 group_vars/all/vault_goaurrpc.yml
create mode 100644 roles/grafana/files/dashboards/goaurrpc.json
create mode 120000 roles/grafana/files/public-dashboards/goaurrpc.json
diff --git a/group_vars/all/vault_goaurrpc.yml b/group_vars/all/vault_goaurrpc.yml
new file mode 100644
index 000000000..9c62827f1
--- /dev/null
+++ b/group_vars/all/vault_goaurrpc.yml
@@ -0,0 +1,8 @@
+$ANSIBLE_VAULT;1.1;AES256
+64356530383261396436353036356166656231363964356266313061393935636365353931333632
+3566333830343965653766316236633563353662633031610a663030643166313432336337346430
+35326662346566383630653433633230613665333264313036303038663063366561636466633635
+6131656134333232640a386363643834383363643665343333653531653231346565393764366137
+35303637316530623136373838626536373265633836623937326234353133383235646461343635
+31346335353939333532336463616130323338316236376163383033313134323133333539613139
+623764623337366432393537353733613538
diff --git a/roles/aurweb/tasks/main.yml b/roles/aurweb/tasks/main.yml
index 588c83d02..fa560cabf 100644
--- a/roles/aurweb/tasks/main.yml
+++ b/roles/aurweb/tasks/main.yml
@@ -233,7 +233,7 @@
domains: ["{{ aurweb_domain }}"]
- name: Set up nginx
- template: src=nginx.d.conf.j2 dest={{ aurweb_nginx_conf }} owner=root group=root mode=644
+ template: src=nginx.d.conf.j2 dest={{ aurweb_nginx_conf }} owner=root group=http mode=640
notify: Reload nginx
tags: ['nginx']
diff --git a/roles/aurweb/templates/goaurrpc.conf.j2 b/roles/aurweb/templates/goaurrpc.conf.j2
index a36e5f951..ab10b3d51 100644
--- a/roles/aurweb/templates/goaurrpc.conf.j2
+++ b/roles/aurweb/templates/goaurrpc.conf.j2
@@ -16,5 +16,6 @@
"KeyFile": "",
"EnableSearchCache": true,
"CacheCleanupInterval": 60,
- "CacheExpirationTime": 180
+ "CacheExpirationTime": 180,
+ "EnableMetrics": true
}
diff --git a/roles/aurweb/templates/nginx.d.conf.j2 b/roles/aurweb/templates/nginx.d.conf.j2
index 0ddfdea2a..3af9f27e5 100644
--- a/roles/aurweb/templates/nginx.d.conf.j2
+++ b/roles/aurweb/templates/nginx.d.conf.j2
@@ -118,6 +118,15 @@ server {
proxy_set_header X-Forwarded-For $remote_addr;
}
+ location /rpc/metrics {
+ if ($http_authorization != "Bearer {{ vault_goaurrpc_metrics_token }}") {
+ return 403;
+ }
+
+ proxy_pass http://127.0.0.1:10666/metrics;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ }
+
location / {
# Proxy over to aurweb's ASGI application.
proxy_pass http://{{ aurweb_asgi_bind }};
diff --git a/roles/grafana/files/dashboards/goaurrpc.json b/roles/grafana/files/dashboards/goaurrpc.json
new file mode 100644
index 000000000..cecedb2e2
--- /dev/null
+++ b/roles/grafana/files/dashboards/goaurrpc.json
@@ -0,0 +1,792 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_GOAURRPC",
+ "label": "goaurrpc",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "piechart",
+ "name": "Pie chart",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "/rpc metrics",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "requests_rate_limited"
+ },
+ "properties": [
+ {
+ "id": "thresholds",
+ "value": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "last_refresh"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "dateTimeAsIsoNoDateIfToday"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 14,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "rpc_data_last_refresh * 1000",
+ "hide": false,
+ "legendFormat": "Last metadata refresh",
+ "range": true,
+ "refId": "last_refresh"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "sum(rpc_requests)",
+ "hide": false,
+ "legendFormat": "Requests served",
+ "range": true,
+ "refId": "requests"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "sum(rpc_requests_rate_limited)",
+ "legendFormat": "Requests rate-limited",
+ "range": true,
+ "refId": "requests_rate_limited"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "rpc_cache_hits",
+ "hide": false,
+ "legendFormat": "Cache hits",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "mappings": []
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 5,
+ "x": 6,
+ "y": 0
+ },
+ "id": 22,
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "sum by(type) (rpc_requests)",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Requests by type",
+ "type": "piechart"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 13,
+ "x": 11,
+ "y": 0
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "go_memstats_heap_alloc_bytes{job=\"goaurrpc\"}",
+ "legendFormat": "HEAP",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "process_resident_memory_bytes{job=\"goaurrpc\"}",
+ "hide": false,
+ "legendFormat": "RSS",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Memory usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 24,
+ "x": 0,
+ "y": 7
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "sum by(type) (rate(rpc_requests[5m]))",
+ "legendFormat": "{{type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Requests per second by type",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 24,
+ "x": 0,
+ "y": 12
+ },
+ "id": 8,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "asc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "exemplar": false,
+ "expr": "sum by(by) (rate(rpc_requests{type=\"search\"}[5m]))",
+ "instant": false,
+ "legendFormat": "{{by}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Search requests per second",
+ "transformations": [],
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "id": 10,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(rpc_request_duration_seconds_bucket[5m])))",
+ "legendFormat": "quantile95",
+ "range": true,
+ "refId": "quantile95"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(rpc_request_duration_seconds_bucket[5m])))",
+ "hide": false,
+ "legendFormat": "quantile50",
+ "range": true,
+ "refId": "quantile50"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "rate(rpc_request_duration_seconds_sum[5m]) / rate(rpc_request_duration_seconds_count[5m])",
+ "hide": false,
+ "legendFormat": "average",
+ "range": true,
+ "refId": "average"
+ }
+ ],
+ "title": "Request duration",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 24,
+ "x": 0,
+ "y": 22
+ },
+ "id": 16,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "Prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "sum(rate(rpc_response_size_bytes_sum[5m])) / sum(rate(rpc_response_size_bytes_count[5m]))",
+ "legendFormat": "average",
+ "range": true,
+ "refId": "average"
+ }
+ ],
+ "title": "Response size",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 37,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "goaurrpc",
+ "uid": "mGJE0LS4z",
+ "version": 1,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/roles/grafana/files/public-dashboards/goaurrpc.json b/roles/grafana/files/public-dashboards/goaurrpc.json
new file mode 120000
index 000000000..bc91edb61
--- /dev/null
+++ b/roles/grafana/files/public-dashboards/goaurrpc.json
@@ -0,0 +1 @@
+../dashboards/goaurrpc.json
\ No newline at end of file
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
index 6e6a1199b..95c79764e 100644
--- a/roles/prometheus/defaults/main.yml
+++ b/roles/prometheus/defaults/main.yml
@@ -8,6 +8,8 @@ prometheus_remote_write_relabel_configs:
regex: debuginfod
- label: job
regex: aurweb
+ - label: job
+ regex: goaurrpc
- label: __name__
regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2
index 1265722ee..1b3f7a91f 100644
--- a/roles/prometheus/templates/prometheus.yml.j2
+++ b/roles/prometheus/templates/prometheus.yml.j2
@@ -137,6 +137,14 @@ scrape_configs:
static_configs:
- targets: ['aur.archlinux.org:443']
+ - job_name: goaurrpc
+ metrics_path: /rpc/metrics
+ scheme: https
+ authorization:
+ credentials: {{ vault_goaurrpc_metrics_token }}
+ static_configs:
+ - targets: ['aur.archlinux.org:443']
+
- job_name: grafana
scheme: https
authorization:
--
GitLab