Monitor the used and total size of the storage box

Now that we have enabled snapshots, summing all Borg repository sizes
has become a bad approximation of the total space used on the storage
box.

roles/grafana/files/dashboards/backups.json

@@ -20,6 +20,7 @@
   "panels": [
     {
       "datasource": "Prometheus",
+      "description": "",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -54,6 +55,7 @@
             }
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -72,12 +74,12 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 9,
-        "w": 24,
+        "h": 7,
+        "w": 19,
         "x": 0,
         "y": 0
       },
-      "id": 2,
+      "id": 5,
       "options": {
         "legend": {
           "calcs": [],
@@ -88,37 +90,90 @@
           "mode": "single"
         }
       },
-      "pluginVersion": "8.0.3",
       "targets": [
         {
           "exemplar": true,
-          "expr": "borg_hetzner_repo_size_bytes{}",
-          "format": "time_series",
-          "instant": false,
+          "expr": "hetzner_storage_box_size_bytes{instance=\"monitoring.archlinux.org\"}",
           "interval": "",
-          "legendFormat": "{{ instance }}",
+          "legendFormat": "Size",
           "queryType": "randomWalk",
           "refId": "A"
         },
         {
           "exemplar": true,
-          "expr": "sum(borg_hetzner_repo_size_bytes) + 154573468884",
+          "expr": "hetzner_storage_box_used_bytes{instance=\"monitoring.archlinux.org\"}",
           "hide": false,
           "interval": "",
-          "legendFormat": "Total size of all repos (+ apollo)",
+          "legendFormat": "Used",
           "refId": "B"
+        }
+      ],
+      "title": "Disk Usage (Hetzner)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "percentage",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "#EAB839",
+                "value": 85
+              },
+              {
+                "color": "red",
+                "value": 95
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
         },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "text": {}
+      },
+      "pluginVersion": "8.0.3",
+      "targets": [
         {
           "exemplar": true,
-          "expr": "10000*2^30",
-          "hide": false,
+          "expr": "hetzner_storage_box_used_bytes{instance=\"monitoring.archlinux.org\"} / hetzner_storage_box_size_bytes{instance=\"monitoring.archlinux.org\"}",
           "interval": "",
-          "legendFormat": "Storage Box quota",
-          "refId": "C"
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
         }
       ],
-      "title": "Borg Repository Size (Hetzner)",
-      "type": "timeseries"
+      "title": "Used Space (Hetzner)",
+      "type": "gauge"
     },
     {
       "datasource": "Prometheus",
@@ -175,11 +230,11 @@
       },
       "gridPos": {
         "h": 9,
-        "w": 24,
+        "w": 12,
         "x": 0,
-        "y": 9
+        "y": 7
       },
-      "id": 3,
+      "id": 2,
       "options": {
         "legend": {
           "calcs": [],
@@ -194,28 +249,106 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "borg_offsite_repo_size_bytes{}",
+          "expr": "borg_hetzner_repo_size_bytes{}",
           "format": "time_series",
           "instant": false,
           "interval": "",
           "legendFormat": "{{ instance }}",
           "queryType": "randomWalk",
           "refId": "A"
+        }
+      ],
+      "title": "Borg Repository Size (Hetzner)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
         },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.0.3",
+      "targets": [
         {
           "exemplar": true,
-          "expr": "sum(borg_offsite_repo_size_bytes) + 2502953482529 + 151779047659",
-          "hide": false,
+          "expr": "borg_offsite_repo_size_bytes{}",
+          "format": "time_series",
+          "instant": false,
           "interval": "",
-          "legendFormat": "Total size of all repos (+ orion & apollo)",
-          "refId": "B"
+          "legendFormat": "{{ instance }}",
+          "queryType": "randomWalk",
+          "refId": "A"
         }
       ],
       "title": "Borg Repository Size (Offsite)",
       "type": "timeseries"
     }
   ],
-  "refresh": "",
+  "refresh": false,
   "schemaVersion": 30,
   "style": "dark",
   "tags": [],
@@ -223,12 +356,12 @@
     "list": []
   },
   "time": {
-    "from": "now-6M",
+    "from": "now-90d",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "Borg Backups",
   "uid": "Rnqpymznz",
-  "version": 27
+  "version": 28
 }
\ No newline at end of file

roles/prometheus_exporters/files/borg-textcollector.sh

@@ -34,6 +34,18 @@ if [[ -f /usr/local/bin/borg ]]; then
   echo "# HELP borg_hetzner_repo_size_bytes amount of data stored in the repo in bytes" >> $TMP_FILE
   echo "# TYPE borg_hetzner_repo_size_bytes gauge" >> $TMP_FILE
   echo "borg_hetzner_repo_size_bytes $REPO_SIZE" >> $TMP_FILE
+
+  STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
+  STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+  STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+
+  echo "# HELP hetzner_storage_box_size_bytes storage box total size in bytes" >> $TMP_FILE
+  echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
+  echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
+
+  echo "# HELP hetzner_storage_box_used_bytes storage box space used in bytes" >> $TMP_FILE
+  echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
+  echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
 fi
 
 # rsync.net borg