roles/grafana/files/dashboards/backups.json

@@ -20,6 +20,7 @@
   "panels": [
     {
       "datasource": "Prometheus",
+      "description": "",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -72,12 +73,12 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 9,
-        "w": 24,
+        "h": 7,
+        "w": 19,
         "x": 0,
         "y": 0
       },
-      "id": 2,
+      "id": 5,
       "options": {
         "legend": {
           "calcs": [],
@@ -88,38 +89,107 @@
           "mode": "single"
         }
       },
-      "pluginVersion": "8.0.3",
       "targets": [
         {
           "exemplar": true,
-          "expr": "borg_hetzner_repo_size_bytes{}",
-          "format": "time_series",
-          "instant": false,
+          "expr": "10000*2^30",
           "interval": "",
-          "legendFormat": "{{ instance }}",
+          "legendFormat": "Quota",
           "queryType": "randomWalk",
           "refId": "A"
         },
         {
           "exemplar": true,
-          "expr": "sum(borg_hetzner_repo_size_bytes) + 154573468884",
+          "expr": "hetzner_storage_box_used_bytes{instance=\"monitoring.archlinux.org\"}",
           "hide": false,
           "interval": "",
-          "legendFormat": "Total size of all repos (+ apollo)",
+          "legendFormat": "Backups",
           "refId": "B"
         },
         {
           "exemplar": true,
-          "expr": "10000*2^30",
+          "expr": "10000*2^30 - hetzner_storage_box_size_bytes{instance=\"monitoring.archlinux.org\"}",
           "hide": false,
           "interval": "",
-          "legendFormat": "Storage Box quota",
+          "legendFormat": "Snapshots",
           "refId": "C"
+        },
+        {
+          "exemplar": true,
+          "expr": "10000*2^30 - hetzner_storage_box_free_bytes{instance=\"monitoring.archlinux.org\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Total Used",
+          "refId": "D"
         }
       ],
-      "title": "Borg Repository Size (Hetzner)",
+      "title": "Disk Usage (Hetzner)",
       "type": "timeseries"
     },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 10737418240000,
+          "min": 0,
+          "thresholds": {
+            "mode": "percentage",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "green",
+                "value": 15
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "id": 7,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "text": {}
+      },
+      "pluginVersion": "8.0.3",
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "hetzner_storage_box_free_bytes{instance=\"monitoring.archlinux.org\"}",
+          "interval": "",
+          "legendFormat": "",
+          "queryType": "randomWalk",
+          "refId": "A"
+        }
+      ],
+      "title": "Free Space (Hetzner)",
+      "type": "gauge"
+    },
     {
       "datasource": "Prometheus",
       "fieldConfig": {
@@ -175,11 +245,11 @@
       },
       "gridPos": {
         "h": 9,
-        "w": 24,
+        "w": 12,
         "x": 0,
-        "y": 9
+        "y": 7
       },
-      "id": 3,
+      "id": 2,
       "options": {
         "legend": {
           "calcs": [],
@@ -194,28 +264,106 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "borg_offsite_repo_size_bytes{}",
+          "expr": "borg_hetzner_repo_size_bytes{}",
           "format": "time_series",
           "instant": false,
           "interval": "",
           "legendFormat": "{{ instance }}",
           "queryType": "randomWalk",
           "refId": "A"
+        }
+      ],
+      "title": "Borg Repository Size (Hetzner)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
         },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.0.3",
+      "targets": [
         {
           "exemplar": true,
-          "expr": "sum(borg_offsite_repo_size_bytes) + 2502953482529 + 151779047659",
-          "hide": false,
+          "expr": "borg_offsite_repo_size_bytes{}",
+          "format": "time_series",
+          "instant": false,
           "interval": "",
-          "legendFormat": "Total size of all repos (+ orion & apollo)",
-          "refId": "B"
+          "legendFormat": "{{ instance }}",
+          "queryType": "randomWalk",
+          "refId": "A"
         }
       ],
       "title": "Borg Repository Size (Offsite)",
       "type": "timeseries"
     }
   ],
-  "refresh": "",
+  "refresh": false,
   "schemaVersion": 30,
   "style": "dark",
   "tags": [],
@@ -223,12 +371,12 @@
     "list": []
   },
   "time": {
-    "from": "now-6M",
+    "from": "now-90d",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "Borg Backups",
   "uid": "Rnqpymznz",
-  "version": 27
+  "version": 30
 }
\ No newline at end of file

roles/prometheus_exporters/files/borg-textcollector.sh

@@ -34,6 +34,23 @@ if [[ -f /usr/local/bin/borg ]]; then
   echo "# HELP borg_hetzner_repo_size_bytes amount of data stored in the repo in bytes" >> $TMP_FILE
   echo "# TYPE borg_hetzner_repo_size_bytes gauge" >> $TMP_FILE
   echo "borg_hetzner_repo_size_bytes $REPO_SIZE" >> $TMP_FILE
+
+  STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
+  STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+  STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+  STORAGE_BOX_FREE=$(( 1024 * $(awk '{print $3}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+
+  echo "# HELP hetzner_storage_box_size_bytes storage box size in bytes (excl. snapshots)" >> $TMP_FILE
+  echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
+  echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
+
+  echo "# HELP hetzner_storage_box_used_bytes storage box used space in bytes" >> $TMP_FILE
+  echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
+  echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
+
+  echo "# HELP hetzner_storage_box_free_bytes storage box free space in bytes" >> $TMP_FILE
+  echo "# TYPE hetzner_storage_box_free_bytes gauge" >> $TMP_FILE
+  echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
 fi
 
 # rsync.net borg