From 9795a8c3863457a7597e520dc5d935c1e0946098 Mon Sep 17 00:00:00 2001
From: Evangelos Foutras <evangelos@foutrelis.com>
Date: Thu, 1 Jul 2021 21:24:28 +0300
Subject: [PATCH] Also track the free space on the storage box

As it turns out, the total size does not include the snapshots (feature
which we recently enabled). Record the free space so we can have a more
accurate picture of the usable space remaining.
---
 roles/grafana/files/dashboards/backups.json   | 43 +++++++++++++------
 .../files/borg-textcollector.sh               |  9 +++-
 2 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/roles/grafana/files/dashboards/backups.json b/roles/grafana/files/dashboards/backups.json
index 8aae25e05..6bd9d015c 100644
--- a/roles/grafana/files/dashboards/backups.json
+++ b/roles/grafana/files/dashboards/backups.json
@@ -55,7 +55,6 @@
             }
           },
           "mappings": [],
-          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -93,9 +92,9 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "hetzner_storage_box_size_bytes{instance=\"monitoring.archlinux.org\"}",
+          "expr": "10000*2^30",
           "interval": "",
-          "legendFormat": "Size",
+          "legendFormat": "Quota",
           "queryType": "randomWalk",
           "refId": "A"
         },
@@ -104,8 +103,24 @@
           "expr": "hetzner_storage_box_used_bytes{instance=\"monitoring.archlinux.org\"}",
           "hide": false,
           "interval": "",
-          "legendFormat": "Used",
+          "legendFormat": "Backups",
           "refId": "B"
+        },
+        {
+          "exemplar": true,
+          "expr": "10000*2^30 - hetzner_storage_box_size_bytes{instance=\"monitoring.archlinux.org\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Snapshots",
+          "refId": "C"
+        },
+        {
+          "exemplar": true,
+          "expr": "10000*2^30 - hetzner_storage_box_free_bytes{instance=\"monitoring.archlinux.org\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Total Used",
+          "refId": "D"
         }
       ],
       "title": "Disk Usage (Hetzner)",
@@ -119,26 +134,26 @@
             "mode": "thresholds"
           },
           "mappings": [],
-          "max": 1,
+          "max": 10737418240000,
           "min": 0,
           "thresholds": {
             "mode": "percentage",
             "steps": [
               {
-                "color": "green",
+                "color": "red",
                 "value": null
               },
               {
-                "color": "#EAB839",
-                "value": 85
+                "color": "yellow",
+                "value": 5
               },
               {
-                "color": "red",
-                "value": 95
+                "color": "green",
+                "value": 15
               }
             ]
           },
-          "unit": "percentunit"
+          "unit": "bytes"
         },
         "overrides": []
       },
@@ -165,14 +180,14 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "hetzner_storage_box_used_bytes{instance=\"monitoring.archlinux.org\"} / hetzner_storage_box_size_bytes{instance=\"monitoring.archlinux.org\"}",
+          "expr": "hetzner_storage_box_free_bytes{instance=\"monitoring.archlinux.org\"}",
           "interval": "",
           "legendFormat": "",
           "queryType": "randomWalk",
           "refId": "A"
         }
       ],
-      "title": "Used Space (Hetzner)",
+      "title": "Free Space (Hetzner)",
       "type": "gauge"
     },
     {
@@ -363,5 +378,5 @@
   "timezone": "",
   "title": "Borg Backups",
   "uid": "Rnqpymznz",
-  "version": 28
+  "version": 30
 }
\ No newline at end of file
diff --git a/roles/prometheus_exporters/files/borg-textcollector.sh b/roles/prometheus_exporters/files/borg-textcollector.sh
index af88fe38e..5f723a891 100755
--- a/roles/prometheus_exporters/files/borg-textcollector.sh
+++ b/roles/prometheus_exporters/files/borg-textcollector.sh
@@ -38,14 +38,19 @@ if [[ -f /usr/local/bin/borg ]]; then
   STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
   STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
   STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+  STORAGE_BOX_FREE=$(( 1024 * $(awk '{print $3}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
 
-  echo "# HELP hetzner_storage_box_size_bytes storage box total size in bytes" >> $TMP_FILE
+  echo "# HELP hetzner_storage_box_size_bytes storage box size in bytes (excl. snapshots)" >> $TMP_FILE
   echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
   echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
 
-  echo "# HELP hetzner_storage_box_used_bytes storage box space used in bytes" >> $TMP_FILE
+  echo "# HELP hetzner_storage_box_used_bytes storage box used space in bytes" >> $TMP_FILE
   echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
   echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
+
+  echo "# HELP hetzner_storage_box_free_bytes storage box free space in bytes" >> $TMP_FILE
+  echo "# TYPE hetzner_storage_box_free_bytes gauge" >> $TMP_FILE
+  echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
 fi
 
 # rsync.net borg
-- 
GitLab