Run borg-textcollector after each backup completes

Instead of gathering borg statistics every hour or so, run the text
collector script only once after each borg-backup service finishes.

Also split the borg text collector script into two similar scripts,
where each one gathers borg statistics for its respective borg host.

roles/prometheus_exporters/files/borg-offsite-textcollector.sh

+#!/usr/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/borg-offsite.prom
+
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+# rsync.net borg
+if [[ -f /usr/local/bin/borg-offsite ]]; then
+  LAST_ARCHIVE=$(/usr/local/bin/borg-offsite list --last 1)
+  LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}')
+  LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}')
+  LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s")
+
+  echo "# HELP borg_offsite_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE
+  echo "# TYPE borg_offsite_last_archive_timestamp counter" >> $TMP_FILE
+  echo "borg_offsite_last_archive_timestamp $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE;
+
+  REPO_SIZE=$(/usr/local/bin/borg-offsite info --json | jq '.cache.stats.unique_csize')
+
+  echo "# HELP borg_offsite_repo_size_bytes amount of data stored in the repo in bytes" >> $TMP_FILE
+  echo "# TYPE borg_offsite_repo_size_bytes gauge" >> $TMP_FILE
+  echo "borg_offsite_repo_size_bytes $REPO_SIZE" >> $TMP_FILE
+fi
+
+mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/files/borg-textcollector.sh

@@ -53,22 +53,4 @@ if [[ -f /usr/local/bin/borg ]]; then
   echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
 fi
 
-# rsync.net borg
-if [[ -f /usr/local/bin/borg-offsite ]]; then
-  LAST_ARCHIVE=$(/usr/local/bin/borg-offsite list --last 1)
-  LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}')
-  LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}')
-  LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s")
-
-  echo "# HELP borg_offsite_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE
-  echo "# TYPE borg_offsite_last_archive_timestamp counter" >> $TMP_FILE
-  echo "borg_offsite_last_archive_timestamp $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE;
-
-  REPO_SIZE=$(/usr/local/bin/borg-offsite info --json | jq '.cache.stats.unique_csize')
-
-  echo "# HELP borg_offsite_repo_size_bytes amount of data stored in the repo in bytes" >> $TMP_FILE
-  echo "# TYPE borg_offsite_repo_size_bytes gauge" >> $TMP_FILE
-  echo "borg_offsite_repo_size_bytes $REPO_SIZE" >> $TMP_FILE
-fi
-
 mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/tasks/main.yml

@@ -52,6 +52,7 @@
   with_items:
     - arch-textcollector.sh
     - borg-textcollector.sh
+    - borg-offsite-textcollector.sh
     - rebuilderd-textcollector.sh
     - rebuilderd-status-textcollector.py
     - archive-textcollector.sh
@@ -69,16 +70,18 @@
 - name: enable and start prometheus arch textcollector timer
   systemd: name=prometheus-arch-textcollector.timer enabled=yes daemon_reload=yes state=started
 
-- name: install borg textcollector service
-  template: src=prometheus-borg-textcollector.service.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.service owner=root group=root mode=644
+- name: install borg textcollector services
+  template: src=prometheus-borg-textcollector.service.j2 dest=/etc/systemd/system/prometheus-{{ item.name }}-textcollector.service owner=root group=root mode=644
+  loop:
+    - { name: borg, service: borg-backup }
+    - { name: borg-offsite, service: borg-backup-offsite }
   when: "'borg_clients' in group_names"
 
-- name: install borg textcollector timer
-  template: src=prometheus-borg-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.timer owner=root group=root mode=644
-  when: "'borg_clients' in group_names"
-
-- name: enable and start prometheus borg textcollector timer
-  systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started
+- name: enable borg textcollector services
+  systemd: name=prometheus-{{ item.name }}-textcollector.service enabled=yes daemon_reload=yes
+  loop:
+    - { name: borg, service: borg-backup }
+    - { name: borg-offsite, service: borg-backup-offsite }
   when: "'borg_clients' in group_names"
 
 - name: install fail2ban textcollector service

roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2

 [Unit]
-Description=Prometheus Borg Exporter TextCollector
+Description=Prometheus {{ item.name|capitalize }} Exporter TextCollector
 After=network.target
-# If the backup snapshot path exists, the backup script runs and a borg
-# lockfile exists and we can't run borg list
-ConditionPathExists=!/backup
+After={{ item.service }}.service
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/borg-textcollector.sh {{ prometheus_textfile_dir }}
+ExecStart=/usr/local/bin/{{ item.name }}-textcollector.sh {{ prometheus_textfile_dir }}
 
 NoNewPrivileges=true
 LockPersonality=true
@@ -35,3 +33,6 @@ ProtectKernelModules=true
 ProtectClock=true
 
 SystemCallArchitectures=native
+
+[Install]
+WantedBy={{ item.service }}.service

roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2

-[Unit]
-Description=Prometheus Borg Exporter TextCollector Timer
-
-[Timer]
-OnUnitActiveSec=1h
-OnBootSec=15min
-RandomizedDelaySec=30min
-
-[Install]
-WantedBy=timers.target