From 9f9378b2dcd6516b75b807c301ee9af1d042bcbc Mon Sep 17 00:00:00 2001
From: Jelle van der Waa <jelle@archlinux.org>
Date: Tue, 2 Mar 2021 17:55:55 +0100
Subject: [PATCH 1/2] Adjust prometheus textcollector condition
Currently our textcollector can sometimes fail with 'Failed to
create/acquire the lock /home/backup/$server/lock.exclusive (timeout)."
Instead of checking on a borg lock file, check if our backup snapshot
dir exists which the backup script creates and removes. This should give
less false positives then our current method.
---
.../templates/prometheus-borg-textcollector.service.j2 | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2
index 593a774ee..117b1b4ad 100644
--- a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2
+++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2
@@ -1,7 +1,9 @@
[Unit]
Description=Prometheus Borg Exporter TextCollector
After=network.target
-ConditionPathExistsGlob=!/root/.cache/borg/*/lock.roster
+# If the backup snapshot path exists, the backup script runs and a borg
+# lockfile exists and we can't run borg list
+ConditionPathExists=!/backup
[Service]
Type=oneshot
--
GitLab
From 9b9e9a3e9cb7bc27f06f8cf77432dd4bff5a5c75 Mon Sep 17 00:00:00 2001
From: Jelle van der Waa <jelle@archlinux.org>
Date: Tue, 2 Mar 2021 18:14:30 +0100
Subject: [PATCH 2/2] Add a trap to borg backup for cleanup
Now that our textcollector depends on the condition of /backup being
available, we should be extra careful that we get rid of the directory.
If anything in the script fails, a trap now umounts and removes the
directory automatically if on btrfs.being available, we should be extra
careful that we get rid of the directory. If anything in the script
fails, a trap now umounts and removes the directory automatically if on
btrfs.
---
roles/borg_client/templates/borg-backup.sh.j2 | 25 +++++++++++++------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/roles/borg_client/templates/borg-backup.sh.j2 b/roles/borg_client/templates/borg-backup.sh.j2
index 4a333b978..bfd62ce2a 100644
--- a/roles/borg_client/templates/borg-backup.sh.j2
+++ b/roles/borg_client/templates/borg-backup.sh.j2
@@ -6,6 +6,8 @@ src="/"
snapshotdir="backup-snapshot"
backup_mountdir="/backup"
+declare -a list_of_btrfs_submounts
+
##
# usage : is_btrfs( $path )
# return : whether $path is on a btrfs
@@ -22,6 +24,21 @@ delete_snapshot() {
btrfs subvolume delete --commit-after "$1"
}
+##
+# exit cleanup handler
+##
+cleanup() {
+ if is_btrfs "$src"; then
+ umount -R "$backup_mountdir"
+ for vol in $list_of_btrfs_submounts; do
+ delete_snapshot "$vol/$snapshotdir"
+ done
+ rmdir "$backup_mountdir"
+ fi
+}
+
+trap cleanup EXIT
+
if is_btrfs "$src"; then
# List all btrfs submounts we want to backup, e.g. homedir.archlinux.org with "/ /home"
list_of_btrfs_submounts=$(findmnt -Rl -o target,fstype,options / | grep btrfs | grep -v docker | grep -v "subvol=\/[[:alnum:]]" | cut -d ' ' -f1)
@@ -75,11 +92,3 @@ borg create -v --stats -C zstd \
-e "$backup_mountdir/var/lib/docker" \
{{ item['host'] }}/{{ item['dir'] }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
borg prune -v {{ item['host'] }}/{{ item['dir'] }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
-
-if is_btrfs "$src"; then
- umount -R "$backup_mountdir"
- for vol in $list_of_btrfs_submounts; do
- delete_snapshot "$vol/$snapshotdir"
- done
- rmdir "$backup_mountdir"
-fi
--
GitLab