From 2a5108b75ad10d0a12c808661a0ff91c2c88e4f4 Mon Sep 17 00:00:00 2001
From: Evangelos Foutras <evangelos@foutrelis.com>
Date: Fri, 2 Jul 2021 00:23:35 +0300
Subject: [PATCH] Use sub-accounts for backups to Hetzner Storage Box
This offers improved separation between the server backups and should
avoid bumping against the storage box 10 concurrent connection limit.
Fixes: https://gitlab.archlinux.org/archlinux/infrastructure/-/issues/362
---
docs/backups.md | 4 +-
group_vars/all/vault_hetzner_storagebox.yml | 19 ++---
group_vars/all/vault_hetzner_webservice.yml | 10 +++
roles/borg_client/defaults/main.yml | 4 +-
roles/hetzner_storagebox/tasks/main.yml | 76 ++++++++++++++++++-
.../tasks/upload_client_authorized_keys.yml | 13 ++++
.../templates/authorized_keys.j2 | 5 --
.../templates/authorized_keys_client.j2 | 1 +
8 files changed, 113 insertions(+), 19 deletions(-)
create mode 100644 group_vars/all/vault_hetzner_webservice.yml
create mode 100644 roles/hetzner_storagebox/tasks/upload_client_authorized_keys.yml
create mode 100644 roles/hetzner_storagebox/templates/authorized_keys_client.j2
diff --git a/docs/backups.md b/docs/backups.md
index f5199e9bb..b515b4a54 100644
--- a/docs/backups.md
+++ b/docs/backups.md
@@ -8,7 +8,7 @@ You'll have to get the correct username from the vault.
We use two different borg backup hosts: A primary one and an offsite one.
The URL format for the primary one is
- ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/<hostname>
+ ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/<hostname>/repo
while for the offsite one it's
@@ -22,7 +22,7 @@ placeholder with your desired full address to the backup repository. For instanc
becomes
- misc/borg.sh ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org::20191127-084357
+ misc/borg.sh ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org/repo::20191127-084357
A convenience wrapper script is available at `misc/borg.sh` which makes sure you
use the correct keyfile for the given server.
diff --git a/group_vars/all/vault_hetzner_storagebox.yml b/group_vars/all/vault_hetzner_storagebox.yml
index c02209ef9..264b5c8b4 100644
--- a/group_vars/all/vault_hetzner_storagebox.yml
+++ b/group_vars/all/vault_hetzner_storagebox.yml
@@ -1,10 +1,11 @@
$ANSIBLE_VAULT;1.1;AES256
-30353736373466623531333732393935376435353939366632383839376531653761656631646638
-3831333465373263336232653931643162656363653039320a383736393636613231386465663430
-37313062303933653633626637623539363565316161666433656138393036343538623863386666
-3039346264393066620a396231646534303262616162346261643639323838313635366332653861
-39353239393134326130383766323832383361656431336335616138363865623865356538636139
-63363234343962333166313038646633613534653963613961656336646464393338373635663832
-62396633363932663931633532363732653766356136393137363366376134363135663864313935
-63323635666431353165396235633066313334316161396163646633366536366361643331386461
-6535
+37316639623364363536666561383935376436663233323431626639316438646631643165663734
+6431386565316161653535656137393634656333633863330a353364633135376461343137663938
+34343133336337656237373961303530393765383338613937313332313433363838383064363061
+3061376164316662350a313534656536303164626464353030356339353237313834336632393039
+65333832336635633730326230323934383733653533346135656431356534303765383365323538
+66323437313839653733323063343436386130343139343761363037646437653263333066316665
+63373335656238336136323961356133353833313737303538313936663366303435306134326138
+30396366336130386263356264336361343231313763326239643562666662346634616639663165
+33653962323336613537616363323931366235373930326436643863376463656165303433643635
+3632343163323137633630373561333731616334306135616234
diff --git a/group_vars/all/vault_hetzner_webservice.yml b/group_vars/all/vault_hetzner_webservice.yml
new file mode 100644
index 000000000..286456c8c
--- /dev/null
+++ b/group_vars/all/vault_hetzner_webservice.yml
@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+39623835306534616661653830393863636639343938313031316132376662316530366330623162
+3936383035626238336439363638633634623036343238300a366332303632323235303038343534
+34643833623730346430623961396464353235393465313264313465653864316131636665373864
+6565333732636366330a326664316638353065643838633865333036616331313637303736303865
+39316434636563653237376131636266333834626331356331613039366664373561353631666336
+63643439383163636239396232626131393431303637623236613433313533666562623339653239
+61336566613136383537623863316533343465343237636563313633653136373734376561373164
+30386533346162333662393337633133666138356139343331313662383038633561313265613534
+38343833313534303839316264633436313831633631666539663062396363333334
diff --git a/roles/borg_client/defaults/main.yml b/roles/borg_client/defaults/main.yml
index f0dc852a9..cbd80fa49 100644
--- a/roles/borg_client/defaults/main.yml
+++ b/roles/borg_client/defaults/main.yml
@@ -1,7 +1,7 @@
---
backup_hosts:
- - host: "ssh://{{ hetzner_storagebox_username }}@u236610.your-storagebox.de:23"
- dir: "~/backup/{{ inventory_hostname }}"
+ - host: "ssh://u236610.your-storagebox.de:23"
+ dir: "~/repo"
suffix: ""
- host: "ssh://{{ rsync_net_username }}@prio.ch-s012.rsync.net:22"
dir: "~/backup/{{ inventory_hostname }}"
diff --git a/roles/hetzner_storagebox/tasks/main.yml b/roles/hetzner_storagebox/tasks/main.yml
index 0b6f15b81..d1e1d382f 100644
--- a/roles/hetzner_storagebox/tasks/main.yml
+++ b/roles/hetzner_storagebox/tasks/main.yml
@@ -9,8 +9,17 @@
(?i)password: "{{ hetzner_storagebox_password }}"
delegate_to: localhost
+- name: create a home directory for each sub-account
+ expect:
+ command: bash -c "echo 'mkdir {{ backup_dir }}/{{ item }}' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+ responses:
+ (?i)password: "{{ hetzner_storagebox_password }}"
+ delegate_to: localhost
+ loop: "{{ backup_clients }}"
+
- name: fetch ssh keys from each borg client machine
command: cat /root/.ssh/id_rsa.pub
+ check_mode: false
register: client_ssh_keys
delegate_to: "{{ item }}"
with_items: "{{ backup_clients }}"
@@ -19,16 +28,81 @@
- name: create tempfile
tempfile: state=file
+ check_mode: false
register: tempfile
delegate_to: localhost
- name: fill tempfile
copy: content="{{ lookup('template', 'authorized_keys.j2') }}" dest="{{ tempfile.path }}" mode=preserve
delegate_to: localhost
+ no_log: true
-- name: upload authorized_keys file
+- name: upload authorized_keys for Arch DevOps
expect:
command: bash -c "echo -e 'mkdir .ssh \n chmod 700 .ssh \n put {{ tempfile.path }} .ssh/authorized_keys \n chmod 600 .ssh/authorized_keys' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
responses:
(?i)password: "{{ hetzner_storagebox_password }}"
delegate_to: localhost
+
+- name: upload authorized_keys for each backup client
+ include_tasks: upload_client_authorized_keys.yml
+ loop: "{{ client_ssh_keys.results }}"
+ loop_control:
+ label: "{{ item.item }}"
+
+- name: retrieve sub-account information
+ uri:
+ url: https://robot-ws.your-server.de/storagebox/{{ hetzner_storagebox_id }}/subaccount
+ user: "{{ hetzner_webservice_username }}"
+ password: "{{ hetzner_webservice_password }}"
+ delegate_to: localhost
+ check_mode: false
+ register: subaccounts_raw
+ no_log: true
+
+- name: get list of sub-accounts
+ set_fact:
+ subaccounts: "{{ subaccounts_raw.json | json_query('[].subaccount') }}"
+
+- name: create missing sub-accounts
+ uri:
+ timeout: 60
+ url: https://robot-ws.your-server.de/storagebox/{{ hetzner_storagebox_id }}/subaccount
+ user: "{{ hetzner_webservice_username }}"
+ password: "{{ hetzner_webservice_password }}"
+ method: POST
+ body_format: form-urlencoded
+ body:
+ homedirectory: "{{ backup_dir }}/{{ item }}"
+ comment: "{{ item }}"
+ ssh: "true"
+ delegate_to: localhost
+ loop: "{{ backup_clients | difference(subaccounts | json_query('[].comment')) }}"
+ register: new_subaccounts_raw
+ no_log: true
+
+- name: update list of sub-accounts
+ set_fact:
+ subaccounts: "{{ subaccounts + [item.json.subaccount | combine({'comment': item.invocation.module_args.body.comment})] }}"
+ loop: "{{ new_subaccounts_raw.results }}"
+ loop_control:
+ label: "{{ item.invocation.module_args.body.comment }}"
+
+- name: match usernames to backup clients
+ set_fact:
+ backup_client_usernames: "{{ backup_client_usernames|default({}) | combine({item.comment: item.username}) }}"
+ loop: "{{ subaccounts }}"
+ loop_control:
+ label: "{{ {item.comment: item.username} }}"
+
+- name: configure ssh on backup clients
+ blockinfile:
+ path: /root/.ssh/config
+ create: true
+ mode: 0600
+ block: |
+ Host {{ inventory_hostname }}
+ User {{ backup_client_usernames[item] }}
+ marker: '# {mark} HETZNER STORAGE BOX BACKUP CLIENT CONFIG'
+ delegate_to: "{{ item }}"
+ loop: "{{ backup_clients }}"
diff --git a/roles/hetzner_storagebox/tasks/upload_client_authorized_keys.yml b/roles/hetzner_storagebox/tasks/upload_client_authorized_keys.yml
new file mode 100644
index 000000000..72a544361
--- /dev/null
+++ b/roles/hetzner_storagebox/tasks/upload_client_authorized_keys.yml
@@ -0,0 +1,13 @@
+---
+
+- name: fill tempfile
+ copy: content="{{ lookup('template', 'authorized_keys_client.j2') }}" dest="{{ tempfile.path }}" mode=preserve
+ delegate_to: localhost
+ no_log: true
+
+- name: upload authorized_keys file to {{ backup_dir }}/{{ item.item }}
+ expect:
+ command: bash -c "echo -e 'mkdir {{ backup_dir }}/{{ item.item }}/.ssh \n chmod 700 {{ backup_dir }}/{{ item.item }}/.ssh \n put {{ tempfile.path }} {{ backup_dir }}/{{ item.item }}/.ssh/authorized_keys \n chmod 600 {{ backup_dir }}/{{ item.item }}/.ssh/authorized_keys' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+ responses:
+ (?i)password: "{{ hetzner_storagebox_password }}"
+ delegate_to: localhost
diff --git a/roles/hetzner_storagebox/templates/authorized_keys.j2 b/roles/hetzner_storagebox/templates/authorized_keys.j2
index 038383da6..d175f50f6 100644
--- a/roles/hetzner_storagebox/templates/authorized_keys.j2
+++ b/roles/hetzner_storagebox/templates/authorized_keys.j2
@@ -10,8 +10,3 @@
{% endif %}
{% endif %}
{% endfor %}
-
-# Client machines keys
-{% for client_key in client_ssh_keys.results %}
-command="borg serve --restrict-to-path {{ backup_dir }}/{{ client_key['item'] }}",restrict {{ client_key['stdout'] }}
-{% endfor %}
diff --git a/roles/hetzner_storagebox/templates/authorized_keys_client.j2 b/roles/hetzner_storagebox/templates/authorized_keys_client.j2
new file mode 100644
index 000000000..4358f3435
--- /dev/null
+++ b/roles/hetzner_storagebox/templates/authorized_keys_client.j2
@@ -0,0 +1 @@
+restrict {{ item['stdout'] }}
--
GitLab