Switch from vostok to storagebox (fixes #51)

README.md

@@ -147,11 +147,6 @@ The following steps should be used to update our managed servers:
 
 ## Servers
 
-### vostok
-
-#### Services
-  - backups
-
 ### orion
 
 #### Services
@@ -291,8 +286,8 @@ Adding a new server to be backed up goes as following:
 
 * Add the server to [borg-clients] in hosts
 
-* Run the borg role on vostok to allow the new machine to create backups
-  ansibe-playbook playbooks/vostok.yml -t borg
+* Run the borg role on u236610.your-storagebox.de to allow the new machine to create backups
+  ansibe-playbook playbooks/hetzner_storagebox.yml
 
 * Run the borg role for rsync.net to allow the new machine to create backups
   ansibe-playbook playbooks/rsync.net.yml
@@ -300,23 +295,25 @@ Adding a new server to be backed up goes as following:
 * Run the borg role on the new machine to initialize the repository
   ansibe-playbook playbooks/$machine.yml -t borg
 
-Backups should be checked now and then. Some common tasks:
+Backups should be checked now and then. Some common tasks are listed below.
+You'll have to get the correct username from the vault.
 
 ### Listing current backups per server
 
-    borg list borg@vostok.archlinux.org:/backup/<hostname>
+    borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/<hostname>
+    borg list $rsync_net_username@ch-s012.rsync.net:backup/<hostname>
 
 Example
 
-    borg list borg@vostok.archlinux.org:/backup/homedir.archlinux.org
+    borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/homedir.archlinux.org
 
 ### Listing files in a backup
 
-    borg list borg@vostok.archlinux.org:/backup/<hostname>::<archive name>
+    borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/<hostname>::<archive name>
 
 Example
 
-    borg list borg@vostok.archlinux.org:/backup/homedir.archlinux.org::20191127-084357
+    borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/homedir.archlinux.org::20191127-084357
 
 ## One-shots
 

docs/ssh-hostkeys.txt

@@ -294,15 +294,3 @@
 256 MD5:62:eb:27:c4:a1:6f:a4:21:ed:50:6f:dd:bf:37:4e:ab root@archlinux-packer (ECDSA)
 256 MD5:9a:97:48:f7:11:b3:32:ba:fa:ab:9f:0c:41:41:da:e4 root@archlinux-packer (ED25519)
 3072 MD5:f3:11:d6:58:f9:32:d1:34:fa:4e:d9:e3:d7:c8:6b:f2 root@archlinux-packer (RSA)
-
-# vostok.archlinux.org
-1024 SHA256:FddVsY5JTplRbgQA3m3XA3bYbQ9SOQ8i/gZBlj4NUss root@vostok (DSA)
-256 SHA256:VNBI73QTTzMskhFYissdeC5ZxzsqONu0DuudJWkbxiI root@vostok (ECDSA)
-256 SHA256:NaPE8gB9f0CHLpDf8PexYFKA9OYQ0h6tcG5dIjr7NPw root@vostok (ED25519)
-2048 SHA256:Ap2YWeHDxUVub9qSxEVh1FgefWvjDGDBGjPfHIs/Pv0 root@vostok (RSA)
-
-1024 MD5:52:26:17:aa:59:ce:9b:ef:89:3f:8a:56:04:ed:29:a7 root@vostok (DSA)
-256 MD5:34:cb:51:e7:9f:e0:4c:eb:00:5c:8b:59:9d:ab:4c:60 root@vostok (ECDSA)
-256 MD5:15:11:f7:63:1b:e1:9b:52:e1:c6:87:ac:0f:1e:fb:4a root@vostok (ED25519)
-2048 MD5:b0:03:6c:cc:4d:bf:7c:e9:8b:3f:98:4f:80:ea:4e:56 root@vostok (RSA)
-

docs/ssh-known_hosts.txt

@@ -131,12 +131,11 @@ svn2gittest ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYA
 svn2gittest ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/2iOTxrCAXRfpzLfJzvwlnISD+G7wxH5aONHybVga3O1p/KQv8TiTK/+dqnH9ZW6KBPnQjtsIEmzx/ZIr/aAMeZX8OwZAl1wbKdFSpeCAhJtK7zKGr/f8qlYttHgttiGq9PJMQ8bwldaAWEllgFhr/hveKPckfACrgADTJStaWW7eeqAe28BjqHm7BpT/jO3DAeVaHrcY8CVtHsCFUXCoHSFf5ER3QOH4LTfqM7Imz0cK8i29x1H5RruM+pzIMaHthabWHERwE/V/j0Xt0gEyfXZOWK5K+3ueZjVl5yUsvpSxoYSQzpQg+rvsnN7L7qRYjA04bI6NWopFdpMKzk0NcVF64tR4wfPfaaTjYdH2HU3uO/JRtN4IZmFdI2vL1UtycM9w44humZyNUGjeMUBFbBwwPGTAkcTWjPW8HLtVYjgS6Zl5/cBjLSG90qpg7hPo0V80ybRRRJbOH14txlX4z2btxCiurHCi+nxzYBABGINsZgd9s+7AORWtvfBGo0U=
 svn2gittest ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFgvu1kXUUucO0ss+A+cDR1dsn71N77T9U/wWtcf+1w5
 
-# vostok.archlinux.org
-vostok.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK/pvIigdOhz3THBtkk9DjZitryhSHaxgVRDBPJQ+C1oGtAdUpolc9G1BctfZhcbbpqTmE9ELGPJus2vK0maARg=
-vostok.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHl3sguy1lY6HMBBNggR4t/svRAM5+NkrQhKytLKO0Oq
-vostok.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVaHKi8eFkVYuVzVxta0CbjxyssIPkQD1ufXg6xUDPI5y3/wEE9c/6g3emhwHN/NRixk70xlH5lkSfv6zH1cY5PO0uOWxBXrTDU0VtP0l4LH5gFsp9G8FSZht39erBMR/aIvmSMciC+TPoBfilwVrOb5RLYzXkft/z9QwBFGN/quCwGddQ0FSvyAUwGQctBC5NUsYCbSe+KipNPBPfdJEE0+KtM4L7NSG1sDBKQq2H8W1+BopXRh42d1clOmcVUmLqMCwSfvdd6jQwez2q9f6fJGY+iGpJkBYBeV+nHRVdXdBlohdzLn5N2+YeW3Nx9jF9pg9B/IhoSXzZ284bC7zZ
-
 # ch-s012.rsync.net
 ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3
 
 
+# u236610.your-storagebox.de
+[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
+
+

group_vars/all/vault_hetzner_storagebox.yml

+$ANSIBLE_VAULT;1.1;AES256
+30353736373466623531333732393935376435353939366632383839376531653761656631646638
+3831333465373263336232653931643162656363653039320a383736393636613231386465663430
+37313062303933653633626637623539363565316161666433656138393036343538623863386666
+3039346264393066620a396231646534303262616162346261643639323838313635366332653861
+39353239393134326130383766323832383361656431336335616138363865623865356538636139
+63363234343962333166313038646633613534653963613961656336646464393338373635663832
+62396633363932663931633532363732653766356136393137363366376134363135663864313935
+63323635666431353165396235633066313334316161396163646633366536366361643331386461
+6535

host_vars/u236610.your-storagebox.de

+---
+ansible_ssh_user: "{{ hetzner_storagebox_username }}"
+known_host: "[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs"

host_vars/vostok.archlinux.org

----
-hostname: "vostok"
-
-ipv4_address: "5.9.158.171"
-ipv4_netmask: "/32"
-ipv6_address: "2a01:4f8:190:51aa::1"
-ipv6_netmask: "/128"
-ipv4_gateway: "5.9.158.161"
-ipv6_gateway: "fe80::1"
-filesystem: ext4
-system_disks:
-  - /dev/sda
-  - /dev/sdb
-
-zabbix_agent_templates:
-  - Template OS Linux

hosts

 [hetzner]
 orion.archlinux.org
-vostok.archlinux.org
 apollo.archlinux.org
 luna.archlinux.org
 dragon.archlinux.org
@@ -10,6 +9,9 @@ gemini.archlinux.org
 [rsync_net]
 ch-s012.rsync.net
 
+[hetzner_storageboxes]
+u236610.your-storagebox.de
+
 [pia]
 jpn.mirror.pkgbuild.com
 ger.mirror.pkgbuild.com
@@ -44,8 +46,8 @@ accounts.archlinux.org
 gemini.archlinux.org
 
 [borg_hosts]
-vostok.archlinux.org
 ch-s012.rsync.net
+u236610.your-storagebox.de
 
 [public_html]
 homedir.archlinux.org

playbooks/hetzner_storagebox.yml

+---
+
+- name: setup Hetzner storagebox account
+  hosts: u236610.your-storagebox.de
+  gather_facts: False
+  roles:
+    - { role: hetzner_storagebox, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }

playbooks/tasks/sync-ssh-hostkeys.yml

 ---
 
 - name: fetch ssh hostkeys
-  hosts: all,!rsync_net
+  hosts: all,!rsync_net,!hetzner_storageboxes
   tasks:
       - name: fetch hostkey checksums
         shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done"
@@ -18,21 +18,26 @@
       - name: store hostkeys
         copy:
           dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt"
-          content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
+          content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
         delegate_to: localhost
       - name: store known_hosts
         copy:
           dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-          content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
+          content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
         delegate_to: localhost
       - name: manually append rsync.net host keys
         lineinfile:
           path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
           line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
         delegate_to: localhost
+      - name: manually append Hetzner Storageboxes host keys
+        lineinfile:
+          path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
+          line: "{% for host in query('inventory_hostnames', 'hetzner_storageboxes') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
+        delegate_to: localhost
 
 - name: upload known_hosts to all nodes
-  hosts: all,!rsync_net
+  hosts: all,!rsync_net,!hetzner_storageboxes
   tasks:
       - name: upload known_hosts
         copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"

playbooks/vostok.yml

----
-
-- name: setup vostok
-  hosts: vostok.archlinux.org
-  remote_user: root
-  roles:
-    - { role: common }
-    - { role: tools }
-    - { role: sshd }
-    - { role: unbound }
-    - { role: root_ssh }
-    - { role: borg-server, backup_dir: "/backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }

roles/borg-client/defaults/main.yml

 ---
 backup_hosts:
-  - host: "borg@vostok.archlinux.org"
-    dir: "/backup/{{ inventory_hostname }}"
+  - host: "{{ hetzner_storagebox_username }}@u236610.your-storagebox.de"
+    dir: "backup/{{ inventory_hostname }}"
     suffix: ""
   - host: "{{ rsync_net_username }}@ch-s012.rsync.net"
     dir: "backup/{{ inventory_hostname }}"

roles/borg-client/templates/borg-backup.sh.j2

@@ -72,7 +72,6 @@ borg create -v --stats -C zstd \
     -e "$backup_mountdir/var/lib/archbuild" \
     -e "$backup_mountdir/var/lib/archbuilddest" \
     -e "$backup_mountdir/var/lib/docker" \
-    -e "$backup_mountdir/srv/archive" \
     {{ item['host'] }}:{{ item['dir'] }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
 borg prune -v {{ item['host'] }}:{{ item['dir'] }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
 

roles/hetzner_storagebox/tasks/main.yml

+---
+
+# We have to set up the Hetzner Storagebox account in a weird fashion because
+# they don't even allow direct SSH.
+- name: create the root backup directory at {{ backup_dir }}
+  expect:
+    command: bash -c "echo 'mkdir {{ backup_dir }}' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+    responses:
+      (?i)password: "{{ hetzner_storagebox_password }}"
+  delegate_to: localhost
+
+- name: fetch ssh keys from each borg client machine
+  command: cat /root/.ssh/id_rsa.pub
+  register: client_ssh_keys
+  delegate_to: "{{ item }}"
+  with_items: "{{ backup_clients }}"
+  remote_user: root
+  changed_when: client_ssh_keys.changed
+
+- name: create tempfile
+  tempfile: state=file
+  register: tempfile
+  delegate_to: localhost
+
+- name: fill tempfile
+  copy: content="{{ lookup('template', 'authorized_keys.j2') }}" dest="{{ tempfile.path }}"
+  delegate_to: localhost
+
+- name: upload authorized_keys file
+  expect:
+    command: bash -c "echo -e 'mkdir .ssh \n chmod 700 .ssh \n put {{ tempfile.path }} .ssh/authorized_keys \n chmod 600 .ssh/authorized_keys' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+    responses:
+      (?i)password: "{{ hetzner_storagebox_password }}"
+  delegate_to: localhost

roles/hetzner_storagebox/templates/authorized_keys.j2

+#jinja2: lstrip_blocks: True
+# Arch DevOps keys
+{% for user in root_ssh_keys | sort -%}
+	{{ lookup('file', '../pubkeys/' + user) }}
+{% endfor %}
+
+# Client machines keys
+{% for client_key in client_ssh_keys.results %}
+command="/usr/bin/borg serve --restrict-to-path {{ backup_dir }}/{{ client_key['item'] }}",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc {{ client_key['stdout'] }}
+{% endfor %}