Merge branch 'inventory-remove-borg-hosts' into 'master'

Remove our two borg hosts from the inventory

See merge request !540

docs/otp.md

@@ -56,7 +56,7 @@ Run
 
     pass otp insert -i rsync.net -a archlinux Rsync.net/archlinux-master-token -s
 
-When asked for a secret, provide the `2FA token seed` from `group_vars/all/vault_rsync.net.yml`.
+When asked for a secret, provide the `2FA token seed` from `host_vars/localhost/vault_rsync.net.yml`.
 You can then run
 
     pass otp code Rsync.net/archlinux-master-token

docs/ssh-known_hosts.txt

+# u236610.your-storagebox.de
+[u236610.your-storagebox.de]:23 ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAGK0po6usux4Qv2d8zKZN1dDvbWjxKkGsx7XwFdSUCnF19Q8psHEUWR7C/LtSQ5crU/g+tQVRBtSgoUcE8T+FWp5wBxKvWG2X9gD+s9/4zRmDeSJR77W6gSA/+hpOZoSE+4KgNdnbYSNtbZH/dN74EG7GLb/gcIpbUUzPNXpfKl7mQitw==
+[u236610.your-storagebox.de]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
+[u236610.your-storagebox.de]:23 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==
+
+# zh1905.rsync.net
+zh1905.rsync.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLR2uz+YLn2KiQK0Luu8rhfWS6LHgUfGAWB1j8rM2MKn4KZ2/LhIX1CYkPKMTPxHr6mzayeL1T1hyJIylxXv0BY=
+zh1905.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJtclizeBy1Uo3D86HpgD3LONGVH0CJ0NT+YfZlldAJd
+zh1905.rsync.net ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPgHxQyaDaVxUefoUJZO/lITh0Gp0sqbP7HejQcCfZi7gAcuM6/IAuUXLHFImefCHh52x6T/cHxgL1qz26GKgdxykl06WRXlRIuE45QFSy/cd9JKr6l58fKq30ApmXRsCNwFrMlFPoEpCTqxzddZ9cLXs1Yt9dRxvFlQVEuAzw7ayvt8DE6RP9/CHYVp54wbbvUToECGwu70sxY1vFg51K+vNpvJ3J0t5j3s4c1Wls4BrIwqi2U8kqCq9Nj2CUIQqjM+93CSqEacR3qOGvG/6QMzd733wzpJ/iZee+lcyTYzA0YNMosnaF01hrv7NMwtZ6xRFLlJZtMZ7JpfySrOBr
+
+# BEGIN ANSIBLE MANAGED BLOCK
+
 # accounts.archlinux.org
 accounts.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFi9o8CPcvujoELaKVRqMh92KiMJrBvvoTpf3FlTNAfAo641IdkGqzqCFyJA1FeFXLYOS+Zeehi1AMe1iI/b1js=
 accounts.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBu3+qlfqd8FwqodNzem7cCVcNA5RQpidYHkDRPdsZzq
@@ -183,9 +195,4 @@ wiki.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzd
 wiki.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILFxxvRi7khrt6mUQGiXX35O1MBrrDeEmvaAnWo9ql/7
 wiki.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZQmj2D2B66bBHzze7+LEITju0rmtJT2rYsV/1GGpEPg6q6GAkUnIgRpyYxRn+UKRO9akDFXLv7W02h86cfbxOjeHufGMy/Y7NCPl2OSP9VauavJ3c3v/n80nmntU/Ji/U/p/roP0z+/OPgdWymFm0n33cl+XhmNOUumYQ7Y3z7EzrvCFZo2gt1EYChXb5Pd32rkd9tiwr3O0/M7TEiUxODzoD1dum+TJafUttC20V/4Sj8HztPx2BzhRugXfeEDbVlYvMXMMYgxbbhXLZyuE/dCaYpRvuekouAob7voIRSUaNqFXBLcGgo0udRI0mnKLgTb5bMprrRZ1zXIBU77H3gWyfHqe2I20arhXivtsHLEOZi0hc2/ni15WqkNS8G23n/+hJ+H16bjVb6t+8opErY4mL8T+F6OkxmNo8d0ztwUdxHEa+fvNPQ8UO6W4CN6kNVB9JE4f8j9FeHQq8rtlzo0wjUof4D7PhDn2WYA1l9RDiuRUxlGS4waStmttM3dE=
 
-# zh1905.rsync.net
-zh1905.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJtclizeBy1Uo3D86HpgD3LONGVH0CJ0NT+YfZlldAJd
-
-# u236610.your-storagebox.de
-[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
-
+# END ANSIBLE MANAGED BLOCK

host_vars/u236610.your-storagebox.de

----
-ansible_ssh_user: "{{ hetzner_storagebox_username }}"
-known_host: "[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs"

host_vars/zh1905.rsync.net

----
-ansible_ssh_user: "{{ rsync_net_username }}"
-known_host: "zh1905.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJtclizeBy1Uo3D86HpgD3LONGVH0CJ0NT+YfZlldAJd"

hosts

@@ -2,12 +2,6 @@
 secure-runner1.archlinux.org
 gemini.archlinux.org
 
-[rsync_net]
-zh1905.rsync.net
-
-[hetzner_storageboxes]
-u236610.your-storagebox.de
-
 [packet_net]
 runner2.archlinux.org
 repro1.pkgbuild.com
@@ -47,10 +41,6 @@ md.archlinux.org
 lists.archlinux.org
 gluebuddy.archlinux.org
 
-[borg_hosts]
-zh1905.rsync.net
-u236610.your-storagebox.de
-
 [public_html]
 homedir.archlinux.org
 

playbooks/all-hosts-basic.yml

 ---
 
 - name: basic setup for all hosts
-  hosts: all,!hetzner_storageboxes,!rsync_net
+  hosts: all
   remote_user: root
   roles:
     - { role: common }

playbooks/hetzner_storagebox.yml

 ---
 
 - name: setup Hetzner storagebox account
-  hosts: u236610.your-storagebox.de
+  hosts: localhost
   gather_facts: false
   roles:
-    - { role: hetzner_storagebox, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }
+    - role: hetzner_storagebox
+      backup_clients: "{{ groups['borg_clients'] }}"
+      backup_dir: backup
+      storagebox_id: "{{ hetzner_storagebox_id }}"
+      storagebox_hostname: "{{ hetzner_storagebox_username }}.your-storagebox.de"
+      storagebox_username: "{{ hetzner_storagebox_username }}"
+      storagebox_password: "{{ hetzner_storagebox_password }}"
+      tags: ["borg"]

playbooks/rsync.net.yml

 ---
 
 - name: setup rsync.net account
-  hosts: zh1905.rsync.net
+  hosts: localhost
   gather_facts: false
   roles:
-    - { role: rsync_net, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }
+    - role: rsync_net
+      backup_clients: "{{ groups['borg_clients'] }}"
+      backup_dir: backup
+      tags: ["borg"]

playbooks/tasks/sync-ssh-hostkeys.yml

 ---
 
 - name: fetch ssh hostkeys
-  hosts: all,!rsync_net,!hetzner_storageboxes
+  hosts: all
+  gather_facts: false
   tasks:
     - name: fetch hostkey checksums
-      shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done"
+      shell: |
+        for type in sha256 md5; do
+          for file in /etc/ssh/ssh_host_*.pub; do
+            ssh-keygen -l -f $file -E $type
+          done
+          echo
+        done
       register: ssh_hostkeys
       changed_when: ssh_hostkeys | length > 0
+
     - name: fetch known_hosts
-      shell: "set -o pipefail && ssh-keyscan 127.0.0.1 2>/dev/null | sed 's#^127.0.0.1#{{ inventory_hostname }}#' | sort"
+      shell: |
+        set -eo pipefail
+        ssh-keyscan 127.0.0.1 2>/dev/null \
+        | sed 's#^127.0.0.1#{{ inventory_hostname }}#' \
+        | sort
       environment:
         LC_COLLATE: C  # to ensure reproducible ordering
       args:
-        executable: /bin/bash  # required for repro3.pkgbuild.com which is ubuntu and has dash as default shell
+        executable: /bin/bash
       register: known_hosts
       changed_when: known_hosts | length > 0
 
@@ -22,28 +34,27 @@
     - name: store hostkeys
       copy:
         dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt"
-        content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
+        content: |
+          {% for host in query('inventory_hostnames', 'all') | sort %}
+          # {{ host }}
+          {{ hostvars[host].ssh_hostkeys.stdout }}
+
+          {% endfor %}
         mode: preserve
-      delegate_to: localhost
+
     - name: store known_hosts
-      copy:
-        dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
-        mode: preserve
-      delegate_to: localhost
-    - name: manually append rsync.net host keys
-      lineinfile:
-        path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n{% endfor %}"
-      delegate_to: localhost
-    - name: manually append Hetzner Storageboxes host keys
-      lineinfile:
+      blockinfile:
         path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        line: "{% for host in query('inventory_hostnames', 'hetzner_storageboxes') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n{% endfor %}"
-      delegate_to: localhost
+        block: |
+
+          {% for host in query('inventory_hostnames', 'all') | sort %}
+          # {{ host }}
+          {{ hostvars[host].known_hosts.stdout }}
+
+          {% endfor %}
 
 - name: upload known_hosts to all nodes
-  hosts: all,!rsync_net,!hetzner_storageboxes
+  hosts: all
   tasks:
     - name: upload known_hosts
       copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt" owner=root group=root mode=0644

playbooks/tasks/upgrade-servers.yml

 ---
 
 - name: upgrade and reboot all hetzner servers
-  hosts: all,!kape_servers,!packet_net,!rsync_net,!hetzner_storageboxes
+  hosts: all,!kape_servers,!packet_net
   max_fail_percentage: 0
   serial: 20%
   gather_facts: false

roles/borg_client/defaults/main.yml

@@ -4,7 +4,7 @@ backup_hosts:
     dir: "~/repo"
     suffix: ""
     borg_cmd: "borg"
-  - host: "ssh://{{ rsync_net_username }}@zh1905.rsync.net:22"
+  - host: "ssh://zh1905@zh1905.rsync.net:22"
     dir: "~/backup/{{ inventory_hostname }}"
     suffix: "-offsite"
     borg_cmd: "borg --remote-path=borg1"

roles/hetzner_storagebox/tasks/main.yml

 ---
 
-# We have to set up the Hetzner Storagebox account in a weird fashion because
-# they don't even allow direct SSH.
+# This role runs on localhost; use commands like sftp to upload configuration
+
 - name: create the root backup directory at {{ backup_dir }}
   expect:
-    command: bash -c "echo 'mkdir {{ backup_dir }}' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+    command: bash -c "echo 'mkdir {{ backup_dir }}' | sftp -P 23 {{ storagebox_username }}@{{ storagebox_hostname }}"
     responses:
-      (?i)password: "{{ hetzner_storagebox_password }}"
-  delegate_to: localhost
+      (?i)password: "{{ storagebox_password }}"
 
 - name: create a home directory for each sub-account
   expect:
-    command: bash -c "echo 'mkdir {{ backup_dir }}/{{ item }}' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+    command: |
+      bash -c 'sftp -P 23 {{ storagebox_username }}@{{ storagebox_hostname }} <<EOF
+        {% for client in backup_clients %}
+          mkdir {{ backup_dir }}/{{ client }}
+        {% endfor %}
+      EOF'
     responses:
-      (?i)password: "{{ hetzner_storagebox_password }}"
-  delegate_to: localhost
-  loop: "{{ backup_clients }}"
+      (?i)password: "{{ storagebox_password }}"
 
 - name: fetch ssh keys from each borg client machine
   command: cat /root/.ssh/id_rsa.pub
@@ -23,26 +25,28 @@
   register: client_ssh_keys
   delegate_to: "{{ item }}"
   with_items: "{{ backup_clients }}"
-  remote_user: root
   changed_when: client_ssh_keys.changed
 
 - name: create tempfile
   tempfile: state=file
   check_mode: false
   register: tempfile
-  delegate_to: localhost
 
 - name: fill tempfile
   copy: content="{{ lookup('template', 'authorized_keys.j2') }}" dest="{{ tempfile.path }}" mode=preserve
-  delegate_to: localhost
   no_log: true
 
 - name: upload authorized_keys for Arch DevOps
   expect:
-    command: bash -c "echo -e 'mkdir .ssh \n chmod 700 .ssh \n put {{ tempfile.path }} .ssh/authorized_keys \n chmod 600 .ssh/authorized_keys' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+    command: |
+      bash -c 'sftp -P 23 {{ storagebox_username }}@{{ storagebox_hostname }} <<EOF
+        mkdir .ssh
+        chmod 700 .ssh
+        put {{ tempfile.path }} .ssh/authorized_keys
+        chmod 600 .ssh/authorized_keys
+      EOF'
     responses:
-      (?i)password: "{{ hetzner_storagebox_password }}"
-  delegate_to: localhost
+      (?i)password: "{{ storagebox_password }}"
 
 - name: upload authorized_keys for each backup client
   include_tasks: upload_client_authorized_keys.yml
@@ -52,10 +56,9 @@
 
 - name: retrieve sub-account information
   uri:
-    url: https://robot-ws.your-server.de/storagebox/{{ hetzner_storagebox_id }}/subaccount
+    url: https://robot-ws.your-server.de/storagebox/{{ storagebox_id }}/subaccount
     user: "{{ hetzner_webservice_username }}"
     password: "{{ hetzner_webservice_password }}"
-  delegate_to: localhost
   check_mode: false
   register: subaccounts_raw
   no_log: true
@@ -67,7 +70,7 @@
 - name: create missing sub-accounts
   uri:
     timeout: 60
-    url: https://robot-ws.your-server.de/storagebox/{{ hetzner_storagebox_id }}/subaccount
+    url: https://robot-ws.your-server.de/storagebox/{{ storagebox_id }}/subaccount
     user: "{{ hetzner_webservice_username }}"
     password: "{{ hetzner_webservice_password }}"
     method: POST
@@ -76,7 +79,6 @@
       homedirectory: "{{ backup_dir }}/{{ item }}"
       comment: "{{ item }}"
       ssh: "true"
-  delegate_to: localhost
   loop: "{{ backup_clients | difference(subaccounts | json_query('[].comment')) }}"
   register: new_subaccounts_raw
   no_log: true
@@ -101,7 +103,7 @@
     create: true
     mode: 0600
     block: |
-      Host {{ inventory_hostname }}
+      Host {{ storagebox_hostname }}
         User {{ backup_client_usernames[item] }}
     marker: '# {mark} HETZNER STORAGE BOX BACKUP CLIENT CONFIG'
   delegate_to: "{{ item }}"

roles/hetzner_storagebox/tasks/upload_client_authorized_keys.yml

@@ -2,12 +2,16 @@
 
 - name: fill tempfile
   copy: content="{{ lookup('template', 'authorized_keys_client.j2') }}" dest="{{ tempfile.path }}" mode=preserve
-  delegate_to: localhost
   no_log: true
 
 - name: upload authorized_keys file to {{ backup_dir }}/{{ item.item }}
   expect:
-    command: bash -c "echo -e 'mkdir {{ backup_dir }}/{{ item.item }}/.ssh \n chmod 700 {{ backup_dir }}/{{ item.item }}/.ssh \n put {{ tempfile.path }} {{ backup_dir }}/{{ item.item }}/.ssh/authorized_keys \n chmod 600 {{ backup_dir }}/{{ item.item }}/.ssh/authorized_keys' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
+    command: |
+      bash -c 'sftp {{ storagebox_username }}@{{ storagebox_hostname }} <<EOF
+        mkdir {{ backup_dir }}/{{ item.item }}/.ssh
+        chmod 700 {{ backup_dir }}/{{ item.item }}/.ssh
+        put {{ tempfile.path }} {{ backup_dir }}/{{ item.item }}/.ssh/authorized_keys
+        chmod 600 {{ backup_dir }}/{{ item.item }}/.ssh/authorized_keys'
+      EOF'
     responses:
-      (?i)password: "{{ hetzner_storagebox_password }}"
-  delegate_to: localhost
+      (?i)password: "{{ storagebox_password }}"

roles/rsync_net/tasks/main.yml

 ---
 
-# We have to set up the rsync.net account in a weird fashion because
-# they don't support ansible directly (no Python and such).
+# This role runs on localhost; use commands like sftp to upload configuration
+
 - name: create the root backup directory at {{ backup_dir }}
-  raw: mkdir -p {{ backup_dir }}
-  tags:
-    - skip_ansible_lint
+  expect:
+    command: bash -c "echo 'mkdir {{ backup_dir }}' | sftp {{ rsync_net_username }}@{{ rsync_net_username }}.rsync.net"
+    responses:
+      (?i)password: "{{ rsync_net_password }}"
 
 - name: fetch ssh keys from each borg client machine
   command: cat /root/.ssh/id_rsa.pub
   register: client_ssh_keys
   delegate_to: "{{ item }}"
   with_items: "{{ backup_clients }}"
-  remote_user: root
   changed_when: client_ssh_keys.changed
 
 - name: create tempfile
   tempfile: state=file
   register: tempfile
-  delegate_to: localhost
 
 - name: fill tempfile
   copy: content="{{ lookup('template', 'authorized_keys.j2') }}" dest="{{ tempfile.path }}" mode=0644  # noqa 208
-  delegate_to: localhost
 
 - name: upload authorized_keys file
-  command: scp "{{ tempfile.path }}" "{{ rsync_net_username }}@{{ inventory_hostname }}":.ssh/authorized_keys
-  delegate_to: localhost
-  register: scp
-  changed_when: "scp.rc == 0"
+  expect:
+    command: |
+      bash -c 'sftp {{ rsync_net_username }}@{{ rsync_net_username }}.rsync.net <<EOF
+        mkdir .ssh
+        chmod 700 .ssh
+        put {{ tempfile.path }} .ssh/authorized_keys
+        chmod 600 .ssh/authorized_keys
+      EOF'
+    responses:
+      (?i)password: "{{ rsync_net_password }}"