Merge branch 'offiste-backup'

docs/ssh-known_hosts.txt

@@ -128,3 +128,5 @@ vostok.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbml
 vostok.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHl3sguy1lY6HMBBNggR4t/svRAM5+NkrQhKytLKO0Oq
 vostok.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVaHKi8eFkVYuVzVxta0CbjxyssIPkQD1ufXg6xUDPI5y3/wEE9c/6g3emhwHN/NRixk70xlH5lkSfv6zH1cY5PO0uOWxBXrTDU0VtP0l4LH5gFsp9G8FSZht39erBMR/aIvmSMciC+TPoBfilwVrOb5RLYzXkft/z9QwBFGN/quCwGddQ0FSvyAUwGQctBC5NUsYCbSe+KipNPBPfdJEE0+KtM4L7NSG1sDBKQq2H8W1+BopXRh42d1clOmcVUmLqMCwSfvdd6jQwez2q9f6fJGY+iGpJkBYBeV+nHRVdXdBlohdzLn5N2+YeW3Nx9jF9pg9B/IhoSXzZ284bC7zZ
 
+# ch-s012.rsync.net
+ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3

group_vars/all/vault_rsync.net.yml

+$ANSIBLE_VAULT;1.1;AES256
+31363037363934306662343830323131313361383561656330316537366362656133643237666339
+3932386437386338336663613461653463326461336666350a346536333865333338376638643834
+30306264356232333761333361313239383931663632396534346333643131326364303066363464
+6130363230346562310a626439396536656231643265316438623835656363333034323038616234
+39383731303637346233653332333965623961343163363935633532623235316633373566336464
+38363433663862636233376164313566613732323138333135663530333866333732666539376563
+663861353832316436633761323334313362

host_vars/ch-s012.rsync.net

+---
+ansible_ssh_user: "{{ rsync_net_username }}"
+known_host: "ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3"

host_vars/quassel.archlinux.org

 ---
 filesystem: btrfs
+
+postgres_backup_dir: "/var/lib/postgres/backup"
 zabbix_agent_templates:
   - Template OS Linux
   - Template App Borg Backup

hosts

@@ -5,6 +5,9 @@ apollo.archlinux.org
 luna.archlinux.org
 dragon.archlinux.org
 
+[rsync_net]
+ch-s012.rsync.net
+
 [pia]
 ind.mirror.pkgbuild.com
 jpn.mirror.pkgbuild.com
@@ -40,6 +43,11 @@ phrik.archlinux.org
 bbs.archlinux.org
 homedir.archlinux.org
 bugs.archlinux.org
+aur-dev.archlinux.org
+
+[borg_hosts]
+vostok.archlinux.org
+ch-s012.rsync.net
 
 [public_html]
 homedir.archlinux.org

playbooks/matrix.archlinux.org.yml

@@ -10,7 +10,7 @@
     - { role: sshd }
     - { role: unbound }
     - { role: root_ssh }
-    - { role: borg-client }
+    - { role: borg-client, tags: ["borg"] }
     - { role: certbot }
     - { role: nginx }
     - role: postgres

playbooks/quassel.archlinux.org.yml

@@ -10,6 +10,7 @@
     - { role: sshd }
     - { role: unbound }
     - { role: root_ssh }
+    - { role: borg-client, tags: ["borg"] }
     - { role: certbot }
     - role: postgres
       postgres_max_connections: 100

playbooks/rsync.net.yml

+---
+
+- name: setup rsync.net account
+  hosts: ch-s012.rsync.net
+  gather_facts: False
+  roles:
+    - { role: rsync_net, backup_dir: "backup", backup_clients: "{{groups['borg_clients']}}", tags: ["borg"] }

playbooks/tasks/sync-ssh-hostkeys.yml

 ---
 
 - name: fetch ssh hostkeys
-  hosts: all
+  hosts: all,!rsync_net
   tasks:
       - name: fetch hostkey checksums
         shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done"
@@ -11,20 +11,27 @@
         register: known_hosts
 
 - name: store hostkeys
-  hosts: 127.0.0.1
+  hosts: localhost
   tasks:
       - name: store hostkeys
-        copy:
-            dest: "{{playbook_dir}}/../../docs/ssh-hostkeys.txt"
-            content: "{% for host in groups['all'] | sort %}# {{host}}\n{{hostvars[host].ssh_hostkeys.stdout}}\n\n{% endfor %}"
+        local_action:
+            module: copy
+            dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt"
+            content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
       - name: store known_hosts
-        copy:
-            dest: "{{playbook_dir}}/../../docs/ssh-known_hosts.txt"
-            content: "{% for host in groups['all'] | sort %}# {{host}}\n{{hostvars[host].known_hosts.stdout}}\n\n{% endfor %}"
+        local_action:
+            module: copy
+            dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
+            content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
+      - name: manually append rsync.net host keys
+        local_action:
+            module: lineinfile
+            path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
+            line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
 
 - name: upload known_hosts to all nodes
-  hosts: all
+  hosts: all,!rsync_net
   tasks:
       - name: upload known_hosts
-        copy: dest=/etc/ssh/ssh_known_hosts src="{{playbook_dir}}/../../docs/ssh-known_hosts.txt"
+        copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
         tags: ['upload-known-hosts']

roles/borg-client/defaults/main.yml

 ---
-backup_host: "borg@vostok.archlinux.org"
-backup_dir: "/backup/{{inventory_hostname}}"
+backup_hosts:
+  - host: "borg@vostok.archlinux.org"
+    dir: "/backup/{{ inventory_hostname }}"
+    suffix: ""
+  - host: "{{ rsync_net_username }}@ch-s012.rsync.net"
+    dir: "backup/{{ inventory_hostname }}"
+    suffix: "-offsite"

roles/borg-client/tasks/main.yml

@@ -7,22 +7,26 @@
   template: src=borg-restore.cfg.j2 dest=/etc/borg-restore.cfg owner=root group=root mode=0644
 
 - name: check if borg repository already exists
-  command: borg list {{ backup_host }}:{{ backup_dir }}
+  command: borg list {{ item['host'] }}:{{ item['dir'] }}
   register: borg_list
   ignore_errors: True
+  loop: "{{ backup_hosts }}"
 
 - name: init borg repository
-  command: borg init -e keyfile {{ backup_host }}:{{ backup_dir }}
+  command: borg init -e keyfile {{ item['host'] }}:{{ item['dir'] }}
   when: borg_list is failed
   environment:
     BORG_PASSPHRASE: ""
   ignore_errors: True # This can sometimes fail if a backup is in progress :/
+  loop: "{{ backup_hosts }}"
 
-- name: install scripts
-  template: src={{item}}.j2 dest=/usr/local/bin/{{item}} owner=root group=root mode=0755
-  with_items:
-    - borg-backup.sh
-    - borg
+- name: install convenience scripts
+  template: src=borg.j2 dest=/usr/local/bin/borg{{ item['suffix'] }} owner=root group=root mode=0755
+  loop: "{{ backup_hosts }}"
+
+- name: install borg backup scripts
+  template: src=borg-backup.sh.j2 dest=/usr/local/bin/borg-backup{{ item['suffix'] }}.sh owner=root group=root mode=0755
+  loop: "{{ backup_hosts }}"
 
 - name: install postgres backup script
   template: src=backup-postgres.sh.j2 dest=/usr/local/bin/backup-postgres.sh owner=root group=root mode=0755
@@ -42,22 +46,22 @@
   when: mysql_backup_dir is defined
 
 - name: install mysql backup config
-  template: src=backup-my.cnf.j2 dest={{mysql_backup_defaults}}
+  template: src=backup-my.cnf.j2 dest={{ mysql_backup_defaults }}
   when: mysql_backup_defaults is defined
 
 - name: create mysql backup directory
-  file: path={{mysql_backup_dir}} state=directory owner=root group=root
+  file: path={{ mysql_backup_dir }} state=directory owner=root group=root
   when: mysql_backup_dir is defined
 
 - name: remove xtrabackup, mariadb ships its own fork
   pacman: name=xtrabackup state=absent
   when: mysql_backup_dir is defined
 
-- name: install systemd timers for backup
-  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
+- name: install systemd timer and service for backup
+  template: src={{ item }}.j2 dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
   with_items:
     - borg-backup.timer
     - borg-backup.service
 
 - name: activate systemd timers for backup
-  service: name=borg-backup.timer enabled=yes state=started
+  systemd: name=borg-backup.timer enabled=yes state=started daemon-reload=yes

roles/borg-client/files/borg-backup.service → roles/borg-client/templates/borg-backup.service.j2

@@ -3,4 +3,6 @@ Description=Borg backup
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/borg-backup.sh
+{% for backup in backup_hosts %}
+ExecStart=/usr/local/bin/borg-backup{{ backup['suffix'] }}.sh
+{% endfor %}

roles/borg-client/templates/borg-backup.sh.j2

@@ -60,8 +60,8 @@ borg create -v --stats -C lz4 \
     -e "$backup_mountdir/var/lib/archbuild" \
     -e "$backup_mountdir/var/lib/archbuilddest" \
     -e "$backup_mountdir/srv/archive" \
-    {{ backup_host }}:{{ backup_dir }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
-borg prune -v {{ backup_host }}:{{ backup_dir }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
+    {{ item['host'] }}:{{ item['dir'] }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
+borg prune -v {{ item['host'] }}:{{ item['dir'] }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
 
 if is_btrfs "$src"; then
     umount -R "$backup_mountdir"

roles/borg-client/templates/borg.j2

 #!/bin/bash
 
-BORG_REPO="{{ backup_host }}:{{ backup_dir }}" exec /usr/bin/borg "$@"
+BORG_REPO="{{ item['host'] }}:{{ item['dir'] }}" exec /usr/bin/borg "$@"

roles/borg-server/tasks/main.yml

@@ -25,7 +25,7 @@
     mode: 0700
   with_items: "{{ backup_clients }}"
 
-- name: fetch ssh keys
+- name: fetch ssh keys from each borg client machine
   command: cat /root/.ssh/id_rsa.pub
   register: ssh_keys
   delegate_to: "{{ item }}"

roles/rsync_net/tasks/main.yml

+---
+
+# We have to set up the rsync.net account in a weird fashion because
+# they don't support ansible directly (no Python and such).
+- name: create the root backup directory at {{ backup_dir }}
+  raw: mkdir -p {{ backup_dir }}
+
+- name: fetch ssh keys from each borg client machine
+  command: cat /root/.ssh/id_rsa.pub
+  register: client_ssh_keys
+  delegate_to: "{{ item }}"
+  with_items: "{{ backup_clients }}"
+  remote_user: root
+
+- local_action: tempfile state=file
+  register: tempfile
+
+- local_action: copy content="{{ lookup('template', 'authorized_keys.j2')}}" dest="{{ tempfile.path }}"
+
+- name: upload authorized_keys file
+  local_action: command scp "{{ tempfile.path }}" "{{ rsync_net_username }}@{{ inventory_hostname }}":.ssh/authorized_keys

roles/rsync_net/templates/authorized_keys.j2

+#jinja2: lstrip_blocks: True
+# Arch DevOps keys
+{% for user in root_ssh_keys | sort -%}
+	{{ lookup('file', '../pubkeys/' + user) }}
+{% endfor %}
+
+# Client machines keys
+{% for client_key in client_ssh_keys.results %}
+command="/usr/bin/borg serve --restrict-to-path {{ backup_dir }}/{{ client_key['item'] }}",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc {{ client_key['stdout'] }}
+{% endfor %}