Commit 8fda08ae authored by Sven-Hendrik Haase's avatar Sven-Hendrik Haase
Browse files

Add offsite backup with rsync.net

parent 95301ebc
# accounts.archlinux.org
1024 SHA256:KiPj4U+TqEEooOVD3zzXvYWfEOc3VNyiRD7tJ0nSKIs root@archlinux-packer (DSA)
256 SHA256:uG2Rr2pgYZf3dNeeTo8rC5JY4vE5jKpjjHI6mtSbQZ4 root@archlinux-packer (ECDSA)
256 SHA256:UdPV6/G7VnD7YO05mo18pAaCTL0s5Wna8LtbC86B8WI root@archlinux-packer (ED25519)
3072 SHA256:eO8MhAgZa4piCSugrneqAt7O2tJ6AkaxwWzqEgNivwE root@archlinux-packer (RSA)
1024 SHA256:wKsZHC2Gc1I2NqloqGGZG9VJi8AD6QvMRzTN8UxRWZg root@archlinux-packer (DSA)
256 SHA256:agqAqFncYFxrKtHeT4GOzKOdBQ72vutVoi44B+shAPQ root@archlinux-packer (ECDSA)
256 SHA256:nId6jHW1cWQE4JZolGDGoy8miQV0I9NcmsXwC/WrfeA root@archlinux-packer (ED25519)
3072 SHA256:H+QddiCCMbVlYeTK0plKwLNw4djtcVfAprHi+S6Soqs root@archlinux-packer (RSA)
1024 MD5:06:6e:39:5a:82:d1:ac:57:1c:70:ff:da:2f:61:fb:e1 root@archlinux-packer (DSA)
256 MD5:d0:72:51:0e:9f:e5:c1:9d:bd:0f:22:56:2f:cd:a9:0b root@archlinux-packer (ECDSA)
256 MD5:6e:8d:53:3e:e8:52:c4:ed:23:0b:b1:74:66:f1:ad:6a root@archlinux-packer (ED25519)
3072 MD5:b0:9f:69:c1:b7:e8:91:28:a2:be:7f:a5:0d:90:f9:c0 root@archlinux-packer (RSA)
1024 MD5:0a:0b:40:08:21:57:1a:60:ea:9c:d2:6e:26:fe:f1:5a root@archlinux-packer (DSA)
256 MD5:86:be:86:50:8c:11:43:de:07:89:f3:c5:c2:ae:e4:3d root@archlinux-packer (ECDSA)
256 MD5:07:73:26:f4:27:f2:c1:90:8d:13:f5:d1:95:51:83:50 root@archlinux-packer (ED25519)
3072 MD5:1b:10:0b:4c:b2:64:0c:fb:45:df:58:f6:cc:f9:fe:b6 root@archlinux-packer (RSA)
# apollo.archlinux.org
1024 SHA256:WArxFzvhf5HknYxil2EQSHHRirM2cyjqbtLvhbQAYC8 root@apollo (DSA)
......@@ -20,6 +20,17 @@
256 MD5:a7:84:8b:95:4f:53:ac:b6:9d:24:79:79:fc:c7:bf:1f root@apollo (ED25519)
2048 MD5:77:b0:17:18:57:74:38:91:47:31:43:04:47:e9:9e:30 root@apollo (RSA)
# aur-dev.archlinux.org
1024 SHA256:UPSaOwfVUU5XnBARVikOGxksKlZx48aUyIPjZE9zpAc root@archlinux-packer (DSA)
256 SHA256:b1/sK6szU73jV3XdtoWFgXcSN3FP4QQEBPdw+g0KMro root@archlinux-packer (ECDSA)
256 SHA256:eSsnneEKh60EYqc08//of2SrdbL3tg1y07XSNF25ZwA root@archlinux-packer (ED25519)
3072 SHA256:4yKdHD71M5yxsu2LiLKaOYfFzoFStwgF+HP4stk0/nI root@archlinux-packer (RSA)
1024 MD5:12:c4:cb:12:cf:9e:4d:13:f6:9b:9c:8d:a0:9f:ef:2c root@archlinux-packer (DSA)
256 MD5:5b:c6:00:09:3b:e4:ec:f3:e6:87:a8:0d:ce:69:c7:13 root@archlinux-packer (ECDSA)
256 MD5:fd:d3:a4:31:64:9b:4b:51:b8:89:dc:05:76:a9:49:84 root@archlinux-packer (ED25519)
3072 MD5:0a:58:e7:1e:a1:05:22:8f:c7:f4:2c:c4:ea:91:78:8c root@archlinux-packer (RSA)
# bbs.archlinux.org
1024 SHA256:8D8LNOrQ4wByBgNJ3n19B7SH7OF1CONh1rU5wbEd53w root@archlinux-packer (DSA)
256 SHA256:N35ylQxDBW9lohn+NBxcG8aW8Qfz2+nMYN+mnaojzgg root@archlinux-packer (ECDSA)
......@@ -31,6 +42,17 @@
256 MD5:46:23:93:5c:db:68:8e:a3:0a:eb:cb:18:13:94:73:dc root@archlinux-packer (ED25519)
3072 MD5:13:8f:2f:f6:c6:90:10:6b:ee:e8:66:e5:60:ef:d8:f8 root@archlinux-packer (RSA)
# bugs.archlinux.org
1024 SHA256:c8CCzrXjPnUEi0d0B2yLzMWK935TyjzoCOdcP12BwEM root@archlinux-packer (DSA)
256 SHA256:z9CfWniDILraPxPn4e8Sao/vaAseI29KyXEhGU3sNRk root@archlinux-packer (ECDSA)
256 SHA256:ZL2RVyqM9FsvoSNqyXg9J7keN4QxRMD6+m6i4dDYkao root@archlinux-packer (ED25519)
3072 SHA256:u1iIRQp0fVyM2pgTTca/nxG/iO1QxbfR2nGhnIkohfg root@archlinux-packer (RSA)
1024 MD5:cf:10:49:2f:d2:35:99:35:59:8f:e2:54:b3:05:cb:a7 root@archlinux-packer (DSA)
256 MD5:d1:94:76:51:bb:7b:88:41:03:6d:12:63:a5:03:5f:58 root@archlinux-packer (ECDSA)
256 MD5:d6:d3:a9:2e:c1:7d:69:c1:9a:21:c9:6f:30:53:e6:74 root@archlinux-packer (ED25519)
3072 MD5:06:6d:1f:87:6b:fb:60:b3:a8:c7:64:37:15:b5:b5:6c root@archlinux-packer (RSA)
# ciprototype.archlinux.org
1024 SHA256:f0cmzd2NU98fsVEc/WBXK7cDw+KCBd3V3TddYOZAnPw root@archlinux-packer (DSA)
256 SHA256:CHg9VczcKVRyINPb6jEp8HYqW8DeNRrYUU76A1ym5TE root@archlinux-packer (ECDSA)
......@@ -75,6 +97,17 @@
256 MD5:3b:a2:48:8c:33:ac:8b:74:43:d4:4f:f1:fc:78:be:20 root@archlinux-packer (ED25519)
3072 MD5:a3:ce:cf:2c:59:05:5d:eb:a0:1f:fb:30:81:cc:07:d8 root@archlinux-packer (RSA)
# homedir.archlinux.org
1024 SHA256:3iibcYCSLNa+WeOM62/9p3MxJFvmm2qJVnLr2vNljV0 root@archlinux-packer (DSA)
256 SHA256:j7PC/+H3R4buq32hZZjsoLumNlBJ+Qiw7IZSL7yjn+k root@archlinux-packer (ECDSA)
256 SHA256:RPB4mVNaSndrR8PydmsAKJvLpIFm+s4w4MEHYDPOqBM root@archlinux-packer (ED25519)
3072 SHA256:sTcibF9dz2CgFfY6a0pUBPZd6G9P9zFtoWbMuDNdJrk root@archlinux-packer (RSA)
1024 MD5:0b:01:ff:bb:1a:04:ca:05:eb:98:0e:30:a7:35:71:85 root@archlinux-packer (DSA)
256 MD5:f9:bc:d6:b3:77:07:d0:80:d6:47:0f:b4:94:fe:35:e4 root@archlinux-packer (ECDSA)
256 MD5:e2:34:b1:dc:24:00:45:08:4a:62:48:17:b2:69:23:2d root@archlinux-packer (ED25519)
3072 MD5:50:c8:93:43:05:d5:73:a4:84:b1:07:66:a7:20:a5:79 root@archlinux-packer (RSA)
# ind.mirror.pkgbuild.com
1024 SHA256:nfyohw9fqhP3SRTOpsZvjVH7OP6GkqNWv+vPIa7Xr54 root@archlinux (DSA)
256 SHA256:d5s5vHVfEh2U0oCFsvCxgidvX4Fm2BdX4EJvwcsRLqk root@archlinux (ECDSA)
......@@ -108,6 +141,17 @@
256 MD5:1d:92:08:da:8e:a1:fb:1c:c5:65:00:c8:15:a4:87:32 root@alderaan.archlinux.org (ED25519)
2048 MD5:c4:7f:00:d4:5e:c7:23:45:97:bb:40:ec:15:ce:7c:a9 root@alderaan (RSA)
# mailman3.archlinux.org
1024 SHA256:Vs/PxyU74qe6uR5EUUMWhDLA+B8lBQO2PEbRSmZwzYA root@archlinux-packer (DSA)
256 SHA256:ARXQTmcvjHISznthbjI04GBOUEuQAIT2v/fRdAg3Zqw root@archlinux-packer (ECDSA)
256 SHA256:R6sapXFYhonwFNXA90p6OMy3vhKD9P9oPd00/BeuPTA root@archlinux-packer (ED25519)
3072 SHA256:xIJSPj5r2b3WEwwyx1qG7cCysqFHQfELUGE3vaRlxsM root@archlinux-packer (RSA)
1024 MD5:93:7b:7f:47:09:5b:b5:bf:a3:ad:f7:5f:a2:a1:e5:dd root@archlinux-packer (DSA)
256 MD5:e5:30:24:b4:03:0a:8b:07:23:5b:8b:9e:68:f3:7e:45 root@archlinux-packer (ECDSA)
256 MD5:91:95:e9:e2:1f:17:24:66:10:ae:29:ea:90:41:d9:fb root@archlinux-packer (ED25519)
3072 MD5:97:9f:77:0e:f5:99:44:f3:ab:db:4b:f4:4a:98:cd:dc root@archlinux-packer (RSA)
# matrix.archlinux.org
1024 SHA256:4xl3Vzj2VTffMV6zCiAx0DSrsYIBmMnWo41kjR4ZWUo root@archlinux-packer (DSA)
256 SHA256:+v4KFzSadzQmENY2HvHpn8Zse0opJc7FaixR7/K3y0Y root@archlinux-packer (ECDSA)
......
This diff is collapsed.
$ANSIBLE_VAULT;1.1;AES256
31363037363934306662343830323131313361383561656330316537366362656133643237666339
3932386437386338336663613461653463326461336666350a346536333865333338376638643834
30306264356232333761333361313239383931663632396534346333643131326364303066363464
6130363230346562310a626439396536656231643265316438623835656363333034323038616234
39383731303637346233653332333965623961343163363935633532623235316633373566336464
38363433663862636233376164313566613732323138333135663530333866333732666539376563
663861353832316436633761323334313362
---
ansible_ssh_user: "{{ rsync_net_username }}"
known_host: "ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3"
---
filesystem: btrfs
postgres_backup_dir: "/var/lib/postgres/backup"
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
......
......@@ -5,6 +5,9 @@ apollo.archlinux.org
luna.archlinux.org
dragon.archlinux.org
[rsync_net]
ch-s012.rsync.net
[pia]
ind.mirror.pkgbuild.com
jpn.mirror.pkgbuild.com
......@@ -40,6 +43,11 @@ phrik.archlinux.org
bbs.archlinux.org
homedir.archlinux.org
bugs.archlinux.org
aur-dev.archlinux.org
[borg_hosts]
vostok.archlinux.org
ch-s012.rsync.net
[public_html]
homedir.archlinux.org
......
......@@ -10,7 +10,7 @@
- { role: sshd }
- { role: unbound }
- { role: root_ssh }
- { role: borg-client }
- { role: borg-client, tags: ["borg"] }
- { role: certbot }
- { role: nginx }
- role: postgres
......
......@@ -10,6 +10,7 @@
- { role: sshd }
- { role: unbound }
- { role: root_ssh }
- { role: borg-client, tags: ["borg"] }
- { role: certbot }
- role: postgres
postgres_max_connections: 100
......
---
- name: setup rsync.net account
hosts: ch-s012.rsync.net
gather_facts: False
roles:
- { role: rsync_net, backup_dir: "backup", backup_clients: "{{groups['borg_clients']}}", tags: ["borg"] }
---
- name: fetch ssh hostkeys
hosts: all
hosts: all,!rsync_net
tasks:
- name: fetch hostkey checksums
shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done"
......@@ -11,20 +11,27 @@
register: known_hosts
- name: store hostkeys
hosts: 127.0.0.1
hosts: localhost
tasks:
- name: store hostkeys
copy:
dest: "{{playbook_dir}}/../../docs/ssh-hostkeys.txt"
content: "{% for host in groups['all'] | sort %}# {{host}}\n{{hostvars[host].ssh_hostkeys.stdout}}\n\n{% endfor %}"
local_action:
module: copy
dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt"
content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
- name: store known_hosts
copy:
dest: "{{playbook_dir}}/../../docs/ssh-known_hosts.txt"
content: "{% for host in groups['all'] | sort %}# {{host}}\n{{hostvars[host].known_hosts.stdout}}\n\n{% endfor %}"
local_action:
module: copy
dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
- name: manually append rsync.net host keys
local_action:
module: lineinfile
path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
- name: upload known_hosts to all nodes
hosts: all
hosts: all,!rsync_net
tasks:
- name: upload known_hosts
copy: dest=/etc/ssh/ssh_known_hosts src="{{playbook_dir}}/../../docs/ssh-known_hosts.txt"
copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
tags: ['upload-known-hosts']
---
backup_host: "borg@vostok.archlinux.org"
backup_dir: "/backup/{{inventory_hostname}}"
backup_hosts:
- host: "borg@vostok.archlinux.org"
dir: "/backup/{{ inventory_hostname }}"
suffix: ""
- host: "{{ rsync_net_username }}@ch-s012.rsync.net"
dir: "backup/{{ inventory_hostname }}"
suffix: "-offsite"
......@@ -7,22 +7,26 @@
template: src=borg-restore.cfg.j2 dest=/etc/borg-restore.cfg owner=root group=root mode=0644
- name: check if borg repository already exists
command: borg list {{ backup_host }}:{{ backup_dir }}
command: borg list {{ item['host'] }}:{{ item['dir'] }}
register: borg_list
ignore_errors: True
loop: "{{ backup_hosts }}"
- name: init borg repository
command: borg init -e keyfile {{ backup_host }}:{{ backup_dir }}
command: borg init -e keyfile {{ item['host'] }}:{{ item['dir'] }}
when: borg_list is failed
environment:
BORG_PASSPHRASE: ""
ignore_errors: True # This can sometimes fail if a backup is in progress :/
loop: "{{ backup_hosts }}"
- name: install scripts
template: src={{item}}.j2 dest=/usr/local/bin/{{item}} owner=root group=root mode=0755
with_items:
- borg-backup.sh
- borg
- name: install convenience scripts
template: src=borg.j2 dest=/usr/local/bin/borg{{ item['suffix'] }} owner=root group=root mode=0755
loop: "{{ backup_hosts }}"
- name: install borg backup scripts
template: src=borg-backup.sh.j2 dest=/usr/local/bin/borg-backup{{ item['suffix'] }}.sh owner=root group=root mode=0755
loop: "{{ backup_hosts }}"
- name: install postgres backup script
template: src=backup-postgres.sh.j2 dest=/usr/local/bin/backup-postgres.sh owner=root group=root mode=0755
......@@ -42,22 +46,22 @@
when: mysql_backup_dir is defined
- name: install mysql backup config
template: src=backup-my.cnf.j2 dest={{mysql_backup_defaults}}
template: src=backup-my.cnf.j2 dest={{ mysql_backup_defaults }}
when: mysql_backup_defaults is defined
- name: create mysql backup directory
file: path={{mysql_backup_dir}} state=directory owner=root group=root
file: path={{ mysql_backup_dir }} state=directory owner=root group=root
when: mysql_backup_dir is defined
- name: remove xtrabackup, mariadb ships its own fork
pacman: name=xtrabackup state=absent
when: mysql_backup_dir is defined
- name: install systemd timers for backup
copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
- name: install systemd timer and service for backup
template: src={{ item }}.j2 dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
with_items:
- borg-backup.timer
- borg-backup.service
- name: activate systemd timers for backup
service: name=borg-backup.timer enabled=yes state=started
systemd: name=borg-backup.timer enabled=yes state=started daemon-reload=yes
......@@ -3,4 +3,6 @@ Description=Borg backup
[Service]
Type=oneshot
ExecStart=/usr/local/bin/borg-backup.sh
{% for backup in backup_hosts %}
ExecStart=/usr/local/bin/borg-backup{{ backup['suffix'] }}.sh
{% endfor %}
......@@ -60,8 +60,8 @@ borg create -v --stats -C lz4 \
-e "$backup_mountdir/var/lib/archbuild" \
-e "$backup_mountdir/var/lib/archbuilddest" \
-e "$backup_mountdir/srv/archive" \
{{ backup_host }}:{{ backup_dir }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
borg prune -v {{ backup_host }}:{{ backup_dir }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
{{ item['host'] }}:{{ item['dir'] }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
borg prune -v {{ item['host'] }}:{{ item['dir'] }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
if is_btrfs "$src"; then
umount -R "$backup_mountdir"
......
#!/bin/bash
BORG_REPO="{{ backup_host }}:{{ backup_dir }}" exec /usr/bin/borg "$@"
BORG_REPO="{{ item['host'] }}:{{ item['dir'] }}" exec /usr/bin/borg "$@"
......@@ -25,7 +25,7 @@
mode: 0700
with_items: "{{ backup_clients }}"
- name: fetch ssh keys
- name: fetch ssh keys from each borg client machine
command: cat /root/.ssh/id_rsa.pub
register: ssh_keys
delegate_to: "{{ item }}"
......
---
# We have to set up the rsync.net account in a weird fashion because
# they don't support ansible directly (no Python and such).
- name: create the root backup directory at {{ backup_dir }}
raw: mkdir -p {{ backup_dir }}
- name: fetch ssh keys from each borg client machine
command: cat /root/.ssh/id_rsa.pub
register: client_ssh_keys
delegate_to: "{{ item }}"
with_items: "{{ backup_clients }}"
remote_user: root
- local_action: tempfile state=file
register: tempfile
- local_action: copy content="{{ lookup('template', 'authorized_keys.j2')}}" dest="{{ tempfile.path }}"
- name: upload authorized_keys file
local_action: command scp "{{ tempfile.path }}" "{{ rsync_net_username }}@{{ inventory_hostname }}":.ssh/authorized_keys
#jinja2: lstrip_blocks: True
# Arch DevOps keys
{% for user in root_ssh_keys | sort -%}
{{ lookup('file', '../pubkeys/' + user) }}
{% endfor %}
# Client machines keys
{% for client_key in client_ssh_keys.results %}
command="/usr/bin/borg serve --restrict-to-path {{ backup_dir }}/{{ client_key['item'] }}",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc {{ client_key['stdout'] }}
{% endfor %}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment