Verified Commit 764df6ee authored by Sven-Hendrik Haase's avatar Sven-Hendrik Haase
Browse files

Switch from vostok to storagebox (fixes #51)

parent b8695f04
...@@ -147,11 +147,6 @@ The following steps should be used to update our managed servers: ...@@ -147,11 +147,6 @@ The following steps should be used to update our managed servers:
## Servers ## Servers
### vostok
#### Services
- backups
### orion ### orion
#### Services #### Services
...@@ -291,8 +286,8 @@ Adding a new server to be backed up goes as following: ...@@ -291,8 +286,8 @@ Adding a new server to be backed up goes as following:
* Add the server to [borg-clients] in hosts * Add the server to [borg-clients] in hosts
* Run the borg role on vostok to allow the new machine to create backups * Run the borg role on u236610.your-storagebox.de to allow the new machine to create backups
ansibe-playbook playbooks/vostok.yml -t borg ansibe-playbook playbooks/hetzner_storagebox.yml
* Run the borg role for rsync.net to allow the new machine to create backups * Run the borg role for rsync.net to allow the new machine to create backups
ansibe-playbook playbooks/rsync.net.yml ansibe-playbook playbooks/rsync.net.yml
...@@ -300,23 +295,25 @@ Adding a new server to be backed up goes as following: ...@@ -300,23 +295,25 @@ Adding a new server to be backed up goes as following:
* Run the borg role on the new machine to initialize the repository * Run the borg role on the new machine to initialize the repository
ansibe-playbook playbooks/$machine.yml -t borg ansibe-playbook playbooks/$machine.yml -t borg
Backups should be checked now and then. Some common tasks: Backups should be checked now and then. Some common tasks are listed below.
You'll have to get the correct username from the vault.
### Listing current backups per server ### Listing current backups per server
borg list borg@vostok.archlinux.org:/backup/<hostname> borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/<hostname>
borg list $rsync_net_username@ch-s012.rsync.net:backup/<hostname>
Example Example
borg list borg@vostok.archlinux.org:/backup/homedir.archlinux.org borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/homedir.archlinux.org
### Listing files in a backup ### Listing files in a backup
borg list borg@vostok.archlinux.org:/backup/<hostname>::<archive name> borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/<hostname>::<archive name>
Example Example
borg list borg@vostok.archlinux.org:/backup/homedir.archlinux.org::20191127-084357 borg list $hetzner_storagebox_username@u236610.your-storagebox.de:backup/homedir.archlinux.org::20191127-084357
## One-shots ## One-shots
......
...@@ -294,15 +294,3 @@ ...@@ -294,15 +294,3 @@
256 MD5:62:eb:27:c4:a1:6f:a4:21:ed:50:6f:dd:bf:37:4e:ab root@archlinux-packer (ECDSA) 256 MD5:62:eb:27:c4:a1:6f:a4:21:ed:50:6f:dd:bf:37:4e:ab root@archlinux-packer (ECDSA)
256 MD5:9a:97:48:f7:11:b3:32:ba:fa:ab:9f:0c:41:41:da:e4 root@archlinux-packer (ED25519) 256 MD5:9a:97:48:f7:11:b3:32:ba:fa:ab:9f:0c:41:41:da:e4 root@archlinux-packer (ED25519)
3072 MD5:f3:11:d6:58:f9:32:d1:34:fa:4e:d9:e3:d7:c8:6b:f2 root@archlinux-packer (RSA) 3072 MD5:f3:11:d6:58:f9:32:d1:34:fa:4e:d9:e3:d7:c8:6b:f2 root@archlinux-packer (RSA)
# vostok.archlinux.org
1024 SHA256:FddVsY5JTplRbgQA3m3XA3bYbQ9SOQ8i/gZBlj4NUss root@vostok (DSA)
256 SHA256:VNBI73QTTzMskhFYissdeC5ZxzsqONu0DuudJWkbxiI root@vostok (ECDSA)
256 SHA256:NaPE8gB9f0CHLpDf8PexYFKA9OYQ0h6tcG5dIjr7NPw root@vostok (ED25519)
2048 SHA256:Ap2YWeHDxUVub9qSxEVh1FgefWvjDGDBGjPfHIs/Pv0 root@vostok (RSA)
1024 MD5:52:26:17:aa:59:ce:9b:ef:89:3f:8a:56:04:ed:29:a7 root@vostok (DSA)
256 MD5:34:cb:51:e7:9f:e0:4c:eb:00:5c:8b:59:9d:ab:4c:60 root@vostok (ECDSA)
256 MD5:15:11:f7:63:1b:e1:9b:52:e1:c6:87:ac:0f:1e:fb:4a root@vostok (ED25519)
2048 MD5:b0:03:6c:cc:4d:bf:7c:e9:8b:3f:98:4f:80:ea:4e:56 root@vostok (RSA)
...@@ -131,12 +131,11 @@ svn2gittest ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYA ...@@ -131,12 +131,11 @@ svn2gittest ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYA
svn2gittest ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/2iOTxrCAXRfpzLfJzvwlnISD+G7wxH5aONHybVga3O1p/KQv8TiTK/+dqnH9ZW6KBPnQjtsIEmzx/ZIr/aAMeZX8OwZAl1wbKdFSpeCAhJtK7zKGr/f8qlYttHgttiGq9PJMQ8bwldaAWEllgFhr/hveKPckfACrgADTJStaWW7eeqAe28BjqHm7BpT/jO3DAeVaHrcY8CVtHsCFUXCoHSFf5ER3QOH4LTfqM7Imz0cK8i29x1H5RruM+pzIMaHthabWHERwE/V/j0Xt0gEyfXZOWK5K+3ueZjVl5yUsvpSxoYSQzpQg+rvsnN7L7qRYjA04bI6NWopFdpMKzk0NcVF64tR4wfPfaaTjYdH2HU3uO/JRtN4IZmFdI2vL1UtycM9w44humZyNUGjeMUBFbBwwPGTAkcTWjPW8HLtVYjgS6Zl5/cBjLSG90qpg7hPo0V80ybRRRJbOH14txlX4z2btxCiurHCi+nxzYBABGINsZgd9s+7AORWtvfBGo0U= svn2gittest ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/2iOTxrCAXRfpzLfJzvwlnISD+G7wxH5aONHybVga3O1p/KQv8TiTK/+dqnH9ZW6KBPnQjtsIEmzx/ZIr/aAMeZX8OwZAl1wbKdFSpeCAhJtK7zKGr/f8qlYttHgttiGq9PJMQ8bwldaAWEllgFhr/hveKPckfACrgADTJStaWW7eeqAe28BjqHm7BpT/jO3DAeVaHrcY8CVtHsCFUXCoHSFf5ER3QOH4LTfqM7Imz0cK8i29x1H5RruM+pzIMaHthabWHERwE/V/j0Xt0gEyfXZOWK5K+3ueZjVl5yUsvpSxoYSQzpQg+rvsnN7L7qRYjA04bI6NWopFdpMKzk0NcVF64tR4wfPfaaTjYdH2HU3uO/JRtN4IZmFdI2vL1UtycM9w44humZyNUGjeMUBFbBwwPGTAkcTWjPW8HLtVYjgS6Zl5/cBjLSG90qpg7hPo0V80ybRRRJbOH14txlX4z2btxCiurHCi+nxzYBABGINsZgd9s+7AORWtvfBGo0U=
svn2gittest ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFgvu1kXUUucO0ss+A+cDR1dsn71N77T9U/wWtcf+1w5 svn2gittest ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFgvu1kXUUucO0ss+A+cDR1dsn71N77T9U/wWtcf+1w5
# vostok.archlinux.org
vostok.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK/pvIigdOhz3THBtkk9DjZitryhSHaxgVRDBPJQ+C1oGtAdUpolc9G1BctfZhcbbpqTmE9ELGPJus2vK0maARg=
vostok.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHl3sguy1lY6HMBBNggR4t/svRAM5+NkrQhKytLKO0Oq
vostok.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVaHKi8eFkVYuVzVxta0CbjxyssIPkQD1ufXg6xUDPI5y3/wEE9c/6g3emhwHN/NRixk70xlH5lkSfv6zH1cY5PO0uOWxBXrTDU0VtP0l4LH5gFsp9G8FSZht39erBMR/aIvmSMciC+TPoBfilwVrOb5RLYzXkft/z9QwBFGN/quCwGddQ0FSvyAUwGQctBC5NUsYCbSe+KipNPBPfdJEE0+KtM4L7NSG1sDBKQq2H8W1+BopXRh42d1clOmcVUmLqMCwSfvdd6jQwez2q9f6fJGY+iGpJkBYBeV+nHRVdXdBlohdzLn5N2+YeW3Nx9jF9pg9B/IhoSXzZ284bC7zZ
# ch-s012.rsync.net # ch-s012.rsync.net
ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3 ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3
# u236610.your-storagebox.de
[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
$ANSIBLE_VAULT;1.1;AES256
30353736373466623531333732393935376435353939366632383839376531653761656631646638
3831333465373263336232653931643162656363653039320a383736393636613231386465663430
37313062303933653633626637623539363565316161666433656138393036343538623863386666
3039346264393066620a396231646534303262616162346261643639323838313635366332653861
39353239393134326130383766323832383361656431336335616138363865623865356538636139
63363234343962333166313038646633613534653963613961656336646464393338373635663832
62396633363932663931633532363732653766356136393137363366376134363135663864313935
63323635666431353165396235633066313334316161396163646633366536366361643331386461
6535
---
ansible_ssh_user: "{{ hetzner_storagebox_username }}"
known_host: "[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs"
---
hostname: "vostok"
ipv4_address: "5.9.158.171"
ipv4_netmask: "/32"
ipv6_address: "2a01:4f8:190:51aa::1"
ipv6_netmask: "/128"
ipv4_gateway: "5.9.158.161"
ipv6_gateway: "fe80::1"
filesystem: ext4
system_disks:
- /dev/sda
- /dev/sdb
zabbix_agent_templates:
- Template OS Linux
[hetzner] [hetzner]
orion.archlinux.org orion.archlinux.org
vostok.archlinux.org
apollo.archlinux.org apollo.archlinux.org
luna.archlinux.org luna.archlinux.org
dragon.archlinux.org dragon.archlinux.org
...@@ -10,6 +9,9 @@ gemini.archlinux.org ...@@ -10,6 +9,9 @@ gemini.archlinux.org
[rsync_net] [rsync_net]
ch-s012.rsync.net ch-s012.rsync.net
[hetzner_storageboxes]
u236610.your-storagebox.de
[pia] [pia]
jpn.mirror.pkgbuild.com jpn.mirror.pkgbuild.com
ger.mirror.pkgbuild.com ger.mirror.pkgbuild.com
...@@ -44,8 +46,8 @@ accounts.archlinux.org ...@@ -44,8 +46,8 @@ accounts.archlinux.org
gemini.archlinux.org gemini.archlinux.org
[borg_hosts] [borg_hosts]
vostok.archlinux.org
ch-s012.rsync.net ch-s012.rsync.net
u236610.your-storagebox.de
[public_html] [public_html]
homedir.archlinux.org homedir.archlinux.org
......
---
- name: setup Hetzner storagebox account
hosts: u236610.your-storagebox.de
gather_facts: False
roles:
- { role: hetzner_storagebox, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }
--- ---
- name: fetch ssh hostkeys - name: fetch ssh hostkeys
hosts: all,!rsync_net hosts: all,!rsync_net,!hetzner_storageboxes
tasks: tasks:
- name: fetch hostkey checksums - name: fetch hostkey checksums
shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done" shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done"
...@@ -18,21 +18,26 @@ ...@@ -18,21 +18,26 @@
- name: store hostkeys - name: store hostkeys
copy: copy:
dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt" dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt"
content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}" content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
delegate_to: localhost delegate_to: localhost
- name: store known_hosts - name: store known_hosts
copy: copy:
dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt" dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}" content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
delegate_to: localhost delegate_to: localhost
- name: manually append rsync.net host keys - name: manually append rsync.net host keys
lineinfile: lineinfile:
path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt" path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}" line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
delegate_to: localhost delegate_to: localhost
- name: manually append Hetzner Storageboxes host keys
lineinfile:
path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
line: "{% for host in query('inventory_hostnames', 'hetzner_storageboxes') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
delegate_to: localhost
- name: upload known_hosts to all nodes - name: upload known_hosts to all nodes
hosts: all,!rsync_net hosts: all,!rsync_net,!hetzner_storageboxes
tasks: tasks:
- name: upload known_hosts - name: upload known_hosts
copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt" copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
......
---
- name: setup vostok
hosts: vostok.archlinux.org
remote_user: root
roles:
- { role: common }
- { role: tools }
- { role: sshd }
- { role: unbound }
- { role: root_ssh }
- { role: borg-server, backup_dir: "/backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }
--- ---
backup_hosts: backup_hosts:
- host: "borg@vostok.archlinux.org" - host: "{{ hetzner_storagebox_username }}@u236610.your-storagebox.de"
dir: "/backup/{{ inventory_hostname }}" dir: "backup/{{ inventory_hostname }}"
suffix: "" suffix: ""
- host: "{{ rsync_net_username }}@ch-s012.rsync.net" - host: "{{ rsync_net_username }}@ch-s012.rsync.net"
dir: "backup/{{ inventory_hostname }}" dir: "backup/{{ inventory_hostname }}"
......
...@@ -72,7 +72,6 @@ borg create -v --stats -C zstd \ ...@@ -72,7 +72,6 @@ borg create -v --stats -C zstd \
-e "$backup_mountdir/var/lib/archbuild" \ -e "$backup_mountdir/var/lib/archbuild" \
-e "$backup_mountdir/var/lib/archbuilddest" \ -e "$backup_mountdir/var/lib/archbuilddest" \
-e "$backup_mountdir/var/lib/docker" \ -e "$backup_mountdir/var/lib/docker" \
-e "$backup_mountdir/srv/archive" \
{{ item['host'] }}:{{ item['dir'] }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir" {{ item['host'] }}:{{ item['dir'] }}::$(date "+%Y%m%d-%H%M%S") "$backup_mountdir"
borg prune -v {{ item['host'] }}:{{ item['dir'] }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6 borg prune -v {{ item['host'] }}:{{ item['dir'] }} --keep-daily=7 --keep-weekly=4 --keep-monthly=6
......
---
# We have to set up the Hetzner Storagebox account in a weird fashion because
# they don't even allow direct SSH.
- name: create the root backup directory at {{ backup_dir }}
expect:
command: bash -c "echo 'mkdir {{ backup_dir }}' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
responses:
(?i)password: "{{ hetzner_storagebox_password }}"
delegate_to: localhost
- name: fetch ssh keys from each borg client machine
command: cat /root/.ssh/id_rsa.pub
register: client_ssh_keys
delegate_to: "{{ item }}"
with_items: "{{ backup_clients }}"
remote_user: root
changed_when: client_ssh_keys.changed
- name: create tempfile
tempfile: state=file
register: tempfile
delegate_to: localhost
- name: fill tempfile
copy: content="{{ lookup('template', 'authorized_keys.j2') }}" dest="{{ tempfile.path }}"
delegate_to: localhost
- name: upload authorized_keys file
expect:
command: bash -c "echo -e 'mkdir .ssh \n chmod 700 .ssh \n put {{ tempfile.path }} .ssh/authorized_keys \n chmod 600 .ssh/authorized_keys' | sftp -P 23 {{ hetzner_storagebox_username }}@{{ inventory_hostname }}"
responses:
(?i)password: "{{ hetzner_storagebox_password }}"
delegate_to: localhost
#jinja2: lstrip_blocks: True
# Arch DevOps keys
{% for user in root_ssh_keys | sort -%}
{{ lookup('file', '../pubkeys/' + user) }}
{% endfor %}
# Client machines keys
{% for client_key in client_ssh_keys.results %}
command="/usr/bin/borg serve --restrict-to-path {{ backup_dir }}/{{ client_key['item'] }}",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc {{ client_key['stdout'] }}
{% endfor %}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment