diff --git a/README.md b/README.md index bf58a8e4172b6b0894e81da1252c2273b21d1f03..eb3a657d0da35fee7e6c1df4930260a3b518a9ea 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,23 @@ The following steps should be used to update our managed servers: * checkservices * reboot +##### Semi-automated server updates (experimental) + +For updating a lot of servers in a more unattended manner, the following +playbook can be used: + + ansible-playbook playbooks/tasks/upgrade-servers.yml [-l SUBSET] + +It runs `pacman -Syu` on the targeted hosts in batches and then reboots them. +If any server fails to reboot successfully, the rolling update stops and +further batches are cancelled. To display the packages updated on each host, +you can pass the `--diff` option to ansible-playbook. + +Using this update method, `.pacnew` files are left unmerged which is OK for +most configuration files that are managed by Ansible. However, care must be +taken with updates that require manual intervention (e.g. major PostgreSQL +releases). + ## Servers This section has been moved to [docs/servers.md](docs/servers.md). diff --git a/playbooks/tasks/include/upgrade-server.yml b/playbooks/tasks/include/upgrade-server.yml new file mode 100644 index 0000000000000000000000000000000000000000..a63adb3aaeb293184a93e4457375ca9b51c8796d --- /dev/null +++ b/playbooks/tasks/include/upgrade-server.yml @@ -0,0 +1,47 @@ +--- + +- name: ensure latest keyring + pacman: + name: archlinux-keyring + state: latest + update_cache: yes + +- name: upgrade all packages + pacman: + update_cache: yes + upgrade: yes + register: pacman_upgrade + +- name: check for running builds + block: + - name: list build-related processes + command: pgrep -x 'mkarchroot|makechrootpkg|systemd-nspawn' + register: pgrep + ignore_errors: true + + - name: abort reboot with running builds + meta: end_host + when: pgrep is succeeded + when: "'buildservers' in group_names" + +- name: gemini pre-reboot checks + block: + - name: wait for svntogit to finish + wait_for: + path: /srv/svntogit/update-repos.sh.lock + state: absent + + - name: list logged on users + command: who + register: who + + - name: abort reboot with logged on users + meta: end_host + when: + - who is changed + - who.stdout_lines|length > 1 + when: inventory_hostname == "gemini.archlinux.org" + +- name: reboot + reboot: + when: pacman_upgrade is changed diff --git a/playbooks/tasks/upgrade-servers.yml b/playbooks/tasks/upgrade-servers.yml new file mode 100644 index 0000000000000000000000000000000000000000..6f7edb5d3e92d02356d3d67fb8a732a944d20694 --- /dev/null +++ b/playbooks/tasks/upgrade-servers.yml @@ -0,0 +1,21 @@ +--- + +- name: upgrade and reboot all hetzner servers + hosts: all,!kape_servers,!packet_net,!rsync_net,!hetzner_storageboxes + max_fail_percentage: 0 + serial: 20% + gather_facts: false + + tasks: + - name: upgrade each host in this batch + include_tasks: include/upgrade-server.yml + +- name: upgrade and reboot all kape and packet.net servers + hosts: kape_servers,packet_net + max_fail_percentage: 0 + serial: 1 + gather_facts: false + + tasks: + - name: upgrade each host in this batch + include_tasks: include/upgrade-server.yml