host_vars/gluebuddy.archlinux.org/vault_wireguard.yml

+$ANSIBLE_VAULT;1.1;AES256
+34656337316361363932346233363364323635373936306637373365616636383538313530636662
+3265386661633264313066613464323236336465343531350a646462376330393966376161666230
+63656137343539353661613838663735366430323938356232646438393032333965306430623036
+3132363866316264370a393336636365333366373934323464383039643239313430363333386432
+31616232653862366461653435356461616234633638363633376338653830346431333836393736
+63396238313931646230653539326530613561326339303337633138663833366361316437663666
+33656631316463616563333435356666623639346561333063383036343133666135323162663730
+37316131386333623463

host_vars/mirror.pkgbuild.com/misc

 ---
 mirror_domain: mirror.pkgbuild.com
+mirror_debug_packages: false
 archweb_mirrorcheck_locations: [20, 21]
 filesystem: btrfs
 

host_vars/u236610.your-storagebox.de

----
-ansible_ssh_user: "{{ hetzner_storagebox_username }}"
-known_host: "[u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs"

host_vars/zh1905.rsync.net

----
-ansible_ssh_user: "{{ rsync_net_username }}"
-known_host: "zh1905.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJtclizeBy1Uo3D86HpgD3LONGVH0CJ0NT+YfZlldAJd"

hosts

@@ -2,12 +2,6 @@
 secure-runner1.archlinux.org
 gemini.archlinux.org
 
-[rsync_net]
-zh1905.rsync.net
-
-[hetzner_storageboxes]
-u236610.your-storagebox.de
-
 [packet_net]
 runner2.archlinux.org
 repro1.pkgbuild.com
@@ -45,10 +39,7 @@ patchwork.archlinux.org
 security.archlinux.org
 md.archlinux.org
 lists.archlinux.org
-
-[borg_hosts]
-zh1905.rsync.net
-u236610.your-storagebox.de
+gluebuddy.archlinux.org
 
 [public_html]
 homedir.archlinux.org
@@ -97,7 +88,6 @@ repro1.pkgbuild.com
 repro2.pkgbuild.com
 
 [memcached]
-aur.archlinux.org
 wiki.archlinux.org
 patchwork.archlinux.org
 
@@ -139,6 +129,8 @@ md.archlinux.org
 man.archlinux.org
 dashboards.archlinux.org
 lists.archlinux.org
+gluebuddy.archlinux.org
+debuginfod.archlinux.org
 
 [wireguard]
 archlinux.org
@@ -175,6 +167,8 @@ md.archlinux.org
 man.archlinux.org
 dashboards.archlinux.org
 lists.archlinux.org
+gluebuddy.archlinux.org
+debuginfod.archlinux.org
 
 [kape_servers]
 asia.mirror.pkgbuild.com
@@ -182,3 +176,10 @@ america.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com
 repro2.pkgbuild.com
 runner1.archlinux.org
+
+[dedicated_servers]
+gemini.archlinux.org
+build.archlinux.org
+runner1.archlinux.org
+runner2.archlinux.org
+secure-runner1.archlinux.org

playbooks/all-hosts-basic.yml

 ---
 
 - name: basic setup for all hosts
-  hosts: all,!hetzner_storageboxes,!rsync_net
+  hosts: all
   remote_user: root
   roles:
     - { role: common }

playbooks/aur-dev.archlinux.org.yml

@@ -11,8 +11,7 @@
     - { role: nginx }
     - { role: mariadb }
     - { role: sudo }
-    - { role: php_fpm, php_extensions: ['iconv', 'memcached', 'mysqli', 'pdo_mysql'], zend_extensions: ['opcache'] }
-    - { role: memcached }
+    - { role: redis }
     - { role: uwsgi }
     - { role: borg_client, tags: ["borg"] }
     - { role: postfix_null }

playbooks/aur.archlinux.org.yml

@@ -13,8 +13,7 @@
     - { role: nginx }
     - { role: mariadb, mariadb_innodb_buffer_pool_size: '1G' }
     - { role: sudo }
-    - { role: php_fpm, php_extensions: ['iconv', 'memcached', 'mysqli', 'pdo_mysql'], zend_extensions: ['opcache'] }
-    - { role: memcached }
+    - { role: redis }
     - { role: uwsgi }
     - { role: borg_client, tags: ["borg"] }
     - { role: postfix_null }

playbooks/build.archlinux.org.yml

@@ -5,7 +5,7 @@
   remote_user: root
   roles:
     - { role: common }
-    - { role: tools }
+    - { role: tools, extra_utils: ['setconf'] }
     - { role: sshd }
     - { role: root_ssh }
     - { role: archusers }

playbooks/debuginfod.archlinux.org.yml

+---
+- name: setup debuginfod.archlinux.org
+  hosts: debuginfod.archlinux.org
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: firewalld }
+    - { role: wireguard }
+    - { role: hardening }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: certbot }
+    - { role: nginx }
+    - { role: debuginfod }
+    - { role: syncdebug }
+    - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/gemini.archlinux.org.yml

@@ -5,7 +5,6 @@
   remote_user: root
   vars:
     archweb_db_host: "{{ hostvars['archlinux.org']['wireguard_address'] }}"
-    dbscripts_commit: '20211026'
   roles:
     - { role: common }
     - { role: tools }

playbooks/gluebuddy.archlinux.org.yml

+---
+
+- name: setup gluebuddy.archlinux.org
+  hosts: gluebuddy.archlinux.org
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: firewalld }
+    - { role: wireguard }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: gluebuddy }
+    - { role: borg_client, tags: ["borg"] }
+    - { role: prometheus_exporters }
+    - { role: promtail }
+    - { role: fail2ban }

playbooks/hetzner_storagebox.yml

 ---
 
 - name: setup Hetzner storagebox account
-  hosts: u236610.your-storagebox.de
+  hosts: localhost
   gather_facts: false
   roles:
-    - { role: hetzner_storagebox, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }
+    - role: hetzner_storagebox
+      backup_clients: "{{ groups['borg_clients'] }}"
+      backup_dir: backup
+      storagebox_id: "{{ hetzner_storagebox_id }}"
+      storagebox_hostname: "{{ hetzner_storagebox_username }}.your-storagebox.de"
+      storagebox_username: "{{ hetzner_storagebox_username }}"
+      storagebox_password: "{{ hetzner_storagebox_password }}"
+      tags: ["borg"]

playbooks/mirrors.yml

@@ -9,6 +9,7 @@
     - { role: certbot }
     - { role: nginx }
     - { role: syncrepo, tags: ['nginx'] }
+    - { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
     - { role: archweb, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
     - { role: prometheus_exporters }
     - { role: promtail }

playbooks/rsync.net.yml

 ---
 
 - name: setup rsync.net account
-  hosts: zh1905.rsync.net
+  hosts: localhost
   gather_facts: false
   roles:
-    - { role: rsync_net, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }
+    - role: rsync_net
+      backup_clients: "{{ groups['borg_clients'] }}"
+      backup_dir: backup
+      tags: ["borg"]

playbooks/tasks/include/upgrade-server.yml

@@ -8,10 +8,13 @@
 
 - name: upgrade all packages
   pacman:
-    update_cache: yes
     upgrade: yes
   register: pacman_upgrade
 
+- name: stop if no packages were upgraded
+  meta: end_host
+  when: pacman_upgrade is not changed
+
 - name: check for running builds
   block:
     - name: list build-related processes
@@ -24,13 +27,20 @@
       when: pgrep is succeeded
   when: "'buildservers' in group_names"
 
-- name: gemini pre-reboot checks
+
+- name: check for active borg backup jobs
   block:
-    - name: wait for svntogit to finish
-      wait_for:
-        path: /srv/svntogit/update-repos.sh.lock
-        state: absent
+    - name: check if /backup exists
+      stat: path=/backup
+      register: backup_mountdir
+
+    - name: abort reboot when borg backup is running
+      meta: end_host
+      when: backup_mountdir.stat.exists
+  when: "'borg_clients' in group_names"
 
+- name: gemini pre-reboot checks
+  block:
     - name: list logged on users
       command: who
       register: who
@@ -40,8 +50,15 @@
       when:
         - who is changed
         - who.stdout_lines|length > 1
+
+    - name: stop arch-svntogit.timer
+      service: name=arch-svntogit.timer state=stopped
+
+    - name: wait for svntogit to finish
+      wait_for:
+        path: /srv/svntogit/update-repos.sh.lock
+        state: absent
   when: inventory_hostname == "gemini.archlinux.org"
 
 - name: reboot
   reboot:
-  when: pacman_upgrade is changed

playbooks/tasks/sync-ssh-hostkeys.yml

 ---
 
 - name: fetch ssh hostkeys
-  hosts: all,!rsync_net,!hetzner_storageboxes
+  hosts: all
+  gather_facts: false
   tasks:
     - name: fetch hostkey checksums
-      shell: "for type in sha256 md5; do for file in /etc/ssh/ssh_host_*.pub; do ssh-keygen -l -f $file -E $type; done; echo; done"
+      shell: |
+        for type in sha256 md5; do
+          for file in /etc/ssh/ssh_host_*.pub; do
+            ssh-keygen -l -f $file -E $type
+          done
+          echo
+        done
       register: ssh_hostkeys
       changed_when: ssh_hostkeys | length > 0
+
     - name: fetch known_hosts
-      shell: "set -o pipefail && ssh-keyscan 127.0.0.1 2>/dev/null | sed 's#^127.0.0.1#{{ inventory_hostname }}#' | sort"
+      shell: |
+        set -eo pipefail
+        ssh-keyscan 127.0.0.1 2>/dev/null \
+        | sed 's#^127.0.0.1#{{ inventory_hostname }}#' \
+        | sort
       environment:
         LC_COLLATE: C  # to ensure reproducible ordering
       args:
-        executable: /bin/bash  # required for repro3.pkgbuild.com which is ubuntu and has dash as default shell
+        executable: /bin/bash
       register: known_hosts
       changed_when: known_hosts | length > 0
 
@@ -22,28 +34,27 @@
     - name: store hostkeys
       copy:
         dest: "{{ playbook_dir }}/../../docs/ssh-hostkeys.txt"
-        content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].ssh_hostkeys.stdout }}\n\n{% endfor %}"
+        content: |
+          {% for host in query('inventory_hostnames', 'all') | sort %}
+          # {{ host }}
+          {{ hostvars[host].ssh_hostkeys.stdout }}
+
+          {% endfor %}
         mode: preserve
-      delegate_to: localhost
+
     - name: store known_hosts
-      copy:
-        dest: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        content: "{% for host in query('inventory_hostnames', 'all,!rsync_net,!hetzner_storageboxes,!localhost') | sort %}# {{ host }}\n{{ hostvars[host].known_hosts.stdout }}\n\n{% endfor %}"
-        mode: preserve
-      delegate_to: localhost
-    - name: manually append rsync.net host keys
-      lineinfile:
-        path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n{% endfor %}"
-      delegate_to: localhost
-    - name: manually append Hetzner Storageboxes host keys
-      lineinfile:
+      blockinfile:
         path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        line: "{% for host in query('inventory_hostnames', 'hetzner_storageboxes') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n{% endfor %}"
-      delegate_to: localhost
+        block: |
+
+          {% for host in query('inventory_hostnames', 'all') | sort %}
+          # {{ host }}
+          {{ hostvars[host].known_hosts.stdout }}
+
+          {% endfor %}
 
 - name: upload known_hosts to all nodes
-  hosts: all,!rsync_net,!hetzner_storageboxes
+  hosts: all
   tasks:
     - name: upload known_hosts
       copy: dest=/etc/ssh/ssh_known_hosts src="{{ playbook_dir }}/../../docs/ssh-known_hosts.txt" owner=root group=root mode=0644