README.md

@@ -184,6 +184,8 @@ Our Gitlab installation uses [Omnibus](https://docs.gitlab.com/omnibus/) to run
 
     ansible-playbook playbooks/gitlab.archlinux.org.yml --diff -t gitlab
 
+To view the current Gitlab version visit [this url](https://gitlab.archlinux.org/help/)
+
 ## One-shots
 
 A bunch of once-only admin task scripts can be found in `one-shots/`.

docs/backups.md

@@ -8,21 +8,21 @@ You'll have to get the correct username from the vault.
 We use two different borg backup hosts: A primary one and an offsite one.
 The URL format for the primary one is
 
-    ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/<hostname>/repo
+    ssh://u236610@u236610.your-storagebox.de:23/~/backup/<hostname>/repo
 
 while for the offsite one it's
 
-    ssh://<rsync_net_username>@prio.ch-s012.rsync.net:22/~/backup/<hostname>
+    ssh://zh1905@zh1905.rsync.net:22/~/backup/<hostname>
 
 In the examples below, we'll just abbreviate the full address as `<backup_address>`.
 If you want to use one of the examples below, you'll have to fill in the
 placeholder with your desired full address to the backup repository. For instance,
 
-    misc/borg.sh list <backup_address>::20191127-084357
+    misc/borg.sh list <backup_address>
 
 becomes
 
-    misc/borg.sh ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org/repo::20191127-084357
+    misc/borg.sh list ssh://u236610@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org/repo
 
 A convenience wrapper script is available at `misc/borg.sh` which makes sure you
 use the correct keyfile for the given server.

docs/monitoring.md

@@ -23,7 +23,7 @@ For general system performance monitoring [prometheus-node-exporter](https://git
 
 ### Borg
 
-For monitoring our borg backups prometheus-node-exporter's textfile collector feature is used, the textfile is written by a systemd service run periodically by a systemd timer called prometheus-borg-textcollector. Borg's last backup time is recorded for our Hetzner and rsync.net backups. Adding monitoring to a system is as simple as:
+For monitoring our borg backups prometheus-node-exporter's textfile collector feature is used, the textfile is written by a systemd service called prometheus-borg-textcollector. Borg's last backup time is recorded for our Hetzner and rsync.net backups. Adding monitoring to a system is as simple as:
 
 * Add the host to the `borg_clients` group
 * Rollout exporter on host: `ansible-playbook playbooks/host.yml -t prometheus_exporters`

docs/ssh-hostkeys.txt

@@ -174,6 +174,17 @@
 256 MD5:dd:20:c1:f1:f2:fa:70:86:3a:e2:39:86:b1:01:2f:61 root@archlinux-packer (ED25519)
 3072 MD5:b6:14:30:bd:fe:43:46:6a:20:a2:8b:b0:aa:d4:35:19 root@archlinux-packer (RSA)
 
+# mailman3.archlinux.org
+1024 SHA256:uYhlq19YzcZ8PEModMv2Y65xsiq1H+mjdwZ8PtbPET8 root@archlinux-packer (DSA)
+256 SHA256:85YiWFreKiw2Pv/XaKTqs0J0VInFtyVahpDRx2O9/B4 root@archlinux-packer (ECDSA)
+256 SHA256:b0mcOvNMzGrekDDtx83ZB1p5kN0meFek7zz1LbkfeHM root@archlinux-packer (ED25519)
+3072 SHA256:5hC4XSzA+/CgpL6cLYt0UbHB4aUs/o0IPxSScZwoi4A root@archlinux-packer (RSA)
+
+1024 MD5:3b:20:ad:1e:65:d8:3a:2e:09:69:62:46:e6:d9:6a:3e root@archlinux-packer (DSA)
+256 MD5:8d:ee:10:9b:05:56:b3:c7:4a:de:00:ad:95:c1:95:fa root@archlinux-packer (ECDSA)
+256 MD5:25:a8:b9:3c:fe:74:e7:7f:39:03:8e:23:dc:20:eb:bf root@archlinux-packer (ED25519)
+3072 MD5:20:a0:74:13:bd:97:59:11:75:a4:67:28:92:c3:40:35 root@archlinux-packer (RSA)
+
 # man.archlinux.org
 1024 SHA256:11C7Qa1GSNBBspSlber3Sp+LEMRpfr/VWkypfu6OnhA root@archlinux-packer (DSA)
 256 SHA256:fL79NVaEiwXGfUhTXWLkue/D1seSADYbui+jwQ2dvW0 root@archlinux-packer (ECDSA)

docs/ssh-known_hosts.txt

@@ -78,6 +78,11 @@ mail.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzd
 mail.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICTOoGxsf23f6AjIHcQQuvbTOaeIt48Y0PiBj9qlJi1H
 mail.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPrURadxte8UJiteGa6+Q+OjTAjhvGAQFkNSXj1pr4k03uxkU6l2v2LuTygk+4SZSCyUsKvNx/ljJeHBnuecQ8rRv19ZFqy/GQKB3oEmiNYMo2dYYlJWwTVBHatmghhB1j2y40yqdKWH2xQuXC3HtnS7fHG0g1Rc4R9KB4MQlcXkwnSEMpwpWBoO7sr0M4YTdwE+nSG9aNfyPbPGp3mX4ATz5X5hPJOlSFVDV6NuKrA+5qyt4jSKdeG5IuWeEnEJesYJEvShYdY9DvMCXnZykB0emzzk+5+Cp2lTPf9LOO3wNsTgHV/CwkoAoMgr9+ASefhBr3nxmmrs9T7nwuobGCGFUqQ2D8IKCmsWGVKXYERViz3x/gYUIlHgVJpoIXCFFqbdpWwxKR1aDMug2fFe699/FzuPdqrWPFdQMF2mPQ0w3AH/62KGp+PULE2HxrlCiY/gF2m8iJLgunxVKmi/c0ufgK9QilnKcPO+W4tcISa5MYt7MSTTLV9eVsgVjGhOU=
 
+# mailman3.archlinux.org
+mailman3.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFIHctq5/hKXaU//Jkzifp71ePIzcxdlxE5SZz1e7AcNp0Cci9W8A8NPtP6DMUvv4ezdKp+A/Czcy49tQolI30s=
+mailman3.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL0FZBrH2DQQoGn85t+2PN8t8FmUst9PsEsmGekfFAc+
+mailman3.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqZnMXg8rPkeKh3w9U/rNMc9NXjLu5K7Azfs0o/rIm3Hm2thLJH4i6hVGpSIfCw77WzoW0kotAEMkX3WaO/MXqp7qWiEv5Xy/Y9p2YNYdCdkhDdDhmuHSlYT+acnSHVPSjdN3SaaxXoN1/GAb5jlRMItZncind3gU0G6Kpls6N9tHsbjNqOG9Yv6hAlYfyVn4Q20H1ABLCJSUgWNOXq5DrqOLddblajbg4gQ8u6tYSpKDkPyEA8vmOueoTcwzB7k+aZRJmR9CVh5tsYc6QyBlljkFY8xhwLxVRMAe7dPND7LsTbfO20zVlZ94rPkTN46lb9ekrVwlzLlFd0nnJ2iWFiBU9HKelsxj5r8OorZfRkpXa8QVW04smQxycgHutEmsDGob5OFYkAPrdbuE0vA8KOk02D8NWhFHcG2nR/zMohNEJQsoTIHx7xeagFrC4Xgn7M+nhCVIIpbAOkRFjHAr4j2fGHjLzxV3EPkLN08aIcK53uHZTo2YSHL+JoU936o8=
+
 # man.archlinux.org
 man.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPhnsStoFw6rbVpE1S1vsXNk8de1SyMag1C+v0DWVSuNYzTylYg4322WbYzw45z2XhxrF6XmCSDMvgxvFwnfLQA=
 man.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHzjkN+igIxSIv5N9+ANNoo6knPa51Tj5TAXs4EQ8lY2
@@ -168,11 +173,9 @@ wiki.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzd
 wiki.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILFxxvRi7khrt6mUQGiXX35O1MBrrDeEmvaAnWo9ql/7
 wiki.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZQmj2D2B66bBHzze7+LEITju0rmtJT2rYsV/1GGpEPg6q6GAkUnIgRpyYxRn+UKRO9akDFXLv7W02h86cfbxOjeHufGMy/Y7NCPl2OSP9VauavJ3c3v/n80nmntU/Ji/U/p/roP0z+/OPgdWymFm0n33cl+XhmNOUumYQ7Y3z7EzrvCFZo2gt1EYChXb5Pd32rkd9tiwr3O0/M7TEiUxODzoD1dum+TJafUttC20V/4Sj8HztPx2BzhRugXfeEDbVlYvMXMMYgxbbhXLZyuE/dCaYpRvuekouAob7voIRSUaNqFXBLcGgo0udRI0mnKLgTb5bMprrRZ1zXIBU77H3gWyfHqe2I20arhXivtsHLEOZi0hc2/ni15WqkNS8G23n/+hJ+H16bjVb6t+8opErY4mL8T+F6OkxmNo8d0ztwUdxHEa+fvNPQ8UO6W4CN6kNVB9JE4f8j9FeHQq8rtlzo0wjUof4D7PhDn2WYA1l9RDiuRUxlGS4waStmttM3dE=
 
-# prio.ch-s012.rsync.net
-prio.ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3
-
+# zh1905.rsync.net
+zh1905.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJtclizeBy1Uo3D86HpgD3LONGVH0CJ0NT+YfZlldAJd
 
 # u236610.your-storagebox.de
 [u236610.your-storagebox.de]:23,[2a01:4f8:b16:3000::68]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs
 
-

group_vars/all/vault_rsync.net.yml

 $ANSIBLE_VAULT;1.1;AES256
-35646362653234383130323436333666656264303737633935336264313539353030376261313636
-3339346461323430653162643938303562666664386235660a333561373038373839653430643666
-35633566393330346136646664343065663039386135363461646136303435313430333561313833
-3630313034303638640a646364373062326464653937313430393332643335633166666663376630
-65383530363163303064336235633831353666323536376636616530363539346261333435326635
-38663061643961633536633165646534613933383336393463393233323339306139653462653566
-33653632633733633432393538356461663963366638653937636237616564326365336464343665
-35313237333636646538353665393437373536383161623833653638356133356131376165653238
-3332
+61636661646538643333653838373262333039643437666165333332663337373733363135333639
+6233383866323934306362373036363836623432353363380a393039626130633562646165636635
+63616463616233313135336430343961656333613530633161313365613434306361316564666535
+3361353438326434330a613361633764393833383364303664646535346462386437333437393263
+34633734313762376564386364636131313233376165626533396332303665323131616339383432
+61306439363730356337363266646662333437376133636434313365373839636263326264343439
+39626433376462623532663632336234646339623237366133623230613430356435323030326138
+34643861396563346230626332313835616337346536373463393432316430656464396534393233
+34656634386337356565333634346664323339643466313337333030623939616364626433353834
+38383739646433633666633936393234633038376535366137346363383830396266316164313765
+34633839343734383533393165613234383635633062356166633038396635336332363832363063
+63626666656332646438

host_vars/prio.ch-s012.rsync.net

----
-ansible_ssh_user: "{{ rsync_net_username }}"
-known_host: "prio.ch-s012.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO5lfML3qjBiDXi4yh3xPoXPHqIOeLNp66P3Unrl+8g3"

host_vars/zh1905.rsync.net

+---
+ansible_ssh_user: "{{ rsync_net_username }}"
+known_host: "zh1905.rsync.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJtclizeBy1Uo3D86HpgD3LONGVH0CJ0NT+YfZlldAJd"

hosts

@@ -3,7 +3,7 @@ secure-runner1.archlinux.org
 gemini.archlinux.org
 
 [rsync_net]
-prio.ch-s012.rsync.net
+zh1905.rsync.net
 
 [hetzner_storageboxes]
 u236610.your-storagebox.de
@@ -47,7 +47,7 @@ md.archlinux.org
 lists.archlinux.org
 
 [borg_hosts]
-prio.ch-s012.rsync.net
+zh1905.rsync.net
 u236610.your-storagebox.de
 
 [public_html]

misc/borg.sh

@@ -3,13 +3,19 @@
 set -eu
 shopt -s extglob
 
-OFFSITE_HOST=ch-s012.rsync.net
+OFFSITE_HOST=rsync.net
 ROOT_DIR=$(dirname "${0}")/..
 
 decrypted_gpg=$(mktemp arch-infrastructure-borg-XXXXXXXXX)
 trap "rm -f \"${decrypted_gpg}\"" EXIT
 [[ "$*" =~ $OFFSITE_HOST ]] && is_offsite=true || is_offsite=false
 
+# Use borg1 as the borg executable on offsite
+remote_path=borg
+if $is_offsite; then
+    remote_path=borg1
+fi
+
 # Find matching key
 matching_key=""
 for gpgkey in "$ROOT_DIR"/borg-keys/!(*-offsite.gpg); do
@@ -28,6 +34,6 @@ if [[ -z "$matching_key" ]]; then
 fi
 gpg --batch --yes --decrypt -aq --output "$decrypted_gpg" "$ROOT_DIR/borg-keys/$matching_key.gpg"
 
-BORG_KEY_FILE="$decrypted_gpg" borg "$@"
+BORG_KEY_FILE="$decrypted_gpg" borg --remote-path=$remote_path "$@"
 
 rm "$decrypted_gpg"

playbooks/gemini.archlinux.org.yml

@@ -5,7 +5,7 @@
   remote_user: root
   vars:
     archweb_db_host: "{{ hostvars['archlinux.org']['wireguard_address'] }}"
-    dbscripts_commit: '20191022'
+    dbscripts_commit: '20211026'
   roles:
     - { role: common }
     - { role: tools }

playbooks/rsync.net.yml

 ---
 
 - name: setup rsync.net account
-  hosts: prio.ch-s012.rsync.net
+  hosts: zh1905.rsync.net
   gather_facts: false
   roles:
     - { role: rsync_net, backup_dir: "backup", backup_clients: "{{ groups['borg_clients'] }}", tags: ["borg"] }

playbooks/tasks/sync-ssh-hostkeys.yml

@@ -34,12 +34,12 @@
     - name: manually append rsync.net host keys
       lineinfile:
         path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
+        line: "{% for host in query('inventory_hostnames', 'rsync_net') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n{% endfor %}"
       delegate_to: localhost
     - name: manually append Hetzner Storageboxes host keys
       lineinfile:
         path: "{{ playbook_dir }}/../../docs/ssh-known_hosts.txt"
-        line: "{% for host in query('inventory_hostnames', 'hetzner_storageboxes') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n\n{% endfor %}"
+        line: "{% for host in query('inventory_hostnames', 'hetzner_storageboxes') | sort %}# {{ host }}\n{{ hostvars[host].known_host }}\n{% endfor %}"
       delegate_to: localhost
 
 - name: upload known_hosts to all nodes

roles/archbuild/files/gitpkg

@@ -426,7 +426,7 @@ if gitify
     package: case
     when meson
       <<~END
-        DESTDIR="$pkgdir" meson install -C build
+        meson install -C build --destdir "$pkgdir"
       END
     end,
   }.reverse_each do |name, content|

roles/archweb/defaults/main.yml

@@ -13,7 +13,7 @@ archweb_domains_templates:
 archweb_allowed_hosts: ["{{ archweb_domain }}", 'ipxe.archlinux.org']
 archweb_nginx_conf: '/etc/nginx/nginx.d/archweb.conf'
 archweb_repository: 'https://github.com/archlinux/archweb.git'
-archweb_version: '45bf03a24ce72532477f21c0c6e603354b71a27c'
+archweb_version: 'release_2021-10-16'
 archweb_pgp_key: ['E499C79F53C96A54E572FEE1C06086337C50773E']
 archweb_site: true
 archweb_mirrorcheck: false

roles/aurweb/tasks/main.yml

@@ -17,6 +17,7 @@
       - python-fastapi
       - python-jinja
       - python-email-validator
+      - python-orjson
       - sudo
       - uwsgi-plugin-cgi
 

roles/aurweb/templates/aurweb-mkpkglists.service.j2

@@ -6,7 +6,7 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-mkpkglists
+ExecStart=/usr/local/bin/aurweb-mkpkglists --extended
 
 NoNewPrivileges=true
 LockPersonality=true

roles/aurweb/templates/config.j2

@@ -41,5 +41,7 @@ server = https://mirror.pkgbuild.com/%s/os/x86_64
 
 [mkpkglists]
 packagesfile = {{ aurweb_dir }}/web/html/packages.gz
+packagesmetafile = {{ aurweb_dir }}/web/html/packages-meta-v1.json.gz
+packagesmetaextfile = {{ aurweb_dir }}/web/html/packages-meta-ext-v1.json.gz
 pkgbasefile = {{ aurweb_dir }}/web/html/pkgbase.gz
 userfile = {{ aurweb_dir }}/web/html/users.gz

roles/borg_client/defaults/main.yml

@@ -3,6 +3,8 @@ backup_hosts:
   - host: "ssh://u236610.your-storagebox.de:23"
     dir: "~/repo"
     suffix: ""
-  - host: "ssh://{{ rsync_net_username }}@prio.ch-s012.rsync.net:22"
+    borg_cmd: "borg"
+  - host: "ssh://{{ rsync_net_username }}@zh1905.rsync.net:22"
     dir: "~/backup/{{ inventory_hostname }}"
     suffix: "-offsite"
+    borg_cmd: "borg --remote-path=borg1"

roles/borg_client/tasks/main.yml

@@ -3,7 +3,7 @@
   pacman: name=borg state=present
 
 - name: check if borg repository already exists
-  command: borg list {{ item['host'] }}/{{ item['dir'] }}
+  command: "{{ item['borg_cmd'] }} list {{ item['host'] }}/{{ item['dir'] }}"
   environment:
     BORG_RELOCATED_REPO_ACCESS_IS_OK: "yes"
   register: borg_list
@@ -12,7 +12,7 @@
   changed_when: borg_list.stdout | length > 0
 
 - name: init borg repository
-  command: borg init -e keyfile {{ item['host'] }}/{{ item['dir'] }}
+  command: "{{ item['borg_cmd'] }} init -e keyfile {{ item['host'] }}/{{ item['dir'] }}"
   when: borg_list is failed
   environment:
     BORG_PASSPHRASE: ""