roles/borg_client/templates/backup-postgres.sh.j2

@@ -3,16 +3,17 @@
 # Script to backup all postgres databases individually
 #
 # Requires local login with `postgres` user and either trusted or peer auth.
-#
+
+set -e
 
 DBLIST=($(sudo -u postgres psql -d postgres -qt -c 'SELECT datname from pg_database'))
 for db in "${DBLIST[@]}"; do
         if [[ $db =~ template[01] ]]; then
                 continue;
         fi
-        echo "Dumping $db to {{ postgres_backup_dir }}";
-        sudo -u postgres pg_dump --serializable-deferrable -Fc "$db" > "{{ postgres_backup_dir }}/$db.dump"
+        echo "Dumping $db to {{ backup_postgres_dir }}";
+        sudo -u postgres pg_dump --serializable-deferrable -Fc "$db" > "{{ backup_postgres_dir }}/$db.dump"
 done
 
-echo "Dumping globals to {{ postgres_backup_dir }}"
-sudo -u postgres pg_dumpall --globals-only > "{{ postgres_backup_dir }}/globals.sql.dump"
+echo "Dumping globals to {{ backup_postgres_dir }}"
+sudo -u postgres pg_dumpall --globals-only > "{{ backup_postgres_dir }}/globals.sql.dump"

roles/borg_client/templates/borg-backup-cpu-quota.j2

-{% if ansible_processor_vcpus == 1 %}
-{# Limit CPU usage to avoid triggering a HostHighCpuLoad alert #}
-CPUQuota=50%
-{% endif %}

roles/borg_client/templates/borg-backup.sh.j2

@@ -40,9 +40,11 @@ cleanup() {
 trap cleanup EXIT
 
 # Dump databases to /root/backup-{postgres,mysql} before taking a btrfs snapshot
-systemctl is-active postgresql && /usr/local/bin/backup-postgres.sh || true
-if systemctl is-active mysqld || systemctl is-active mariadb; then
-    /usr/local/bin/backup-mysql.sh || true
+if systemctl is-active postgresql; then
+    /usr/local/bin/backup-postgres.sh
+fi
+if systemctl is-active mariadb; then
+    /usr/local/bin/backup-mysql.sh
 fi
 
 if is_btrfs "$src"; then

roles/bugbuddy/files/bugbuddy-download.sh

@@ -20,9 +20,11 @@ readonly NAME=bugbuddy
 readonly PROJECT_ID="archlinux%2F${NAME}"
 readonly TRUSTED_UIDs=(
 	anthraxx@archlinux.org
+	gromit@archlinux.org
 )
 readonly TRUSTED_KEYS=(
 	E240B57E2C4630BA768E2F26FC1B547C8D8172C8
+	F00B96D15228013FFC9C9D0393B11DAA4C197E3D
 )
 
 readonly CURRENT_RELEASE="/root/${NAME}-current_release"
@@ -60,7 +62,7 @@ for link in "${LINKS[@]}"; do
 done
 
 for uid in "${TRUSTED_UIDs[@]}"; do
-	sq wkd get "${uid}"
+	sq network wkd fetch "${uid}"
 done
 
 for fp in "${TRUSTED_KEYS[@]}"; do

roles/certbot/files/certbot-renewal.service

@@ -3,8 +3,8 @@ Description=Let's Encrypt renewal
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/certbot renew --rsa-key-size 4096 \
-    --no-random-sleep-on-renew \
-    --pre-hook   "/etc/letsencrypt/hook.sh pre"      \
-    --post-hook  "/etc/letsencrypt/hook.sh post"     \
+ExecStart=/usr/bin/certbot renew                  \
+    --no-random-sleep-on-renew                    \
+    --pre-hook   "/etc/letsencrypt/hook.sh pre"   \
+    --post-hook  "/etc/letsencrypt/hook.sh post"  \
     --renew-hook "/etc/letsencrypt/hook.sh renew"

roles/certificate/defaults/main.yml

 certificate_challenge: "HTTP-01"
 certificate_contact_email: "webmaster@archlinux.org"
-certificate_rsa_key_size: 4096

roles/certificate/tasks/main.yml

-- name: Create ssl cert (HTTP-01)
+- name: Create ssl cert (HTTP-01) named {{ cert_name | default(domains | first) }}
   shell: |
     set -o pipefail
     # We can't start nginx without the certificate and we can't issue a certificate without nginx running.
     # So use Python built-in http.server for the initial certificate issuance
     python -m http.server --directory {{ letsencrypt_validation_dir }} 80 &
     trap "jobs -p | xargs --no-run-if-empty kill" EXIT
-    certbot certonly --email {{ certificate_contact_email }} --agree-tos --rsa-key-size {{ certificate_rsa_key_size }} --renew-by-default --webroot -w {{ letsencrypt_validation_dir }} -d {{ domains | join(' -d ') }}
+    certbot certonly --email {{ certificate_contact_email }} --agree-tos --key-type {{ 'ecdsa' if not (legacy | default(false)) else 'rsa --rsa-key-size 4096' }} --renew-by-default --webroot -w {{ letsencrypt_validation_dir }} -d {{ domains | join(' -d ') }} --cert-name {{ cert_name | default(domains | first) }}
   args:
-    creates: '/etc/letsencrypt/live/{{ domains | first }}/fullchain.pem'
+    creates: '/etc/letsencrypt/live/{{ cert_name | default(domains | first) }}/fullchain.pem'
   when: challenge | default(certificate_challenge) == "HTTP-01"
 
-- name: Create ssl cert (DNS-01)
-  command: certbot certonly --email {{ certificate_contact_email }} --agree-tos --rsa-key-size {{ certificate_rsa_key_size }} --renew-by-default --dns-rfc2136 --dns-rfc2136-credentials /etc/letsencrypt/rfc2136.ini -d {{ domains | join(' -d ') }}
+- name: Create ssl cert (DNS-01) named {{ cert_name | default(domains | first) }}
+  command: certbot certonly --email {{ certificate_contact_email }} --agree-tos --key-type {{ 'ecdsa' if not (legacy | default(false)) else 'rsa --rsa-key-size 4096' }} --renew-by-default --dns-rfc2136 --dns-rfc2136-credentials /etc/letsencrypt/rfc2136.ini -d {{ domains | join(' -d ') }} --cert-name {{ cert_name | default(domains | first) }}
   args:
-    creates: '/etc/letsencrypt/live/{{ domains | first }}/fullchain.pem'
+    creates: '/etc/letsencrypt/live/{{ cert_name | default(domains | first) }}/fullchain.pem'
   when: challenge | default(certificate_challenge) == "DNS-01"

roles/common/tasks/main.yml

@@ -153,8 +153,8 @@
     - zshrc
     - dircolors
 
-- name: Install pacman-contrib,archlinux-contrib
-  pacman: name=pacman-contrib,archlinux-contrib state=installed
+- name: Install pacman-contrib and archlinux-contrib
+  pacman: name=pacman-contrib,fakeroot,archlinux-contrib state=installed
 
 - name: Install custom paccache.service
   copy: src=paccache.service dest=/etc/systemd/system/paccache.service owner=root group=root mode=0644

roles/common/templates/pacman.conf.j2

@@ -38,6 +38,8 @@ Color
 CheckSpace
 VerbosePkgLists
 ParallelDownloads = 5
+DownloadUser = alpm
+#DisableSandbox
 
 # By default, pacman accepts packages signed by keys that its local keyring
 # trusts (see pacman-key and its man page), as well as unsigned packages.

roles/dbscripts/defaults/main.yml

-dbscripts_commit: 7e5df6a5adb4b9c4531568a5113020daad3af5b0
+dbscripts_commit: cde46716a8cde5424297c9ed350d43ad26e8b1f1
 dbscripts_update: true
 dbscripts_pgp_emails: ['jelle@archlinux.org', 'anthraxx@archlinux.org']
 

roles/dbscripts/files/lastsync.service

@@ -3,5 +3,5 @@ Description=lastsync
 
 [Service]
 Type=oneshot
-User=ftp
-ExecStart=/usr/bin/bash -c "date +%%s > /srv/ftp/lastsync"
+ExecStart=/usr/bin/bash -c "date +%%s > /srv/ftp/lastsync.tmp && \
+    mv /srv/ftp/lastsync.tmp /srv/ftp/lastsync"

roles/dbscripts/files/rsyncd-tmpfiles.d

-D /run/rsyncd 0700 nobody nobody -

roles/dbscripts/tasks/main.yml

@@ -28,7 +28,7 @@
   user: name=sourceballs shell=/sbin/nologin
 
 - name: Set up sudoers.d for special users
-  copy: src=sudoers.d dest=/etc/sudoers.d/dbscripts owner=root group=root mode=0600
+  copy: src=sudoers.d dest=/etc/sudoers.d/dbscripts owner=root group=root mode=0600 validate='visudo -cf %s'
 
 - name: Create ssl cert
   include_role:
@@ -210,14 +210,6 @@
 - name: Make /srv/ftp/other/packages available
   file: path=/srv/ftp/other/packages state=directory owner=root group=junior-dev mode=0775
 
-- name: Put rsyncd.conf into tmpfiles
-  copy: src=rsyncd-tmpfiles.d dest=/etc/tmpfiles.d/rsyncd.conf owner=root group=root mode=0644
-  register: rsyncdtmpfiles
-
-- name: Use tmpfiles.d/rsyncd.conf
-  command: systemd-tmpfiles --create creates=/run/rsyncd
-  when: rsyncdtmpfiles.changed
-
 - name: Create rsyncd-conf-genscripts
   file: path=/etc/rsyncd-conf-genscripts state=directory owner=root group=root mode=0700
 

roles/dbscripts/templates/nginx.d.conf.j2

@@ -3,8 +3,7 @@ proxy_cache_path  /var/lib/nginx/cache levels=1:2 keys_zone=auth_cache:5m inacti
 server {
     listen       80;
     listen       [::]:80;
-    listen       443 ssl http2;
-    listen       [::]:443 ssl http2;
+    include      snippets/listen-443.conf;
     server_name  {{ repos_domain }} {{repos_rsync_domain}};
     root         /srv/ftp;
 
@@ -46,14 +45,16 @@ server {
     }
 
     location = /devel/mirrorauth/ {
+        # Authentication to archweb
         internal;
+        proxy_pass https://archlinux.org;
 
         # Do not pass the request body, only http authorisation header is required
         proxy_pass_request_body off;
         proxy_set_header        Content-Length "";
 
         # Proxy headers
-        proxy_set_header        Host                    "archlinux.org";
+        proxy_set_header        Host                    $proxy_host;
         proxy_set_header        X-Sent-From             "{{ vault_archweb_x_sent_from_secret }}";
 
         # Cache responses from the auth proxy
@@ -61,12 +62,16 @@ server {
         proxy_cache_key         $scheme$proxy_host$uri$http_authorization;
 
         # Minimize the number of requests to archweb
-        proxy_cache_lock        on;
-        proxy_cache_use_stale   updating;
+        proxy_cache_lock              on;
+        proxy_cache_use_stale         updating;
+        proxy_cache_background_update on;
 
-        # Authentication to archweb
-        proxy_pass https://archlinux.org;
-        proxy_ssl_verify on;
+        # Verify destination TLS cert
+        proxy_ssl_verify              on;
         proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
+        # Send SNI with TLS handshake
+        proxy_ssl_server_name         on;
+        proxy_ssl_name                $proxy_host;
     }
 }

roles/dbscripts/templates/rsyncd.conf.proto.j2

@@ -4,7 +4,6 @@
 
 use chroot = no
 max connections = 12
-lock file = /var/run/rsyncd/main.lock
 syslog facility = local5
 pid file = /var/run/rsyncd.pid
 #transfer logging = yes
@@ -31,19 +30,19 @@ hosts deny = *
 [ftp]
 	path = /srv/ftp
 	comment = ftp area (most mirrors should use this)
-	exclude = /archive/ /other/ /sources/ /*-debug/ /pool/*-debug/
+	exclude = /archive/ /other/ /sources/ /*-debug/ /pool/*-debug/ /lastsync.tmp
 
 [ftp_tier1]
 	path = /srv/ftp
 	comment = ftp area (most mirrors should use this)
-	exclude = /archive/ /other/ /sources/ /*-debug/ /pool/*-debug/
+	exclude = /archive/ /other/ /sources/ /*-debug/ /pool/*-debug/ /lastsync.tmp
 	hosts allow = @@ALLOWHOSTS_TIER1@@
 	max connections = 0
 
 [ftp_auth]
 	path = /srv/ftp
 	comment = ftp area, passworded (same as 'ftp')
-	exclude = /archive/ /other/ /sources/ /*-debug/ /pool/*-debug/
+	exclude = /archive/ /other/ /sources/ /*-debug/ /pool/*-debug/ /lastsync.tmp
 	hosts allow = *
 	auth users = *
 	secrets file = /etc/rsyncd.secrets
@@ -53,19 +52,19 @@ hosts deny = *
 [ftpfull]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions, except sources)
-	exclude = /sources/
+	exclude = /sources/ /lastsync.tmp
 
 [ftpfull_tier1]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions, except sources)
-	exclude = /sources/
+	exclude = /sources/ /lastsync.tmp
 	hosts allow = @@ALLOWHOSTS_TIER1@@
 	max connections = 0
 
 [ftpfull_auth]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions, except sources)
-	exclude = /sources/
+	exclude = /sources/ /lastsync.tmp
 	hosts allow = *
 	auth users = *
 	secrets file = /etc/rsyncd.secrets
@@ -75,17 +74,20 @@ hosts deny = *
 [kitchensink]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions)
+	exclude = /lastsync.tmp
 	hosts allow = {{ hostvars['archlinux.org']['ipv4_address'] }} {{ hostvars['archlinux.org']['ipv6_address'] }}
 
 [kitchensink_tier1]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions)
-	hosts allow = @@ALLOWHOSTS_TIER1@@ {{ hostvars['gemini.archlinux.org']['ipv4_address'] }} {{ hostvars['gemini.archlinux.org']['ipv6_address'] }}
+	exclude = /lastsync.tmp
+	hosts allow = @@ALLOWHOSTS_TIER1@@ {{ hostvars['gemini.archlinux.org']['ipv4_address'] }} {{ hostvars['gemini.archlinux.org']['ipv6_address'] }} {{ hostvars['repos.archlinux.org']['ipv4_address'] }} {{ hostvars['repos.archlinux.org']['ipv6_address'] }}
 	max connections = 0
 
 [kitchensink_auth]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions)
+	exclude = /lastsync.tmp
 	hosts allow = *
 	auth users = *
 	secrets file = /etc/rsyncd.secrets

roles/debuginfod/templates/nginx.d.conf.j2

@@ -16,8 +16,7 @@ server {
 }
 
 server {
-    listen       443 ssl http2;
-    listen       [::]:443 ssl http2;
+    include      snippets/listen-443.conf;
     server_name  {{ debuginfod_domain }};
 
     access_log   /var/log/nginx/{{ debuginfod_domain }}/access.log reduced;

roles/dovecot/templates/dovecot.conf.j2

@@ -10,8 +10,8 @@ namespace inbox {
 	hidden = no
 	inbox = yes
 	list = yes
-	location = 
-	prefix = 
+	location =
+	prefix =
 	separator = .
 	type = private
 
@@ -44,7 +44,7 @@ plugin {
 	sieve = ~/.dovecot.sieve
 	sieve_dir = ~/.sieve
 	# fileinto and mailbox is required by the global spam-to-folder.sieve script
-	sieve_extensions = duplicate envelope fileinto imap4flags mailbox regex variables
+	sieve_extensions = body duplicate envelope fileinto imap4flags mailbox regex variables
 	sieve_global_dir = /etc/dovecot/sieve/global/
 	sieve_global_path = /etc/dovecot/sieve/default.sieve
 	sieve_before = /etc/dovecot/sieve/spam-to-folder.sieve

roles/fail2ban/tasks/main.yml

@@ -34,7 +34,7 @@
   notify:
     - Restart fail2ban
 
-- name: Install firewallcmd-allports.local
+- name: Install firewallcmd-ipset-allports.conf
   template:
     src: "firewallcmd-ipset-allports.conf.j2"
     dest: "/etc/fail2ban/action.d/firewallcmd-ipset-allports.conf"

roles/fail2ban/templates/jail.local.j2

@@ -13,7 +13,7 @@ usedns = no
 # if f2b ever needs to send emails, send them to root and make sure the sender
 # address clearly identifies the host the message originated from
 destemail = root
-sender = fail2ban@{{ansible_fqdn}}
+sender = fail2ban@{{inventory_hostname}}
 
 # use firewalld to manage bans - if we don't specify this, then fail2ban will
 # default to use iptables, which we don't want as our systems are running

roles/firewalld/tasks/main.yml

@@ -19,4 +19,5 @@
     service: dhcpv6-client
     state: disabled
     immediate: true
+    permanent: true
   when: configure_firewall