group_vars/geo_mirrors.yml

+---
+certbot_dns_support: true

host_vars/mirror.pkgbuild.com/misc

 ---
 mirror_domain: mirror.pkgbuild.com
 mirror_debug_packages: false
+geomirror_acme_challenge: true
 archweb_mirrorcheck_locations: [20, 21]
 filesystem: btrfs
 

playbooks/mirrors.yml

@@ -6,7 +6,7 @@
     - { role: common }
     - { role: sshd }
     - { role: root_ssh }
-    - { role: certbot, certbot_dns_support: true }
+    - { role: certbot }
     - { role: nginx }
     - { role: syncrepo, tags: ['nginx'] }
     - { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
@@ -15,4 +15,4 @@
     - { role: promtail }
     - { role: fail2ban }
     - { role: wireguard }
-    - { role: geomirror, when: inventory_hostname == "mirror.pkgbuild.com" }
+    - { role: geomirror, when: "inventory_hostname == 'mirror.pkgbuild.com' or 'geo_mirrors' in group_names" }

roles/archweb/defaults/main.yml

@@ -13,7 +13,7 @@ archweb_domains_templates:
 archweb_allowed_hosts: ["{{ archweb_domain }}", 'ipxe.archlinux.org']
 archweb_nginx_conf: '/etc/nginx/nginx.d/archweb.conf'
 archweb_repository: 'https://github.com/archlinux/archweb.git'
-archweb_version: 'release_2022-04-12'
+archweb_version: 'release_2022-04-14'
 archweb_pgp_key: ['E499C79F53C96A54E572FEE1C06086337C50773E']
 archweb_site: true
 archweb_mirrorcheck: false

roles/geoipupdate/tasks/main.yml

@@ -2,9 +2,11 @@
 
 - name: install geoipupdate
   pacman: name=geoipupdate state=present
+  register: installation
 
 - name: configure geoipupdate
   template: src=GeoIP.conf.j2 dest=/etc/GeoIP.conf owner=root group=root mode=0600
+  register: configuration
 
 - name: create drop-in directory for geoipupdate.service
   file: path=/etc/systemd/system/geoipupdate.service.d state=directory owner=root group=root mode=0755
@@ -14,5 +16,9 @@
   notify:
     - daemon reload
 
+- name: run geoipupdate after installation or configuration change
+  systemd: name=geoipupdate state=restarted
+  when: installation is changed or configuration is changed
+
 - name: start and enable geoipupdate.timer
   systemd: name=geoipupdate.timer enabled=yes state=started

roles/geomirror/defaults/main.yml

+---
+geomirror_acme_challenge: false

roles/geomirror/tasks/main.yml

@@ -12,6 +12,7 @@
 
 - name: create directory for sqlite3 dbs
   file: path=/var/lib/powerdns state=directory owner=powerdns group=powerdns mode=0755
+  when: geomirror_acme_challenge
 
 - name: initialize sqlite3 database for _acme-challenge zone
   command: sqlite3 -init /usr/share/doc/powerdns/schema.sqlite3.sql /var/lib/powerdns/pdns.sqlite3 ""
@@ -20,6 +21,7 @@
   args:
     creates: /var/lib/powerdns/pdns.sqlite3
   register: init
+  when: geomirror_acme_challenge
 
 - name: create _acme-challenge zone
   command: "{{ item }}"
@@ -33,6 +35,7 @@
 - name: import TSIG key (for certbot)
   command: pdnsutil import-tsig-key {{ certbot_rfc2136_key }} {{ certbot_rfc2136_algorithm }} {{ certbot_rfc2136_secret }}
   changed_when: false
+  when: geomirror_acme_challenge
 
 - name: open powerdns ipv4 port for monitoring.archlinux.org
   ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
@@ -40,5 +43,8 @@
   tags:
     - firewall
 
+- name: open firewall hole
+  ansible.posix.firewalld: service=dns permanent=true state=enabled immediate=yes
+
 - name: start and enable powerdns
   systemd: name=pdns.service enabled=yes daemon_reload=yes state=started

roles/geomirror/templates/geo.yml.j2

@@ -7,11 +7,18 @@ domains:
       {{ geo_mirror_domain }}:
         - soa: mirror.pkgbuild.com. root.archlinux.org. 2022011501 3600 1800 604800 3600
         - ns: mirror.pkgbuild.com
+      {% for host in groups['geo_mirrors'] %}
+        - ns: {{ host }}
+      {% endfor %}
       {% for host in groups['geo_mirrors'] %}
       {{ host.split(".")[0] }}.{{ geo_mirror_domain }}:
         - a: {{ hostvars[host]['ipv4_address'] }}
         - aaaa: {{ hostvars[host]['ipv6_address'] }}
       {% endfor %}
+      {% if not geomirror_acme_challenge %}
+      _acme-challenge.{{ geo_mirror_domain }}:
+        - ns: mirror.pkgbuild.com
+      {% endif %}
     services:
       {{ geo_mirror_domain }}: '%mp.geo.mirror.pkgbuild.com'
 mapping_lookup_formats: ['%cn']

roles/geomirror/templates/pdns.conf.j2

@@ -4,9 +4,13 @@ local-address={{ ipv4_address }},{{ ipv6_address }}
 webserver=yes
 webserver-address=0.0.0.0
 webserver-allow-from=127.0.0.1,::1,{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
+{% if geomirror_acme_challenge %}
 launch=geoip,gsqlite3
-geoip-database-files=/var/lib/GeoIP/GeoLite2-Country.mmdb
-geoip-zones-file=/etc/powerdns/geo.yml
 gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
 dnsupdate=yes
 lua-dnsupdate-policy-script=/etc/powerdns/dnsupdate-policy.lua
+{% else %}
+launch=geoip
+{% endif %}
+geoip-database-files=/var/lib/GeoIP/GeoLite2-Country.mmdb
+geoip-zones-file=/etc/powerdns/geo.yml

roles/prometheus/templates/prometheus.yml.j2

@@ -77,9 +77,13 @@ scrape_configs:
 
   - job_name: 'powerdns'
     static_configs:
-    - targets: ['{{ hostvars['mirror.pkgbuild.com']['wireguard_address'] }}:8081']
+    {% for host in groups['geo_mirrors'] + ['mirror.pkgbuild.com'] %}
+
+    - targets: ['{{ hostvars[host]['wireguard_address'] }}:8081']
       labels:
-        instance: "mirror.pkgbuild.com"
+        instance: "{{ host }}"
+
+    {% endfor %}
 
   - job_name: 'gitlab_runner_exporter'
     static_configs:

roles/rebuilderd_worker/files/clean-repro

+#!/bin/bash -e
+
+# remove leftover chroots that are more than a week old
+find /var/lib/repro -maxdepth 1 -name '*?_?*' -mtime +6 -exec rm -rf {} +
+
+# clean the package cache but keep recently accessed files
+flock /var/lib/rebuilderd-worker/cache.lock \
+  paccache -r -q -c /var/lib/rebuilderd-worker/cache --min-atime '2 weeks ago'

roles/rebuilderd_worker/files/clean-repro.service

+[Unit]
+Description=Clean up rebuilderd-worker chroots and cache
+ConditionPathExists=/var/lib/repro
+ConditionPathExists=/var/lib/rebuilderd-worker/cache
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/clean-repro
+Nice=19
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7

roles/rebuilderd_worker/files/clean-repro.timer

+[Unit]
+Description=Daily rebuilderd-worker chroot and cache cleanup
+
+[Timer]
+OnCalendar=daily
+RandomizedDelaySec=12h
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/rebuilderd_worker/handlers/main.yml

+---
+
+- name: daemon reload
+  systemd:
+    daemon-reload: true

roles/rebuilderd_worker/tasks/main.yml

@@ -13,3 +13,17 @@
 - name: enable and start rebuilderd-worker@{{ item }}
   systemd: name=rebuilderd-worker@{{ item }} enabled=yes state=started
   with_items: '{{ rebuilderd_workers }}'
+
+- name: install cleanup script
+  copy: src=clean-repro dest=/usr/local/bin/clean-repro owner=root group=root mode=0755
+
+- name: install cleanup units
+  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
+  loop:
+    - clean-repro.timer
+    - clean-repro.service
+  notify:
+    - daemon reload
+
+- name: start and enable cleanup timer
+  service: name=clean-repro.timer enabled=yes state=started

tf-stage1/archlinux.tf

@@ -426,13 +426,34 @@ resource "hetznerdns_record" "pkgbuild_com_origin_txt" {
   type    = "TXT"
 }
 
-resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns" {
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns1" {
   zone_id = hetznerdns_zone.pkgbuild.id
   name    = "geo.mirror"
   value   = "mirror.pkgbuild.com."
   type    = "NS"
 }
 
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_n2" {
+  zone_id = hetznerdns_zone.pkgbuild.id
+  name    = "geo.mirror"
+  value   = "asia.mirror.pkgbuild.com."
+  type    = "NS"
+}
+
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns3" {
+  zone_id = hetznerdns_zone.pkgbuild.id
+  name    = "geo.mirror"
+  value   = "america.mirror.pkgbuild.com."
+  type    = "NS"
+}
+
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns4" {
+  zone_id = hetznerdns_zone.pkgbuild.id
+  name    = "geo.mirror"
+  value   = "europe.mirror.pkgbuild.com."
+  type    = "NS"
+}
+
 resource "hetznerdns_record" "archlinux_org_origin_caa" {
   zone_id = hetznerdns_zone.archlinux.id
   name    = "@"