Avoid single point-of-failure for our GeoIP domain

We don't want mirror.pkgbuild.com's DNS server to be a
single-point-of-failure, so this commit adds multiple authoritative DNS
servers for the zone. The extra DNS servers are run on the geomirror
servers.

The _acme-challenge zone, used for obtaining certificates, is run solely
on mirror.pkgbuild.com's DNS server, to avoid syncing DNS records
between the servers (KISS).

host_vars/mirror.pkgbuild.com/misc

 ---
 mirror_domain: mirror.pkgbuild.com
 mirror_debug_packages: false
+geomirror_acme_challenge: true
 archweb_mirrorcheck_locations: [20, 21]
 filesystem: btrfs
 

playbooks/mirrors.yml

@@ -15,4 +15,4 @@
     - { role: promtail }
     - { role: fail2ban }
     - { role: wireguard }
-    - { role: geomirror, when: inventory_hostname == "mirror.pkgbuild.com" }
+    - { role: geomirror, when: "inventory_hostname == 'mirror.pkgbuild.com' or 'geo_mirrors' in group_names" }

roles/geomirror/defaults/main.yml

+---
+geomirror_acme_challenge: false

roles/geomirror/tasks/main.yml

@@ -12,6 +12,7 @@
 
 - name: create directory for sqlite3 dbs
   file: path=/var/lib/powerdns state=directory owner=powerdns group=powerdns mode=0755
+  when: geomirror_acme_challenge
 
 - name: initialize sqlite3 database for _acme-challenge zone
   command: sqlite3 -init /usr/share/doc/powerdns/schema.sqlite3.sql /var/lib/powerdns/pdns.sqlite3 ""
@@ -20,6 +21,7 @@
   args:
     creates: /var/lib/powerdns/pdns.sqlite3
   register: init
+  when: geomirror_acme_challenge
 
 - name: create _acme-challenge zone
   command: "{{ item }}"
@@ -33,6 +35,7 @@
 - name: import TSIG key (for certbot)
   command: pdnsutil import-tsig-key {{ certbot_rfc2136_key }} {{ certbot_rfc2136_algorithm }} {{ certbot_rfc2136_secret }}
   changed_when: false
+  when: geomirror_acme_challenge
 
 - name: open powerdns ipv4 port for monitoring.archlinux.org
   ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes

roles/geomirror/templates/geo.yml.j2

@@ -7,11 +7,18 @@ domains:
       {{ geo_mirror_domain }}:
         - soa: mirror.pkgbuild.com. root.archlinux.org. 2022011501 3600 1800 604800 3600
         - ns: mirror.pkgbuild.com
+      {% for host in groups['geo_mirrors'] %}
+        - ns: {{ host }}
+      {% endfor %}
       {% for host in groups['geo_mirrors'] %}
       {{ host.split(".")[0] }}.{{ geo_mirror_domain }}:
         - a: {{ hostvars[host]['ipv4_address'] }}
         - aaaa: {{ hostvars[host]['ipv6_address'] }}
       {% endfor %}
+      {% if not geomirror_acme_challenge %}
+      _acme-challenge.{{ geo_mirror_domain }}:
+        - ns: mirror.pkgbuild.com
+      {% endif %}
     services:
       {{ geo_mirror_domain }}: '%mp.geo.mirror.pkgbuild.com'
 mapping_lookup_formats: ['%cn']

roles/geomirror/templates/pdns.conf.j2

@@ -4,9 +4,13 @@ local-address={{ ipv4_address }},{{ ipv6_address }}
 webserver=yes
 webserver-address=0.0.0.0
 webserver-allow-from=127.0.0.1,::1,{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
+{% if geomirror_acme_challenge %}
 launch=geoip,gsqlite3
-geoip-database-files=/var/lib/GeoIP/GeoLite2-Country.mmdb
-geoip-zones-file=/etc/powerdns/geo.yml
 gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
 dnsupdate=yes
 lua-dnsupdate-policy-script=/etc/powerdns/dnsupdate-policy.lua
+{% else %}
+launch=geoip
+{% endif %}
+geoip-database-files=/var/lib/GeoIP/GeoLite2-Country.mmdb
+geoip-zones-file=/etc/powerdns/geo.yml

roles/prometheus/templates/prometheus.yml.j2

@@ -77,9 +77,13 @@ scrape_configs:
 
   - job_name: 'powerdns'
     static_configs:
-    - targets: ['{{ hostvars['mirror.pkgbuild.com']['wireguard_address'] }}:8081']
+    {% for host in groups['geo_mirrors'] + ['mirror.pkgbuild.com'] %}
+
+    - targets: ['{{ hostvars[host]['wireguard_address'] }}:8081']
       labels:
-        instance: "mirror.pkgbuild.com"
+        instance: "{{ host }}"
+
+    {% endfor %}
 
   - job_name: 'gitlab_runner_exporter'
     static_configs:

tf-stage1/archlinux.tf

@@ -426,13 +426,34 @@ resource "hetznerdns_record" "pkgbuild_com_origin_txt" {
   type    = "TXT"
 }
 
-resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns" {
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns1" {
   zone_id = hetznerdns_zone.pkgbuild.id
   name    = "geo.mirror"
   value   = "mirror.pkgbuild.com."
   type    = "NS"
 }
 
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_n2" {
+  zone_id = hetznerdns_zone.pkgbuild.id
+  name    = "geo.mirror"
+  value   = "asia.mirror.pkgbuild.com."
+  type    = "NS"
+}
+
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns3" {
+  zone_id = hetznerdns_zone.pkgbuild.id
+  name    = "geo.mirror"
+  value   = "america.mirror.pkgbuild.com."
+  type    = "NS"
+}
+
+resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns4" {
+  zone_id = hetznerdns_zone.pkgbuild.id
+  name    = "geo.mirror"
+  value   = "europe.mirror.pkgbuild.com."
+  type    = "NS"
+}
+
 resource "hetznerdns_record" "archlinux_org_origin_caa" {
   zone_id = hetznerdns_zone.archlinux.id
   name    = "@"