Verified Commit 9f65f99c authored by Kristian Klausen's avatar Kristian Klausen 🎉
Browse files

Add GeoIP domain for our sponsored mirros

We had a GeoIP mirror in the past based on nginx and its GeoIP module,
but it didn't perform very well, due to the high latency (asking a
central server for the package and then redirected to the closest
mirror).

One of the reasons for offering this service, is so we can relieve
mirror.pkgbuild.com which is burning a ton of traffic (50TB/month),
likely due to it being the default mirror in our Docker image. Another
reason is so we can offer a link to our arch-boxes images in libosinfo
(used by gnome-boxes, virt-install and virt-manager), with good enough
performance for most users.

This time we take a different approach and use a DNS based solution,
which means the latency penalty is only paid once (the first DNS
request). The downside is that the mirrors must have a valid certificate
for the same domain name, which makes using third-party mirrors a
challenge. So for now, we are just using the sponsored mirorrs
controlled by the DevOps team.

Fix #101
parent 0e56211e
......@@ -85,6 +85,7 @@ So to set up this server from scratch, run:
### Services
- Regular mirror.
- Running a authoritative DNS server (PowerDNS) for our GeoIP mirror
## reproducible.archlinux.org
......
---
archweb_db_host: "{{ hostvars['archlinux.org']['wireguard_address'] }}"
geo_mirror_domain: "geo.mirror.pkgbuild.com"
# raise tcp window limits to 32MiB
tcp_rmem: "10240 87380 33554432"
......
$ANSIBLE_VAULT;1.1;AES256
38363230373165316233346438623933326461653833623532316232343134616230653033306266
3762316566373363313464396663323639373337393536350a666565373633333838306331343064
32363135346331646161313964396535316464333338613633303965326161623633373562393736
3462346437623862640a306664393064343836346330626663303739646337343466393735666330
33326335613266393538386439333532386337336231663363333137396335626461393633323038
38613465356435393561643132646165646639373466613731316666363337616239393865653932
62323837363731316130333664313531336435613566386436626165323864356533653765616338
38613430366230333033386333623266313265386562303138633339623963383466343335343931
39303136383430363066323764316466333437666433646666633630393862303139356438303037
33363466303433383864343465666533393263363835356439613466666330636138306130333063
32353939323339313437303631353838373238303836303261353834373962633262666138346434
65373836626230336365396338323466636437323864323930633631643734343939343831366166
38396264363230306564396661353534356662626130326230343562346532316466346331313936
30613937396635633532323461313739346463613830353437313662333636653233633930666664
30353639386663353930353031343735666530353835336230376134346464393566396535653339
30323166323036336433623564366163333330383764303166306463636561326533386133393063
38363934303132336239363833666533636238653263613339663638663334313839
......@@ -13,6 +13,11 @@ asia.mirror.pkgbuild.com
america.mirror.pkgbuild.com
europe.mirror.pkgbuild.com
[geo_mirrors]
asia.mirror.pkgbuild.com
america.mirror.pkgbuild.com
europe.mirror.pkgbuild.com
[archive_mirrors]
asia.mirror.pkgbuild.com
america.mirror.pkgbuild.com
......
......@@ -6,7 +6,7 @@
- { role: common }
- { role: sshd }
- { role: root_ssh }
- { role: certbot }
- { role: certbot, certbot_dns_support: true }
- { role: nginx }
- { role: syncrepo, tags: ['nginx'] }
- { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
......@@ -15,3 +15,4 @@
- { role: promtail }
- { role: fail2ban }
- { role: wireguard }
- { role: geomirror, when: inventory_hostname == "mirror.pkgbuild.com" }
---
certbot_dns_support: false
---
- name: install certbot
pacman: name=certbot state=present
pacman: name=certbot{{ ",certbot-dns-rfc2136" if certbot_dns_support }} state=present
- name: install rfc2136.ini
template: src=rfc2136.ini.j2 dest=/etc/letsencrypt/rfc2136.ini owner=root group=root mode=0600
when: certbot_dns_support
- name: install letsencrypt hook
copy: src=hook.sh dest=/etc/letsencrypt/hook.sh owner=root group=root mode=0755
......
dns_rfc2136_server = {{ certbot_rfc2136_server }}
dns_rfc2136_name = {{ certbot_rfc2136_key }}
dns_rfc2136_secret = {{ certbot_rfc2136_secret }}
dns_rfc2136_algorithm = {{ certbot_rfc2136_algorithm }}
certificate_challenge: "HTTP-01"
certificate_contact_email: "webmaster@archlinux.org"
certificate_rsa_key_size: 4096
- name: create ssl cert
- name: create ssl cert (HTTP-01)
shell: |
set -o pipefail
# We can't start nginx without the certificate and we can't issue a certificate without nginx running.
......@@ -8,3 +8,10 @@
certbot certonly --email {{ certificate_contact_email }} --agree-tos --rsa-key-size {{ certificate_rsa_key_size }} --renew-by-default --webroot -w {{ letsencrypt_validation_dir }} -d {{ domains | join(' -d ') }}
args:
creates: '/etc/letsencrypt/live/{{ domains | first }}/fullchain.pem'
when: challenge | default(certificate_challenge) == "HTTP-01"
- name: create ssl cert (DNS-01)
command: certbot certonly --email {{ certificate_contact_email }} --agree-tos --rsa-key-size {{ certificate_rsa_key_size }} --renew-by-default --dns-rfc2136 --dns-rfc2136-credentials /etc/letsencrypt/rfc2136.ini -d {{ domains | join(' -d ') }}
args:
creates: '/etc/letsencrypt/live/{{ domains | first }}/fullchain.pem'
when: challenge | default(certificate_challenge) == "DNS-01"
---
- name: restart powerdns
service: name=pdns state=restarted
---
dependencies:
- role: geoipupdate
---
- name: install powerdns and geoip
pacman: name=powerdns,libmaxminddb,geoip,yaml-cpp state=present
- name: install PowerDNS configuration
template: src={{ item.src }} dest=/etc/powerdns/{{ item.dest }} owner=root group=root mode=0644
loop:
- {src: pdns.conf.j2, dest: pdns.conf}
- {src: geo.yml.j2, dest: geo.yml}
- {src: dnsupdate-policy.lua.j2, dest: dnsupdate-policy.lua}
notify: restart powerdns
- name: create directory for sqlite3 dbs
file: path=/var/lib/powerdns state=directory owner=powerdns group=powerdns mode=0755
- name: initialize sqlite3 database for _acme-challenge zone
command: sqlite3 -init /usr/share/doc/powerdns/schema.sqlite3.sql /var/lib/powerdns/pdns.sqlite3 ""
become: true
become_user: powerdns
args:
creates: /var/lib/powerdns/pdns.sqlite3
register: init
- name: create _acme-challenge zone
command: "{{ item }}"
loop:
- pdnsutil create-zone _acme-challenge.{{ geo_mirror_domain }} mirror.pkgbuild.com
- pdnsutil replace-rrset _acme-challenge.{{ geo_mirror_domain }} @ SOA "mirror.pkgbuild.com. root.archlinux.org. 0 10800 3600 604800 3600"
become: true
become_user: powerdns
when: init.changed
- name: import TSIG key (for certbot)
command: pdnsutil import-tsig-key {{ certbot_rfc2136_key }} {{ certbot_rfc2136_algorithm }} {{ certbot_rfc2136_secret }}
changed_when: false
- name: open powerdns ipv4 port for monitoring.archlinux.org
ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=8081 accept"
tags:
- firewall
- name: start and enable powerdns
systemd: name=pdns.service enabled=yes daemon_reload=yes state=started
-- Based on https://github.com/PowerDNS/pdns/wiki/Lua-Examples-(Authoritative)#updatepolicy-access-control-for-rfc2136-dynamic-updates
function updatepolicy(input)
acme_challenge_rrname = "_acme-challenge.{{ geo_mirror_domain }}."
-- only allow updates from our servers
mynetworks = newNMG()
mynetworks:addMasks({
{% for host in groups['geo_mirrors'] | sort %}
'{{ hostvars[host]['ipv4_address'] }}/32',
'{{ hostvars[host]['ipv6_address'] }}/128',
{% endfor %}
})
-- ignore non-authorized networks
if not mynetworks:match(input:getRemote())
then
pdnslog("updatepolicy: network check failed from " .. input:getRemote():toString(), pdns.loglevels.Info)
return false
end
-- ignore non-TSIG requests
if input:getTsigName():countLabels() == 0
then
pdnslog("updatepolicy: missing TSIG", pdns.loglevels.Info)
return false
end
-- only accept TXT record updates for _acme_challenge
if input:getQType() == pdns.TXT and input:getQName():toString() == acme_challenge_rrname
then
pdnslog("updatepolicy: query checks successful", pdns.loglevels.Info)
return true
end
pdnslog("updatepolicy: query checks failed", pdns.loglevels.Info)
return false
end
#jinja2:lstrip_blocks: True
---
domains:
- domain: {{ geo_mirror_domain }}
ttl: 3600
records:
{{ geo_mirror_domain }}:
- soa: mirror.pkgbuild.com. root.archlinux.org. 2022011501 3600 1800 604800 3600
- ns: mirror.pkgbuild.com
{% for host in groups['geo_mirrors'] %}
{{ host.split(".")[0] }}.{{ geo_mirror_domain }}:
- a: {{ hostvars[host]['ipv4_address'] }}
- aaaa: {{ hostvars[host]['ipv6_address'] }}
{% endfor %}
services:
{{ geo_mirror_domain }}: '%mp.geo.mirror.pkgbuild.com'
mapping_lookup_formats: ['%cn']
custom_mapping:
af: europe
an: europe
as: asia
eu: europe
na: america
oc: asia
sa: america
unknown: europe
setgid=powerdns
setuid=powerdns
local-address={{ ipv4_address }},{{ ipv6_address }}
webserver=yes
webserver-address=0.0.0.0
webserver-allow-from=127.0.0.1,::1,{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
launch=geoip,gsqlite3
geoip-database-files=/var/lib/GeoIP/GeoLite2-Country.mmdb
geoip-zones-file=/etc/powerdns/geo.yml
gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
dnsupdate=yes
lua-dnsupdate-policy-script=/etc/powerdns/dnsupdate-policy.lua
......@@ -75,6 +75,12 @@ scrape_configs:
labels:
instance: "debuginfod.archlinux.org"
- job_name: 'powerdns'
static_configs:
- targets: ['{{ hostvars['mirror.pkgbuild.com']['wireguard_address'] }}:8081']
labels:
instance: "mirror.pkgbuild.com"
- job_name: 'gitlab_runner_exporter'
static_configs:
{% for host in groups['gitlab_runners'] %}
......
......@@ -7,6 +7,14 @@
domains: ["{{ mirror_domain }}"]
when: 'mirror_domain is defined'
- name: create ssl cert for geo mirror
include_role:
name: certificate
vars:
domains: ["{{ geo_mirror_domain }}"]
challenge: "DNS-01"
when: "'geo_mirrors' in group_names"
- name: install rsync
pacman: name=rsync state=present
......@@ -42,12 +50,15 @@
owner: root
group: root
- name: make nginx log dir
file: path=/var/log/nginx/{{ mirror_domain }} state=directory owner=root group=root mode=0755
- name: make nginx log dirs
file: path=/var/log/nginx/{{ item }} state=directory owner=root group=root mode=0755
loop: "{{ [mirror_domain, geo_mirror_domain] if 'geo_mirrors' in group_names else [mirror_domain] }}"
when: 'mirror_domain is defined'
- name: set up nginx
template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/syncrepo.conf owner=root group=root mode=0644
vars:
mirror_domains: "{{ [mirror_domain, geo_mirror_domain] if 'geo_mirrors' in group_names else [mirror_domain] }}"
notify:
- reload nginx
when: 'mirror_domain is defined'
......
{% for domain in mirror_domains %}
server {
listen 80;
listen [::]:80;
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ mirror_domain }};
server_name {{ domain }};
root /srv/ftp;
access_log /var/log/nginx/{{ mirror_domain }}/access.log reduced;
access_log /var/log/nginx/{{ mirror_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ mirror_domain }}/error.log;
access_log /var/log/nginx/{{ domain }}/access.log reduced;
access_log /var/log/nginx/{{ domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ domain }}/error.log;
include snippets/letsencrypt.conf;
ssl_certificate /etc/letsencrypt/live/{{ mirror_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ mirror_domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ mirror_domain }}/chain.pem;
ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ domain }}/chain.pem;
autoindex on;
}
{% endfor %}
......@@ -426,6 +426,13 @@ resource "hetznerdns_record" "pkgbuild_com_origin_txt" {
type = "TXT"
}
resource "hetznerdns_record" "pkgbuild_com_geo_mirror_ns" {
zone_id = hetznerdns_zone.pkgbuild.id
name = "geo.mirror"
value = "mirror.pkgbuild.com."
type = "NS"
}
resource "hetznerdns_record" "archlinux_org_origin_caa" {
zone_id = hetznerdns_zone.archlinux.id
name = "@"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment