group_vars/all/root_access.yml

@@ -23,11 +23,12 @@ root_ssh_keys:
     additional_keys: [klausenbusk_2.pub]
   - key: artafinde.pub
     hosts:
+      - aur.archlinux.org
       - dashboards.archlinux.org
       - gitlab.archlinux.org
-      - monitoring.archlinux.org
       - lists.archlinux.org
       - mailman3.archlinux.org
+      - monitoring.archlinux.org
 
 # - run 'playbooks/tasks/reencrypt-vault-{super,default}-key.yml' when this
 #   changes; before doing so, make sure to 'gpg --lsign-key' all listed keys

host_vars/asia.mirror.pkgbuild.com/misc

@@ -9,7 +9,7 @@ ipv6_address: "2a02:6ea0:d605::2"
 ipv6_netmask: "/128"
 ipv6_gateway: "2a02:6ea0:d605::1337"
 filesystem: "btrfs"
-network_interface: "enp1s0f0"
+network_interface: "enp175s0f0"
 system_disks:
   - /dev/sda
   - /dev/sdb

host_vars/runner1.archlinux.org/vault_gitlab_runner.yml

 $ANSIBLE_VAULT;1.1;AES256
-65336561343638323331326436643038656633323235323439373730396330366362643537313038
-6566346231333965616165643735346633306632393031300a396138633230363964386533646431
-35626132383035643431323839323830306435616463613934373435313565353263393735636662
-3739326165373931650a386331313133656566363232343635636632363761383366363233356266
-66613865396239626533303134643265366633323431393236643763316362323966313466306564
-32623031396365396363666162613664316539636639356333333463653432613536666539323638
-373866396166306432393165353530623534
+63643864346635336437366433613239313463376237393839333936626566633936386237323934
+3935366533633037383139643730373661626136373138360a636533386165323865306465633535
+31303332646438353263366234643235353137656333633663373637386437336162376433393432
+3963323663313862330a636334656334333439666231663564643837666161343634316238323237
+63626331643939613865356332623134626334313032323931353061336338373636353136353363
+35363964666330636330643338643030653134376461326161656165383862663561656336333065
+34376666363763326530333635363934613466663233663934623636613363353838616562356562
+61363135623633393230363335363732383062653835376461323934346262623237653338613637
+64323435363338613739333739623731336336336161633036316235343236666439

host_vars/secure-runner1.archlinux.org/vault_gitlab_runner.yml

 $ANSIBLE_VAULT;1.1;AES256
-31356534353263303630336136323233343664643962613339303933616134393461636364663633
-3032373939333130633632323035386132366261346332320a346462336333386265303262636331
-61396135363430393937316661613130616338643462323361386331323264343037633765646231
-3262323033663962320a623835383532353333626333656335356533353265663036366132393665
-66613335376333633038373633306239646130383830613139653130613265613135343764383137
-65626161333761343938663262636336616634623731653265393732363233383761653333326636
-613139393130636634343461333965656334
+61656464356262393461303061653330656164613364303364633434393566353732333665343565
+3332666566303439303934663664343032316430656164360a613533616465666465653334613237
+35343536646232333030623736303466396438353537313534613837383336623434656138396634
+3135383232333232640a613765663863356232373363333235393263386438643338653838343936
+63663636383239333437653239636465313861653532636363363038303936363632323237666262
+63386630623165626462356232393438313739356465363038626431623666366431326264383037
+30353130633836336135613239343234396338613732306263353333386632353334356331643630
+64396131383730343366643132353363356637353832643230343739303933386232363737653162
+34666363356265303162656632356361363034303931363362346463323662346636

hosts

@@ -11,7 +11,7 @@ sydney.mirror.pkgbuild.com
 
 [mirrors]
 america.mirror.pkgbuild.com
-#asia.mirror.pkgbuild.com
+asia.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com
 mirror.pkgbuild.com
 seoul.mirror.pkgbuild.com
@@ -19,14 +19,14 @@ sydney.mirror.pkgbuild.com
 
 [geo_mirrors]
 america.mirror.pkgbuild.com
-#asia.mirror.pkgbuild.com
+asia.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com
 seoul.mirror.pkgbuild.com
 sydney.mirror.pkgbuild.com
 
 [archive_mirrors]
 america.mirror.pkgbuild.com
-#asia.mirror.pkgbuild.com
+asia.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com
 
 [borg_clients]
@@ -90,6 +90,10 @@ runner1.archlinux.org
 runner2.archlinux.org
 secure-runner1.archlinux.org
 
+[gitlab_vm_runners]
+runner1.archlinux.org
+secure-runner1.archlinux.org
+
 [reproduciblebuilds]
 repro1.pkgbuild.com
 
@@ -109,7 +113,7 @@ monitoring.archlinux.org
 
 [kape_servers]
 america.mirror.pkgbuild.com
-#asia.mirror.pkgbuild.com
+asia.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com
 repro2.pkgbuild.com
 runner1.archlinux.org

playbooks/gitlab-runners.yml

@@ -11,4 +11,5 @@
     - { role: fail2ban }
     - { role: prometheus_exporters }
     - { role: promtail }
+    - { role: libvirt, when: "'gitlab_vm_runners' in group_names" }
     - { role: gitlab_runner }

roles/gitlab_runner/files/arch-boxes.asc

+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mDMEYpOJrBYJKwYBBAHaRw8BAQdAcSZilBvR58s6aD2qgsDE7WpvHQR2R5exQhNQ
+yuILsTq0JWFyY2gtYm94ZXMgPGFyY2gtYm94ZXNAYXJjaGxpbnV4Lm9yZz6IkAQT
+FggAOBYhBBuaFphKToy0SHEtKuC3i/QybG+PBQJik4msAhsBBQsJCAcCBhUKCQgL
+AgQWAgMBAh4BAheAAAoJEOC3i/QybG+P81YA/A7HUftMGpzlJrPYBFPqW0nFIh7m
+sIZ5yXxh7cTgqtJ7AQDFKSrulrsDa6hsqmEC11PWhv1VN6i9wfRvb1FwQPF6D7gz
+BGKTiecWCSsGAQQB2kcPAQEHQBzLxT2+CwumKUtfi9UEXMMx/oGgpjsgp2ehYPBM
+N8ejiPUEGBYIACYWIQQbmhaYSk6MtEhxLSrgt4v0MmxvjwUCYpOJ5wIbAgUJCWYB
+gACBCRDgt4v0Mmxvj3YgBBkWCAAdFiEEZW5MWsHMO4blOdl+NDY1poWakXQFAmKT
+iecACgkQNDY1poWakXTwaQEAwymt4PgXltHUH8GVUB6Xu7Gb5o6LwV9fNQJc1CMl
+7CABAJw0We0w1q78cJ8uWiomE1MHdRxsuqbuqtsCn2Dn6/0Cj+4A/Apcqm7uzFam
+pA5u9yvz1VJBWZY1PRBICBFSkuRtacUCAQC7YNurPPoWDyjiJPrf0Vzaz8UtKp0q
+BSF/a3EoocLnCA==
+=APeC
+-----END PGP PUBLIC KEY BLOCK-----

roles/gitlab_runner/files/domain_template.xml

+<domain type='kvm'>
+  <name>$vm_name</name>
+  <memory unit='MiB'>1024</memory>
+  <vcpu>4</vcpu>
+  <os>
+    <type arch='x86_64' machine='q35'>hvm</type>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+  </features>
+  <cpu mode='host-passthrough'/>
+  <!-- https://github.com/virt-manager/virt-manager/blob/7ae10b5566ac4d8c7afd94499a9733ed42cf3d07/virtinst/domain/clock.py#L49-L59 -->
+  <clock offset='utc'>
+    <timer name='rtc' tickpolicy='catchup'/>
+    <timer name='pit' tickpolicy='delay'/>
+    <timer name='hpet' present='no'/>
+  </clock>
+  <devices>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='/var/lib/libvirt/images/$vm_name.qcow2'/>
+      <target dev='sdb' bus='scsi'/>
+    </disk>
+    <controller type='pci' model='pcie-root'/>
+    <controller type='scsi' model='virtio-scsi'/>
+    <controller type='usb' model='none'/>
+    <interface type='network'>
+      <source network='default'/>
+      <model type='virtio'/>
+      <filterref filter='clean-traffic'/>
+    </interface>
+    <rng model='virtio'>
+      <backend model='random'>/dev/urandom</backend>
+    </rng>
+    <video>
+      <model type='virtio'/>
+    </video>
+    <graphics type='vnc'/>
+  </devices>
+</domain>

roles/gitlab_runner/files/libvirt-executor

+#!/usr/bin/env bash
+set -o nounset -o errexit -o pipefail
+readonly libvirt_default_pool_path="/var/lib/libvirt/images"
+
+ssh() {
+  command ssh \
+    -i "/etc/libvirt-executor/id_ed25519" \
+    -F /dev/null \
+    -o ServerAliveCountMax=2 \
+    -o ServerAliveInterval=15 \
+    -o UserKnownHostsFile=/dev/null \
+    -o StrictHostKeyChecking=off \
+    "root@${1}" "${@:2}"
+}
+
+vm_name() {
+  printf 'runner-%s-project-%d-pipeline-%d-job-%d\n' "${CUSTOM_ENV_CI_RUNNER_SHORT_TOKEN}" "${CUSTOM_ENV_CI_PROJECT_ID}" "${CUSTOM_ENV_CI_PIPELINE_IID}" "${CUSTOM_ENV_CI_JOB_ID}"
+}
+
+vm_ip() {
+  local ip
+  ip="$(virsh -q domifaddr "${1}" | awk -F'[ /]+' '{print $5}')"
+  if [[ -n ${ip} ]]; then
+    echo "${ip}"
+    return 0
+  fi
+  return 1
+}
+
+wait_for_ssh() {
+  for _ in {1..30}; do
+    if ! ip="$(vm_ip "${1}")"; then
+      echo "Waiting for network"
+      sleep 1
+      continue
+    fi
+    if ! ssh "${ip}" true; then
+      echo "Waiting for SSH to be ready"
+      sleep 1
+      continue
+    fi
+    printf "%s" "${ip}"
+    return 0
+  done
+  echo 'Waited 30 seconds for VM to start, exiting...'
+  exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
+}
+
+# https://docs.gitlab.com/runner/executors/custom.html#prepare
+prepare() {
+  # shellcheck disable=SC2064
+  trap "exit ${SYSTEM_FAILURE_EXIT_CODE:-1}" ERR
+  local base_image
+  base_image="$(compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | tail -n 1)"
+
+  if [[ -z ${base_image} ]]; then
+    echo 'Base image not found...'
+    exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
+  fi
+
+  qemu-img create -f qcow2 -b "${base_image}" -F qcow2 "${libvirt_default_pool_path}/$(vm_name).qcow2"
+  virsh define <(sed "s/\$vm_name/$(vm_name)/" /usr/local/lib/libvirt-executor/domain_template.xml)
+  virsh start "$(vm_name)"
+
+  wait_for_ssh "$(vm_name)"
+}
+
+# https://docs.gitlab.com/runner/executors/custom.html#run
+run() {
+  local ip
+  ip="$(wait_for_ssh "$(vm_name)")"
+  ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+}
+
+# https://docs.gitlab.com/runner/executors/custom.html#cleanup
+cleanup() {
+  virsh destroy "$(vm_name)" || true
+  rm "${libvirt_default_pool_path}/$(vm_name).qcow2"
+  virsh undefine "$(vm_name)"
+}
+
+case "${1:-}" in
+  prepare)
+    prepare
+    ;;
+  run)
+    run "${2}" "${3}"
+    ;;
+  cleanup)
+    cleanup
+    ;;
+  *)
+    echo "Error invalid command: ${1:-}"
+    exit 1;
+esac

roles/gitlab_runner/files/libvirt-executor-update-base-image

+#!/bin/bash
+set -o nounset -o errexit
+readonly libvirt_default_pool_path="/var/lib/libvirt/images"
+
+cleanup() {
+  set +o errexit
+
+  if mountpoint -q mnt; then
+    umount -R mnt
+  fi
+  if [[ -n ${loopdev} ]]; then
+    losetup -d "${loopdev}"
+  fi
+  rm -r "${tmpdir}"
+}
+
+tmpdir="$(mktemp --directory --tmpdir="/var/tmp")"
+trap cleanup EXIT
+
+cd "${tmpdir}"
+curl -sSf --remote-name-all https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2{,.sig}
+sq verify --signer-cert /usr/local/lib/libvirt-executor/arch-boxes.asc --detached Arch-Linux-x86_64-basic.qcow2.sig Arch-Linux-x86_64-basic.qcow2
+
+image=Arch-Linux-x86_64-basic.img
+qemu-img convert -f qcow2 -O raw Arch-Linux-x86_64-basic.qcow2 Arch-Linux-x86_64-basic.img
+
+loopdev="$(losetup --find --partscan --show "${image}")"
+mount --mkdir "${loopdev}p2" mnt
+
+arch-chroot mnt pacman-key --init
+arch-chroot mnt pacman-key --populate
+
+# shellcheck disable=SC2016
+printf 'Server = https://geo.mirror.pkgbuild.com/$repo/os/$arch' > mnt/etc/pacman.d/mirrorlist
+arch-chroot mnt systemctl disable reflector-init
+arch-chroot mnt pacman -Sy --noconfirm --needed archlinux-keyring
+arch-chroot mnt pacman -Syu --noconfirm --needed git git-lfs gitlab-runner
+sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i mnt/etc/pacman.conf
+arch-chroot mnt userdel -r arch
+install -d -m0700 mnt/root/.ssh
+install -m0600 /etc/libvirt-executor/id_ed25519.pub mnt/root/.ssh/authorized_keys
+rm -f mnt/etc/machine-id
+
+cp -a mnt/boot/{initramfs-linux-fallback.img,initramfs-linux.img}
+
+rm -r mnt/etc/pacman.d/gnupg/{openpgp-revocs.d,private-keys-v1.d}/
+arch-chroot mnt pacman-key --delete pacman@localhost
+umount mnt
+losetup -d "${loopdev}"
+loopdev=""
+
+qemu-img convert -f raw -O qcow2 Arch-Linux-x86_64-basic.img Arch-Linux-x86_64-basic.qcow2
+printf -v image_path '%s/runner-base-%(%s)T.qcow2' "${libvirt_default_pool_path}"
+cp Arch-Linux-x86_64-basic.qcow2 "${image_path}.tmp"
+mv "${image_path}"{.tmp,}
+
+# Keep one week of base images
+compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | head -n -7 | xargs --no-run-if-empty rm -vf

roles/gitlab_runner/files/libvirt-executor-update-base-image.service

+[Unit]
+Description=Update libvirt-executor base image
+Wants=network-online.target
+After=network-online.target nss-lookup.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/libvirt-executor-update-base-image

roles/gitlab_runner/files/libvirt-executor-update-base-image.timer

+[Unit]
+Description=Run libvirt-executor-vm-template.service daily
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+RandomizedDelaySec=1d
+
+[Install]
+WantedBy=timers.target

roles/gitlab_runner/tasks/main.yml

@@ -34,7 +34,7 @@
 #   --non-interactive \
 #   --url=https://gitlab.archlinux.org/ \
 #   --docker-image=archlinux:latest \
-#   --tag-list=docker \ # Use docker,secure for secure runners
+#   --tag-list=docker \ # Use docker,secure for secure runners and docker,secure-vm for secure VM runners
 #   --registration-token="{{ vault_gitlab_runner_registration_token }}" \
 #   --executor=docker \
 #   --description="{{ inventory_hostname }}" \
@@ -59,3 +59,37 @@
 
 - name: enable and start gitlab runner service
   systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes
+
+- name: setup libvirt-executor
+  block:
+    - name: install libvirt-executor-update-base-image dependencies
+      pacman: name=arch-install-scripts,sequoia-sq state=present
+
+    - name: create libvirt-executor configuration and data directories
+      file: path={{ item }} state=directory owner=root group=root mode=0755
+      loop:
+        - /etc/libvirt-executor
+        - /usr/local/lib/libvirt-executor
+
+    - name: install libvirt-executor
+      copy: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode={{ item.mode }}
+      loop:
+        - {src: arch-boxes.asc, dest: /usr/local/lib/libvirt-executor/, mode: 644}
+        - {src: domain_template.xml, dest: /usr/local/lib/libvirt-executor/, mode: 755}
+        - {src: libvirt-executor, dest: /usr/local/bin/, mode: 755}
+        - {src: libvirt-executor-update-base-image, dest: /usr/local/bin/, mode: 755}
+
+    - name: create SSH keys for libvirt-executor
+      command: ssh-keygen -N "" -f /etc/libvirt-executor/id_ed25519 -t ed25519
+      args:
+        creates: /etc/libvirt-executor/id_ed25519
+
+    - name: install libvirt-executor-update-base-image.{service,timer}
+      copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
+      loop:
+        - libvirt-executor-update-base-image.service
+        - libvirt-executor-update-base-image.timer
+
+    - name: enable and start libvirt-executor-update-base-image.timer
+      systemd: name=libvirt-executor-update-base-image.timer state=started enabled=yes daemon_reload=yes
+  when: "'gitlab_vm_runners' in group_names"

roles/gitlab_runner/templates/config.toml.j2

@@ -23,3 +23,24 @@ listen_address = ":9252"
     disable_cache = false
     volumes = ["/cache"]
     shm_size = 0
+{% if 'gitlab_vm_runners' in group_names %}
+
+[[runners]]
+  name = "{{ inventory_hostname }}"
+  url = "https://gitlab.archlinux.org"
+  token = "{{ vault_gitlab_vm_runner_token }}"
+  executor = "custom"
+  builds_dir = "/builds"
+  cache_dir = "/cache"
+  limit = {{ (ansible_memtotal_mb * 0.9 / 1024) | round | int }}
+  environment = ["ARCHIVER_STAGING_DIR=/var/tmp"]
+  [runners.custom]
+    prepare_exec = "/usr/local/bin/libvirt-executor"
+    prepare_args = [ "prepare" ]
+
+    run_exec = "/usr/local/bin/libvirt-executor"
+    run_args = [ "run" ]
+
+    cleanup_exec = "/usr/local/bin/libvirt-executor"
+    cleanup_args = [ "cleanup" ]
+{% endif %}

roles/libvirt/tasks/main.yml

+---
+- name: remove iptables to solve iptables<->iptables-nft conflict
+  pacman: name=iptables force=yes state=absent
+
+- name: install libvirt and needed optional dependencies
+  pacman: name=libvirt,qemu-base,dnsmasq,iptables-nft state=present
+  register: result
+
+- name: reload firewalld
+  service: name=firewalld state=reloaded
+  when: result.changed
+
+- name: autostart default network on boot
+  file: src=/etc/libvirt/qemu/networks/default.xml dest=/etc/libvirt/qemu/networks/autostart/default.xml state=link owner=root group=root
+
+- name: start and enable libvirtd
+  systemd: name=libvirtd enabled=yes state=started daemon_reload=yes

tf-stage1/templates.tf

@@ -192,15 +192,15 @@ resource "hetznerdns_record" "geo_ns1" {
   ttl     = lookup(local.geo_domains[each.key], "ttl", 86400)
 }
 
-#resource "hetznerdns_record" "geo_ns2" {
-#  for_each = local.geo_domains
-#
-#  zone_id = each.value.zone_id
-#  name    = each.value.name
-#  value   = "asia.mirror.pkgbuild.com."
-#  type    = "NS"
-#  ttl     = lookup(local.geo_domains[each.key], "ttl", 86400)
-#}
+resource "hetznerdns_record" "geo_ns2" {
+  for_each = local.geo_domains
+
+  zone_id = each.value.zone_id
+  name    = each.value.name
+  value   = "asia.mirror.pkgbuild.com."
+  type    = "NS"
+  ttl     = lookup(local.geo_domains[each.key], "ttl", 86400)
+}
 
 resource "hetznerdns_record" "geo_ns3" {
   for_each = local.geo_domains