diff --git a/host_vars/runner1.archlinux.org/vault_gitlab_runner.yml b/host_vars/runner1.archlinux.org/vault_gitlab_runner.yml index d5f202422f3e5ce6f485b76a6619c7e2a34c6122..4e92236c093cefc481cfab74f870ac0a959b1d22 100644 --- a/host_vars/runner1.archlinux.org/vault_gitlab_runner.yml +++ b/host_vars/runner1.archlinux.org/vault_gitlab_runner.yml @@ -1,8 +1,10 @@ $ANSIBLE_VAULT;1.1;AES256 -65336561343638323331326436643038656633323235323439373730396330366362643537313038 -6566346231333965616165643735346633306632393031300a396138633230363964386533646431 -35626132383035643431323839323830306435616463613934373435313565353263393735636662 -3739326165373931650a386331313133656566363232343635636632363761383366363233356266 -66613865396239626533303134643265366633323431393236643763316362323966313466306564 -32623031396365396363666162613664316539636639356333333463653432613536666539323638 -373866396166306432393165353530623534 +63643864346635336437366433613239313463376237393839333936626566633936386237323934 +3935366533633037383139643730373661626136373138360a636533386165323865306465633535 +31303332646438353263366234643235353137656333633663373637386437336162376433393432 +3963323663313862330a636334656334333439666231663564643837666161343634316238323237 +63626331643939613865356332623134626334313032323931353061336338373636353136353363 +35363964666330636330643338643030653134376461326161656165383862663561656336333065 +34376666363763326530333635363934613466663233663934623636613363353838616562356562 +61363135623633393230363335363732383062653835376461323934346262623237653338613637 +64323435363338613739333739623731336336336161633036316235343236666439 diff --git a/host_vars/secure-runner1.archlinux.org/vault_gitlab_runner.yml b/host_vars/secure-runner1.archlinux.org/vault_gitlab_runner.yml index c7d9dd9574a4a4746c5daa563cb0a24fe2065e8f..c7856bf7292050ed1782b68deed60705bc47cc15 100644 --- a/host_vars/secure-runner1.archlinux.org/vault_gitlab_runner.yml +++ b/host_vars/secure-runner1.archlinux.org/vault_gitlab_runner.yml @@ -1,8 +1,10 @@ $ANSIBLE_VAULT;1.1;AES256 -31356534353263303630336136323233343664643962613339303933616134393461636364663633 -3032373939333130633632323035386132366261346332320a346462336333386265303262636331 -61396135363430393937316661613130616338643462323361386331323264343037633765646231 -3262323033663962320a623835383532353333626333656335356533353265663036366132393665 -66613335376333633038373633306239646130383830613139653130613265613135343764383137 -65626161333761343938663262636336616634623731653265393732363233383761653333326636 -613139393130636634343461333965656334 +61656464356262393461303061653330656164613364303364633434393566353732333665343565 +3332666566303439303934663664343032316430656164360a613533616465666465653334613237 +35343536646232333030623736303466396438353537313534613837383336623434656138396634 +3135383232333232640a613765663863356232373363333235393263386438643338653838343936 +63663636383239333437653239636465313861653532636363363038303936363632323237666262 +63386630623165626462356232393438313739356465363038626431623666366431326264383037 +30353130633836336135613239343234396338613732306263353333386632353334356331643630 +64396131383730343366643132353363356637353832643230343739303933386232363737653162 +34666363356265303162656632356361363034303931363362346463323662346636 diff --git a/hosts b/hosts index b5b87b65dd1f31db9aebf80cd2acbbcd521723e0..d3c2dc3393206038b7d34a0371bfe1eeb23ffc90 100644 --- a/hosts +++ b/hosts @@ -90,6 +90,10 @@ runner1.archlinux.org runner2.archlinux.org secure-runner1.archlinux.org +[gitlab_vm_runners] +runner1.archlinux.org +secure-runner1.archlinux.org + [reproduciblebuilds] repro1.pkgbuild.com diff --git a/playbooks/gitlab-runners.yml b/playbooks/gitlab-runners.yml index a619354c373fd23e78db866784379e6d0db6eacd..b943d70c95d08bd3ca0db9efcf2b0c47390060db 100644 --- a/playbooks/gitlab-runners.yml +++ b/playbooks/gitlab-runners.yml @@ -11,4 +11,5 @@ - { role: fail2ban } - { role: prometheus_exporters } - { role: promtail } + - { role: libvirt, when: "'gitlab_vm_runners' in group_names" } - { role: gitlab_runner } diff --git a/roles/gitlab_runner/files/arch-boxes.asc b/roles/gitlab_runner/files/arch-boxes.asc new file mode 100644 index 0000000000000000000000000000000000000000..8093217cbfdbfcca865506b948c0b5eef8e9b974 --- /dev/null +++ b/roles/gitlab_runner/files/arch-boxes.asc @@ -0,0 +1,16 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYpOJrBYJKwYBBAHaRw8BAQdAcSZilBvR58s6aD2qgsDE7WpvHQR2R5exQhNQ +yuILsTq0JWFyY2gtYm94ZXMgPGFyY2gtYm94ZXNAYXJjaGxpbnV4Lm9yZz6IkAQT +FggAOBYhBBuaFphKToy0SHEtKuC3i/QybG+PBQJik4msAhsBBQsJCAcCBhUKCQgL +AgQWAgMBAh4BAheAAAoJEOC3i/QybG+P81YA/A7HUftMGpzlJrPYBFPqW0nFIh7m +sIZ5yXxh7cTgqtJ7AQDFKSrulrsDa6hsqmEC11PWhv1VN6i9wfRvb1FwQPF6D7gz +BGKTiecWCSsGAQQB2kcPAQEHQBzLxT2+CwumKUtfi9UEXMMx/oGgpjsgp2ehYPBM +N8ejiPUEGBYIACYWIQQbmhaYSk6MtEhxLSrgt4v0MmxvjwUCYpOJ5wIbAgUJCWYB +gACBCRDgt4v0Mmxvj3YgBBkWCAAdFiEEZW5MWsHMO4blOdl+NDY1poWakXQFAmKT +iecACgkQNDY1poWakXTwaQEAwymt4PgXltHUH8GVUB6Xu7Gb5o6LwV9fNQJc1CMl +7CABAJw0We0w1q78cJ8uWiomE1MHdRxsuqbuqtsCn2Dn6/0Cj+4A/Apcqm7uzFam +pA5u9yvz1VJBWZY1PRBICBFSkuRtacUCAQC7YNurPPoWDyjiJPrf0Vzaz8UtKp0q +BSF/a3EoocLnCA== +=APeC +-----END PGP PUBLIC KEY BLOCK----- diff --git a/roles/gitlab_runner/files/domain_template.xml b/roles/gitlab_runner/files/domain_template.xml new file mode 100644 index 0000000000000000000000000000000000000000..7c359a10c512f924bacedd6353f39a72f6e33557 --- /dev/null +++ b/roles/gitlab_runner/files/domain_template.xml @@ -0,0 +1,41 @@ +<domain type='kvm'> + <name>$vm_name</name> + <memory unit='MiB'>1024</memory> + <vcpu>4</vcpu> + <os> + <type arch='x86_64' machine='q35'>hvm</type> + </os> + <features> + <acpi/> + <apic/> + </features> + <cpu mode='host-passthrough'/> + <!-- https://github.com/virt-manager/virt-manager/blob/7ae10b5566ac4d8c7afd94499a9733ed42cf3d07/virtinst/domain/clock.py#L49-L59 --> + <clock offset='utc'> + <timer name='rtc' tickpolicy='catchup'/> + <timer name='pit' tickpolicy='delay'/> + <timer name='hpet' present='no'/> + </clock> + <devices> + <disk type='file' device='disk'> + <driver name='qemu' type='qcow2'/> + <source file='/var/lib/libvirt/images/$vm_name.qcow2'/> + <target dev='sdb' bus='scsi'/> + </disk> + <controller type='pci' model='pcie-root'/> + <controller type='scsi' model='virtio-scsi'/> + <controller type='usb' model='none'/> + <interface type='network'> + <source network='default'/> + <model type='virtio'/> + <filterref filter='clean-traffic'/> + </interface> + <rng model='virtio'> + <backend model='random'>/dev/urandom</backend> + </rng> + <video> + <model type='virtio'/> + </video> + <graphics type='vnc'/> + </devices> +</domain> diff --git a/roles/gitlab_runner/files/libvirt-executor b/roles/gitlab_runner/files/libvirt-executor new file mode 100755 index 0000000000000000000000000000000000000000..7f6a2162ada9e179cb433940ff1ebb31c09d944a --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -o nounset -o errexit -o pipefail +readonly libvirt_default_pool_path="/var/lib/libvirt/images" + +ssh() { + command ssh \ + -i "/etc/libvirt-executor/id_ed25519" \ + -F /dev/null \ + -o ServerAliveCountMax=2 \ + -o ServerAliveInterval=15 \ + -o UserKnownHostsFile=/dev/null \ + -o StrictHostKeyChecking=off \ + "root@${1}" "${@:2}" +} + +vm_name() { + printf 'runner-%s-project-%d-pipeline-%d-job-%d\n' "${CUSTOM_ENV_CI_RUNNER_SHORT_TOKEN}" "${CUSTOM_ENV_CI_PROJECT_ID}" "${CUSTOM_ENV_CI_PIPELINE_IID}" "${CUSTOM_ENV_CI_JOB_ID}" +} + +vm_ip() { + local ip + ip="$(virsh -q domifaddr "${1}" | awk -F'[ /]+' '{print $5}')" + if [[ -n ${ip} ]]; then + echo "${ip}" + return 0 + fi + return 1 +} + +wait_for_ssh() { + for _ in {1..30}; do + if ! ip="$(vm_ip "${1}")"; then + echo "Waiting for network" + sleep 1 + continue + fi + if ! ssh "${ip}" true; then + echo "Waiting for SSH to be ready" + sleep 1 + continue + fi + printf "%s" "${ip}" + return 0 + done + echo 'Waited 30 seconds for VM to start, exiting...' + exit "${SYSTEM_FAILURE_EXIT_CODE:-1}" +} + +# https://docs.gitlab.com/runner/executors/custom.html#prepare +prepare() { + # shellcheck disable=SC2064 + trap "exit ${SYSTEM_FAILURE_EXIT_CODE:-1}" ERR + local base_image + base_image="$(compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | tail -n 1)" + + if [[ -z ${base_image} ]]; then + echo 'Base image not found...' + exit "${SYSTEM_FAILURE_EXIT_CODE:-1}" + fi + + qemu-img create -f qcow2 -b "${base_image}" -F qcow2 "${libvirt_default_pool_path}/$(vm_name).qcow2" + virsh define <(sed "s/\$vm_name/$(vm_name)/" /usr/local/lib/libvirt-executor/domain_template.xml) + virsh start "$(vm_name)" + + wait_for_ssh "$(vm_name)" +} + +# https://docs.gitlab.com/runner/executors/custom.html#run +run() { + local ip + ip="$(wait_for_ssh "$(vm_name)")" + ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}" +} + +# https://docs.gitlab.com/runner/executors/custom.html#cleanup +cleanup() { + virsh destroy "$(vm_name)" || true + rm "${libvirt_default_pool_path}/$(vm_name).qcow2" + virsh undefine "$(vm_name)" +} + +case "${1:-}" in + prepare) + prepare + ;; + run) + run "${2}" "${3}" + ;; + cleanup) + cleanup + ;; + *) + echo "Error invalid command: ${1:-}" + exit 1; +esac diff --git a/roles/gitlab_runner/files/libvirt-executor-update-base-image b/roles/gitlab_runner/files/libvirt-executor-update-base-image new file mode 100755 index 0000000000000000000000000000000000000000..ffd7d0afdb3823ad3270a38edca5ab816d4399e0 --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor-update-base-image @@ -0,0 +1,58 @@ +#!/bin/bash +set -o nounset -o errexit +readonly libvirt_default_pool_path="/var/lib/libvirt/images" + +cleanup() { + set +o errexit + + if mountpoint -q mnt; then + umount -R mnt + fi + if [[ -n ${loopdev} ]]; then + losetup -d "${loopdev}" + fi + rm -r "${tmpdir}" +} + +tmpdir="$(mktemp --directory --tmpdir="/var/tmp")" +trap cleanup EXIT + +cd "${tmpdir}" +curl -sSf --remote-name-all https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2{,.sig} +sq verify --signer-cert /usr/local/lib/libvirt-executor/arch-boxes.asc --detached Arch-Linux-x86_64-basic.qcow2.sig Arch-Linux-x86_64-basic.qcow2 + +image=Arch-Linux-x86_64-basic.img +qemu-img convert -f qcow2 -O raw Arch-Linux-x86_64-basic.qcow2 Arch-Linux-x86_64-basic.img + +loopdev="$(losetup --find --partscan --show "${image}")" +mount --mkdir "${loopdev}p2" mnt + +arch-chroot mnt pacman-key --init +arch-chroot mnt pacman-key --populate + +# shellcheck disable=SC2016 +printf 'Server = https://geo.mirror.pkgbuild.com/$repo/os/$arch' > mnt/etc/pacman.d/mirrorlist +arch-chroot mnt systemctl disable reflector-init +arch-chroot mnt pacman -Sy --noconfirm --needed archlinux-keyring +arch-chroot mnt pacman -Syu --noconfirm --needed git git-lfs gitlab-runner +sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i mnt/etc/pacman.conf +arch-chroot mnt userdel -r arch +install -d -m0700 mnt/root/.ssh +install -m0600 /etc/libvirt-executor/id_ed25519.pub mnt/root/.ssh/authorized_keys +rm -f mnt/etc/machine-id + +cp -a mnt/boot/{initramfs-linux-fallback.img,initramfs-linux.img} + +rm -r mnt/etc/pacman.d/gnupg/{openpgp-revocs.d,private-keys-v1.d}/ +arch-chroot mnt pacman-key --delete pacman@localhost +umount mnt +losetup -d "${loopdev}" +loopdev="" + +qemu-img convert -f raw -O qcow2 Arch-Linux-x86_64-basic.img Arch-Linux-x86_64-basic.qcow2 +printf -v image_path '%s/runner-base-%(%s)T.qcow2' "${libvirt_default_pool_path}" +cp Arch-Linux-x86_64-basic.qcow2 "${image_path}.tmp" +mv "${image_path}"{.tmp,} + +# Keep one week of base images +compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | head -n -7 | xargs --no-run-if-empty rm -vf diff --git a/roles/gitlab_runner/files/libvirt-executor-update-base-image.service b/roles/gitlab_runner/files/libvirt-executor-update-base-image.service new file mode 100644 index 0000000000000000000000000000000000000000..b28f92fc30128aae1e57ab4772d43fddbf6e2b3a --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor-update-base-image.service @@ -0,0 +1,8 @@ +[Unit] +Description=Update libvirt-executor base image +Wants=network-online.target +After=network-online.target nss-lookup.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/libvirt-executor-update-base-image diff --git a/roles/gitlab_runner/files/libvirt-executor-update-base-image.timer b/roles/gitlab_runner/files/libvirt-executor-update-base-image.timer new file mode 100644 index 0000000000000000000000000000000000000000..4c7435bdb36db4381a2ab08f26b03c66095568e1 --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor-update-base-image.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run libvirt-executor-vm-template.service daily + +[Timer] +OnCalendar=daily +Persistent=true +RandomizedDelaySec=1d + +[Install] +WantedBy=timers.target diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml index dbd871061f130f3626838f57a8957de2bd897db9..2bd92f56df2aa3127547f2836304151f45902b04 100644 --- a/roles/gitlab_runner/tasks/main.yml +++ b/roles/gitlab_runner/tasks/main.yml @@ -34,7 +34,7 @@ # --non-interactive \ # --url=https://gitlab.archlinux.org/ \ # --docker-image=archlinux:latest \ -# --tag-list=docker \ # Use docker,secure for secure runners +# --tag-list=docker \ # Use docker,secure for secure runners and docker,secure-vm for secure VM runners # --registration-token="{{ vault_gitlab_runner_registration_token }}" \ # --executor=docker \ # --description="{{ inventory_hostname }}" \ @@ -59,3 +59,37 @@ - name: enable and start gitlab runner service systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes + +- name: setup libvirt-executor + block: + - name: install libvirt-executor-update-base-image dependencies + pacman: name=arch-install-scripts,sequoia-sq state=present + + - name: create libvirt-executor configuration and data directories + file: path={{ item }} state=directory owner=root group=root mode=0755 + loop: + - /etc/libvirt-executor + - /usr/local/lib/libvirt-executor + + - name: install libvirt-executor + copy: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode={{ item.mode }} + loop: + - {src: arch-boxes.asc, dest: /usr/local/lib/libvirt-executor/, mode: 644} + - {src: domain_template.xml, dest: /usr/local/lib/libvirt-executor/, mode: 755} + - {src: libvirt-executor, dest: /usr/local/bin/, mode: 755} + - {src: libvirt-executor-update-base-image, dest: /usr/local/bin/, mode: 755} + + - name: create SSH keys for libvirt-executor + command: ssh-keygen -N "" -f /etc/libvirt-executor/id_ed25519 -t ed25519 + args: + creates: /etc/libvirt-executor/id_ed25519 + + - name: install libvirt-executor-update-base-image.{service,timer} + copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644 + loop: + - libvirt-executor-update-base-image.service + - libvirt-executor-update-base-image.timer + + - name: enable and start libvirt-executor-update-base-image.timer + systemd: name=libvirt-executor-update-base-image.timer state=started enabled=yes daemon_reload=yes + when: "'gitlab_vm_runners' in group_names" diff --git a/roles/gitlab_runner/templates/config.toml.j2 b/roles/gitlab_runner/templates/config.toml.j2 index 4752005a3196c259dd362a28d736e18da88b4899..57dba9dfa48009a3a5f680641afdf23bea95b173 100644 --- a/roles/gitlab_runner/templates/config.toml.j2 +++ b/roles/gitlab_runner/templates/config.toml.j2 @@ -23,3 +23,24 @@ listen_address = ":9252" disable_cache = false volumes = ["/cache"] shm_size = 0 +{% if 'gitlab_vm_runners' in group_names %} + +[[runners]] + name = "{{ inventory_hostname }}" + url = "https://gitlab.archlinux.org" + token = "{{ vault_gitlab_vm_runner_token }}" + executor = "custom" + builds_dir = "/builds" + cache_dir = "/cache" + limit = {{ (ansible_memtotal_mb * 0.9 / 1024) | round | int }} + environment = ["ARCHIVER_STAGING_DIR=/var/tmp"] + [runners.custom] + prepare_exec = "/usr/local/bin/libvirt-executor" + prepare_args = [ "prepare" ] + + run_exec = "/usr/local/bin/libvirt-executor" + run_args = [ "run" ] + + cleanup_exec = "/usr/local/bin/libvirt-executor" + cleanup_args = [ "cleanup" ] +{% endif %} diff --git a/roles/libvirt/tasks/main.yml b/roles/libvirt/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..efa37cdbaedd08f9fdd31741c99a5dda9d65c709 --- /dev/null +++ b/roles/libvirt/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: remove iptables to solve iptables<->iptables-nft conflict + pacman: name=iptables force=yes state=absent + +- name: install libvirt and needed optional dependencies + pacman: name=libvirt,qemu-base,dnsmasq,iptables-nft state=present + register: result + +- name: reload firewalld + service: name=firewalld state=reloaded + when: result.changed + +- name: autostart default network on boot + file: src=/etc/libvirt/qemu/networks/default.xml dest=/etc/libvirt/qemu/networks/autostart/default.xml state=link owner=root group=root + +- name: start and enable libvirtd + systemd: name=libvirtd enabled=yes state=started daemon_reload=yes