Skip to content
Snippets Groups Projects
Verified Commit ff367f36 authored by Kristian Klausen's avatar Kristian Klausen :tada:
Browse files

Merge branch 'custom-executor' into 'master'

gitlab_runner: Add VM based executor (libvirt-executor)

Closes #283

See merge request !385
parents afa3206d 466230e4
No related branches found
No related tags found
1 merge request!385gitlab_runner: Add VM based executor (libvirt-executor)
Pipeline #23762 passed
Showing
with 324 additions and 15 deletions
$ANSIBLE_VAULT;1.1;AES256
65336561343638323331326436643038656633323235323439373730396330366362643537313038
6566346231333965616165643735346633306632393031300a396138633230363964386533646431
35626132383035643431323839323830306435616463613934373435313565353263393735636662
3739326165373931650a386331313133656566363232343635636632363761383366363233356266
66613865396239626533303134643265366633323431393236643763316362323966313466306564
32623031396365396363666162613664316539636639356333333463653432613536666539323638
373866396166306432393165353530623534
63643864346635336437366433613239313463376237393839333936626566633936386237323934
3935366533633037383139643730373661626136373138360a636533386165323865306465633535
31303332646438353263366234643235353137656333633663373637386437336162376433393432
3963323663313862330a636334656334333439666231663564643837666161343634316238323237
63626331643939613865356332623134626334313032323931353061336338373636353136353363
35363964666330636330643338643030653134376461326161656165383862663561656336333065
34376666363763326530333635363934613466663233663934623636613363353838616562356562
61363135623633393230363335363732383062653835376461323934346262623237653338613637
64323435363338613739333739623731336336336161633036316235343236666439
$ANSIBLE_VAULT;1.1;AES256
31356534353263303630336136323233343664643962613339303933616134393461636364663633
3032373939333130633632323035386132366261346332320a346462336333386265303262636331
61396135363430393937316661613130616338643462323361386331323264343037633765646231
3262323033663962320a623835383532353333626333656335356533353265663036366132393665
66613335376333633038373633306239646130383830613139653130613265613135343764383137
65626161333761343938663262636336616634623731653265393732363233383761653333326636
613139393130636634343461333965656334
61656464356262393461303061653330656164613364303364633434393566353732333665343565
3332666566303439303934663664343032316430656164360a613533616465666465653334613237
35343536646232333030623736303466396438353537313534613837383336623434656138396634
3135383232333232640a613765663863356232373363333235393263386438643338653838343936
63663636383239333437653239636465313861653532636363363038303936363632323237666262
63386630623165626462356232393438313739356465363038626431623666366431326264383037
30353130633836336135613239343234396338613732306263353333386632353334356331643630
64396131383730343366643132353363356637353832643230343739303933386232363737653162
34666363356265303162656632356361363034303931363362346463323662346636
......@@ -90,6 +90,10 @@ runner1.archlinux.org
runner2.archlinux.org
secure-runner1.archlinux.org
[gitlab_vm_runners]
runner1.archlinux.org
secure-runner1.archlinux.org
[reproduciblebuilds]
repro1.pkgbuild.com
......
......@@ -11,4 +11,5 @@
- { role: fail2ban }
- { role: prometheus_exporters }
- { role: promtail }
- { role: libvirt, when: "'gitlab_vm_runners' in group_names" }
- { role: gitlab_runner }
-----BEGIN PGP PUBLIC KEY BLOCK-----
mDMEYpOJrBYJKwYBBAHaRw8BAQdAcSZilBvR58s6aD2qgsDE7WpvHQR2R5exQhNQ
yuILsTq0JWFyY2gtYm94ZXMgPGFyY2gtYm94ZXNAYXJjaGxpbnV4Lm9yZz6IkAQT
FggAOBYhBBuaFphKToy0SHEtKuC3i/QybG+PBQJik4msAhsBBQsJCAcCBhUKCQgL
AgQWAgMBAh4BAheAAAoJEOC3i/QybG+P81YA/A7HUftMGpzlJrPYBFPqW0nFIh7m
sIZ5yXxh7cTgqtJ7AQDFKSrulrsDa6hsqmEC11PWhv1VN6i9wfRvb1FwQPF6D7gz
BGKTiecWCSsGAQQB2kcPAQEHQBzLxT2+CwumKUtfi9UEXMMx/oGgpjsgp2ehYPBM
N8ejiPUEGBYIACYWIQQbmhaYSk6MtEhxLSrgt4v0MmxvjwUCYpOJ5wIbAgUJCWYB
gACBCRDgt4v0Mmxvj3YgBBkWCAAdFiEEZW5MWsHMO4blOdl+NDY1poWakXQFAmKT
iecACgkQNDY1poWakXTwaQEAwymt4PgXltHUH8GVUB6Xu7Gb5o6LwV9fNQJc1CMl
7CABAJw0We0w1q78cJ8uWiomE1MHdRxsuqbuqtsCn2Dn6/0Cj+4A/Apcqm7uzFam
pA5u9yvz1VJBWZY1PRBICBFSkuRtacUCAQC7YNurPPoWDyjiJPrf0Vzaz8UtKp0q
BSF/a3EoocLnCA==
=APeC
-----END PGP PUBLIC KEY BLOCK-----
<domain type='kvm'>
<name>$vm_name</name>
<memory unit='MiB'>1024</memory>
<vcpu>4</vcpu>
<os>
<type arch='x86_64' machine='q35'>hvm</type>
</os>
<features>
<acpi/>
<apic/>
</features>
<cpu mode='host-passthrough'/>
<!-- https://github.com/virt-manager/virt-manager/blob/7ae10b5566ac4d8c7afd94499a9733ed42cf3d07/virtinst/domain/clock.py#L49-L59 -->
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<devices>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/$vm_name.qcow2'/>
<target dev='sdb' bus='scsi'/>
</disk>
<controller type='pci' model='pcie-root'/>
<controller type='scsi' model='virtio-scsi'/>
<controller type='usb' model='none'/>
<interface type='network'>
<source network='default'/>
<model type='virtio'/>
<filterref filter='clean-traffic'/>
</interface>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
</rng>
<video>
<model type='virtio'/>
</video>
<graphics type='vnc'/>
</devices>
</domain>
#!/usr/bin/env bash
set -o nounset -o errexit -o pipefail
readonly libvirt_default_pool_path="/var/lib/libvirt/images"
ssh() {
command ssh \
-i "/etc/libvirt-executor/id_ed25519" \
-F /dev/null \
-o ServerAliveCountMax=2 \
-o ServerAliveInterval=15 \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=off \
"root@${1}" "${@:2}"
}
vm_name() {
printf 'runner-%s-project-%d-pipeline-%d-job-%d\n' "${CUSTOM_ENV_CI_RUNNER_SHORT_TOKEN}" "${CUSTOM_ENV_CI_PROJECT_ID}" "${CUSTOM_ENV_CI_PIPELINE_IID}" "${CUSTOM_ENV_CI_JOB_ID}"
}
vm_ip() {
local ip
ip="$(virsh -q domifaddr "${1}" | awk -F'[ /]+' '{print $5}')"
if [[ -n ${ip} ]]; then
echo "${ip}"
return 0
fi
return 1
}
wait_for_ssh() {
for _ in {1..30}; do
if ! ip="$(vm_ip "${1}")"; then
echo "Waiting for network"
sleep 1
continue
fi
if ! ssh "${ip}" true; then
echo "Waiting for SSH to be ready"
sleep 1
continue
fi
printf "%s" "${ip}"
return 0
done
echo 'Waited 30 seconds for VM to start, exiting...'
exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
}
# https://docs.gitlab.com/runner/executors/custom.html#prepare
prepare() {
# shellcheck disable=SC2064
trap "exit ${SYSTEM_FAILURE_EXIT_CODE:-1}" ERR
local base_image
base_image="$(compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | tail -n 1)"
if [[ -z ${base_image} ]]; then
echo 'Base image not found...'
exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
fi
qemu-img create -f qcow2 -b "${base_image}" -F qcow2 "${libvirt_default_pool_path}/$(vm_name).qcow2"
virsh define <(sed "s/\$vm_name/$(vm_name)/" /usr/local/lib/libvirt-executor/domain_template.xml)
virsh start "$(vm_name)"
wait_for_ssh "$(vm_name)"
}
# https://docs.gitlab.com/runner/executors/custom.html#run
run() {
local ip
ip="$(wait_for_ssh "$(vm_name)")"
ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
}
# https://docs.gitlab.com/runner/executors/custom.html#cleanup
cleanup() {
virsh destroy "$(vm_name)" || true
rm "${libvirt_default_pool_path}/$(vm_name).qcow2"
virsh undefine "$(vm_name)"
}
case "${1:-}" in
prepare)
prepare
;;
run)
run "${2}" "${3}"
;;
cleanup)
cleanup
;;
*)
echo "Error invalid command: ${1:-}"
exit 1;
esac
#!/bin/bash
set -o nounset -o errexit
readonly libvirt_default_pool_path="/var/lib/libvirt/images"
cleanup() {
set +o errexit
if mountpoint -q mnt; then
umount -R mnt
fi
if [[ -n ${loopdev} ]]; then
losetup -d "${loopdev}"
fi
rm -r "${tmpdir}"
}
tmpdir="$(mktemp --directory --tmpdir="/var/tmp")"
trap cleanup EXIT
cd "${tmpdir}"
curl -sSf --remote-name-all https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2{,.sig}
sq verify --signer-cert /usr/local/lib/libvirt-executor/arch-boxes.asc --detached Arch-Linux-x86_64-basic.qcow2.sig Arch-Linux-x86_64-basic.qcow2
image=Arch-Linux-x86_64-basic.img
qemu-img convert -f qcow2 -O raw Arch-Linux-x86_64-basic.qcow2 Arch-Linux-x86_64-basic.img
loopdev="$(losetup --find --partscan --show "${image}")"
mount --mkdir "${loopdev}p2" mnt
arch-chroot mnt pacman-key --init
arch-chroot mnt pacman-key --populate
# shellcheck disable=SC2016
printf 'Server = https://geo.mirror.pkgbuild.com/$repo/os/$arch' > mnt/etc/pacman.d/mirrorlist
arch-chroot mnt systemctl disable reflector-init
arch-chroot mnt pacman -Sy --noconfirm --needed archlinux-keyring
arch-chroot mnt pacman -Syu --noconfirm --needed git git-lfs gitlab-runner
sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i mnt/etc/pacman.conf
arch-chroot mnt userdel -r arch
install -d -m0700 mnt/root/.ssh
install -m0600 /etc/libvirt-executor/id_ed25519.pub mnt/root/.ssh/authorized_keys
rm -f mnt/etc/machine-id
cp -a mnt/boot/{initramfs-linux-fallback.img,initramfs-linux.img}
rm -r mnt/etc/pacman.d/gnupg/{openpgp-revocs.d,private-keys-v1.d}/
arch-chroot mnt pacman-key --delete pacman@localhost
umount mnt
losetup -d "${loopdev}"
loopdev=""
qemu-img convert -f raw -O qcow2 Arch-Linux-x86_64-basic.img Arch-Linux-x86_64-basic.qcow2
printf -v image_path '%s/runner-base-%(%s)T.qcow2' "${libvirt_default_pool_path}"
cp Arch-Linux-x86_64-basic.qcow2 "${image_path}.tmp"
mv "${image_path}"{.tmp,}
# Keep one week of base images
compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | head -n -7 | xargs --no-run-if-empty rm -vf
[Unit]
Description=Update libvirt-executor base image
Wants=network-online.target
After=network-online.target nss-lookup.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/libvirt-executor-update-base-image
[Unit]
Description=Run libvirt-executor-vm-template.service daily
[Timer]
OnCalendar=daily
Persistent=true
RandomizedDelaySec=1d
[Install]
WantedBy=timers.target
......@@ -34,7 +34,7 @@
# --non-interactive \
# --url=https://gitlab.archlinux.org/ \
# --docker-image=archlinux:latest \
# --tag-list=docker \ # Use docker,secure for secure runners
# --tag-list=docker \ # Use docker,secure for secure runners and docker,secure-vm for secure VM runners
# --registration-token="{{ vault_gitlab_runner_registration_token }}" \
# --executor=docker \
# --description="{{ inventory_hostname }}" \
......@@ -59,3 +59,37 @@
- name: enable and start gitlab runner service
systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes
- name: setup libvirt-executor
block:
- name: install libvirt-executor-update-base-image dependencies
pacman: name=arch-install-scripts,sequoia-sq state=present
- name: create libvirt-executor configuration and data directories
file: path={{ item }} state=directory owner=root group=root mode=0755
loop:
- /etc/libvirt-executor
- /usr/local/lib/libvirt-executor
- name: install libvirt-executor
copy: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode={{ item.mode }}
loop:
- {src: arch-boxes.asc, dest: /usr/local/lib/libvirt-executor/, mode: 644}
- {src: domain_template.xml, dest: /usr/local/lib/libvirt-executor/, mode: 755}
- {src: libvirt-executor, dest: /usr/local/bin/, mode: 755}
- {src: libvirt-executor-update-base-image, dest: /usr/local/bin/, mode: 755}
- name: create SSH keys for libvirt-executor
command: ssh-keygen -N "" -f /etc/libvirt-executor/id_ed25519 -t ed25519
args:
creates: /etc/libvirt-executor/id_ed25519
- name: install libvirt-executor-update-base-image.{service,timer}
copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
loop:
- libvirt-executor-update-base-image.service
- libvirt-executor-update-base-image.timer
- name: enable and start libvirt-executor-update-base-image.timer
systemd: name=libvirt-executor-update-base-image.timer state=started enabled=yes daemon_reload=yes
when: "'gitlab_vm_runners' in group_names"
......@@ -23,3 +23,24 @@ listen_address = ":9252"
disable_cache = false
volumes = ["/cache"]
shm_size = 0
{% if 'gitlab_vm_runners' in group_names %}
[[runners]]
name = "{{ inventory_hostname }}"
url = "https://gitlab.archlinux.org"
token = "{{ vault_gitlab_vm_runner_token }}"
executor = "custom"
builds_dir = "/builds"
cache_dir = "/cache"
limit = {{ (ansible_memtotal_mb * 0.9 / 1024) | round | int }}
environment = ["ARCHIVER_STAGING_DIR=/var/tmp"]
[runners.custom]
prepare_exec = "/usr/local/bin/libvirt-executor"
prepare_args = [ "prepare" ]
run_exec = "/usr/local/bin/libvirt-executor"
run_args = [ "run" ]
cleanup_exec = "/usr/local/bin/libvirt-executor"
cleanup_args = [ "cleanup" ]
{% endif %}
---
- name: remove iptables to solve iptables<->iptables-nft conflict
pacman: name=iptables force=yes state=absent
- name: install libvirt and needed optional dependencies
pacman: name=libvirt,qemu-base,dnsmasq,iptables-nft state=present
register: result
- name: reload firewalld
service: name=firewalld state=reloaded
when: result.changed
- name: autostart default network on boot
file: src=/etc/libvirt/qemu/networks/default.xml dest=/etc/libvirt/qemu/networks/autostart/default.xml state=link owner=root group=root
- name: start and enable libvirtd
systemd: name=libvirtd enabled=yes state=started daemon_reload=yes
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment