diff --git a/roles/gitlab_runner/defaults/main.yml b/roles/gitlab_runner/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..e69f3dad2362e3a0bf9be290f423bedd45e45c49 --- /dev/null +++ b/roles/gitlab_runner/defaults/main.yml @@ -0,0 +1 @@ +gitlab_runner_libvirt_vm_memory: 2048 diff --git a/roles/gitlab_runner/files/libvirt-executor b/roles/gitlab_runner/files/libvirt-executor index 7f6a2162ada9e179cb433940ff1ebb31c09d944a..9a5bd1a39c64fde173a62f7425c6604cf7f31349 100755 --- a/roles/gitlab_runner/files/libvirt-executor +++ b/roles/gitlab_runner/files/libvirt-executor @@ -1,15 +1,16 @@ #!/usr/bin/env bash set -o nounset -o errexit -o pipefail -readonly libvirt_default_pool_path="/var/lib/libvirt/images" +readonly libvirt_pool="images" ssh() { command ssh \ - -i "/etc/libvirt-executor/id_ed25519" \ + -i "/run/libvirt-executor/id_ed25519_$(vm_name)" \ -F /dev/null \ -o ServerAliveCountMax=2 \ -o ServerAliveInterval=15 \ -o UserKnownHostsFile=/dev/null \ -o StrictHostKeyChecking=off \ + -o LogLevel=error \ "root@${1}" "${@:2}" } @@ -39,7 +40,6 @@ wait_for_ssh() { sleep 1 continue fi - printf "%s" "${ip}" return 0 done echo 'Waited 30 seconds for VM to start, exiting...' @@ -50,16 +50,23 @@ wait_for_ssh() { prepare() { # shellcheck disable=SC2064 trap "exit ${SYSTEM_FAILURE_EXIT_CODE:-1}" ERR - local base_image - base_image="$(compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | tail -n 1)" - if [[ -z ${base_image} ]]; then - echo 'Base image not found...' + if [[ ! -f /usr/local/lib/libvirt-executor/backing-vol-name ]]; then + echo 'Backing volume not found...' exit "${SYSTEM_FAILURE_EXIT_CODE:-1}" fi + local backing_volume + backing_volume="$(</usr/local/lib/libvirt-executor/backing-vol-name)" - qemu-img create -f qcow2 -b "${base_image}" -F qcow2 "${libvirt_default_pool_path}/$(vm_name).qcow2" - virsh define <(sed "s/\$vm_name/$(vm_name)/" /usr/local/lib/libvirt-executor/domain_template.xml) + mkdir -p /run/libvirt-executor + chmod 700 /run/libvirt-executor + ssh-keygen -q -N "" -f /run/libvirt-executor/id_ed25519_$(vm_name) -t ed25519 + + local ssh_authorized_keys_root + ssh_authorized_keys_root="$(base64 -w 0 /run/libvirt-executor/id_ed25519_$(vm_name).pub)" + + virsh vol-create-as "${libvirt_pool}" "$(vm_name).qcow2" 0 --format qcow2 --backing-vol "${backing_volume}" --backing-vol-format qcow2 + virsh define <(sed -e "s/\$vm_name/$(vm_name)/" -e "s/\$ssh_authorized_keys_root/${ssh_authorized_keys_root}/" /usr/local/lib/libvirt-executor/domain_template.xml) virsh start "$(vm_name)" wait_for_ssh "$(vm_name)" @@ -68,15 +75,21 @@ prepare() { # https://docs.gitlab.com/runner/executors/custom.html#run run() { local ip - ip="$(wait_for_ssh "$(vm_name)")" - ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}" + ip="$(vm_ip "$(vm_name)")" + if [[ ${2} == prepare_script ]]; then + # TODO: Get this fixed upstream or perhaps we should just install inetutils? + # https://gitlab.com/gitlab-org/gitlab-runner/-/blob/v17.5.2/shells/bash.go?ref_type=tags#L452-L456 + ssh "${ip}" bash < <(sed 's/$(hostname)/$(hostnamectl hostname)/' "${1}") || exit "${BUILD_FAILURE_EXIT_CODE:-1}" + else + ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}" + fi } # https://docs.gitlab.com/runner/executors/custom.html#cleanup cleanup() { + rm /run/libvirt-executor/id_ed25519_$(vm_name){,.pub} virsh destroy "$(vm_name)" || true - rm "${libvirt_default_pool_path}/$(vm_name).qcow2" - virsh undefine "$(vm_name)" + virsh undefine --nvram --remove-all-storage "$(vm_name)" } case "${1:-}" in diff --git a/roles/gitlab_runner/files/libvirt-executor-fetch-image b/roles/gitlab_runner/files/libvirt-executor-fetch-image new file mode 100755 index 0000000000000000000000000000000000000000..756717548871d09fc075b9bb7ae3daf95b7ffbd1 --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor-fetch-image @@ -0,0 +1,34 @@ +#!/bin/bash +set -o nounset -o errexit -o pipefail +readonly libvirt_pool="images" +readonly arch_boxes_signing_key=/usr/local/lib/libvirt-executor/arch-boxes.asc +readonly arch_boxes_fingerprint=1B9A16984A4E8CB448712D2AE0B78BF4326C6F8F + +cleanup() { + rm -r "${tmpdir}" +} + +tmpdir="$(mktemp --directory --tmpdir="/var/tmp")" +trap cleanup EXIT + +cd "${tmpdir}" + +version="$(curl -sSfL 'https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/artifacts/master/raw/build.env?job=build:secure' | awk -F= '$1=="BUILD_VERSION" {print $2}')" +image_name="Arch-Linux-x86_64-libvirt-executor-${version}.qcow2" + +if cmp --quiet <(echo "${image_name}") /usr/local/lib/libvirt-executor/backing-vol-name; then + echo "Nothing to do" + exit +fi + +curl -sSfL --remote-name-all https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/artifacts/master/raw/output/${image_name}{,.sig}?job=build:secure +rsop verify "${image_name}.sig" "${arch_boxes_signing_key}" < "${image_name}" + +virsh vol-create-as "${libvirt_pool}" "${image_name}" 0 --format qcow2 +virsh vol-upload "${image_name}" "${image_name}" "${libvirt_pool}" + +echo "${image_name}" > /usr/local/lib/libvirt-executor/backing-vol-name.tmp +mv /usr/local/lib/libvirt-executor/backing-vol-name{.tmp,} + +# Keep one week of images +virsh vol-list "${libvirt_pool}" | awk '$1~"Arch-Linux-x86_64-libvirt-executor-[0-9]*\\.[0-9]*\\.qcow2" {print $1}' | sort -n -t - -k6,6 | head -n -7 | xargs -I{} --no-run-if-empty virsh vol-delete {} "${libvirt_pool}" diff --git a/roles/gitlab_runner/files/libvirt-executor-fetch-image.service b/roles/gitlab_runner/files/libvirt-executor-fetch-image.service new file mode 100644 index 0000000000000000000000000000000000000000..14ed3cf50c535f2d6e59a57835dd314d4b167c91 --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor-fetch-image.service @@ -0,0 +1,8 @@ +[Unit] +Description=Fetch libvirt-executor image +Wants=network-online.target +After=network-online.target nss-lookup.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/libvirt-executor-fetch-image diff --git a/roles/gitlab_runner/files/libvirt-executor-fetch-image.timer b/roles/gitlab_runner/files/libvirt-executor-fetch-image.timer new file mode 100644 index 0000000000000000000000000000000000000000..8ec6abbf3a4af45c97966d23339eed961b004994 --- /dev/null +++ b/roles/gitlab_runner/files/libvirt-executor-fetch-image.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run libvirt-executor-fetch-image.service daily + +[Timer] +# One hour after the "Nightly build" pipeline +# https://gitlab.archlinux.org/archlinux/arch-boxes/-/pipeline_schedules +OnCalendar=06:00 UTC +Persistent=true +RandomizedDelaySec=1h + +[Install] +WantedBy=timers.target diff --git a/roles/gitlab_runner/files/libvirt-executor-update-base-image b/roles/gitlab_runner/files/libvirt-executor-update-base-image deleted file mode 100755 index 87ce385e153992c4f23942d2ba491d5bfd41e582..0000000000000000000000000000000000000000 --- a/roles/gitlab_runner/files/libvirt-executor-update-base-image +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -o nounset -o errexit -readonly libvirt_default_pool_path="/var/lib/libvirt/images" -readonly arch_boxes_signing_key=/usr/local/lib/libvirt-executor/arch-boxes.asc -readonly arch_boxes_fingerprint=1B9A16984A4E8CB448712D2AE0B78BF4326C6F8F - -loopdev="" - -cleanup() { - set +o errexit - - if mountpoint -q mnt; then - umount -R mnt - fi - if [[ -n ${loopdev} ]]; then - losetup -d "${loopdev}" - fi - rm -r "${tmpdir}" -} - -tmpdir="$(mktemp --directory --tmpdir="/var/tmp")" -trap cleanup EXIT - -cd "${tmpdir}" -curl -sSf --remote-name-all https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2{,.sig} -rsop verify Arch-Linux-x86_64-basic.qcow2.sig "${arch_boxes_signing_key}" < Arch-Linux-x86_64-basic.qcow2 - -image=Arch-Linux-x86_64-basic.img -qemu-img convert -f qcow2 -O raw Arch-Linux-x86_64-basic.qcow2 Arch-Linux-x86_64-basic.img - -loopdev="$(losetup --find --partscan --show "${image}")" -mount --mkdir "${loopdev}p3" mnt - -arch-chroot mnt pacman-key --init -arch-chroot mnt pacman-key --populate - -arch-chroot mnt systemctl disable systemd-time-wait-sync -arch-chroot mnt pacman -Sy --noconfirm --needed archlinux-keyring -arch-chroot mnt pacman -Syu --noconfirm --needed git git-lfs gitlab-runner -sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i mnt/etc/pacman.conf -arch-chroot mnt userdel -r arch -sed 's/^\(GRUB_CMDLINE_LINUX=".*\)"$/\1 lockdown=confidentiality"/' -i mnt/etc/default/grub -arch-chroot mnt /usr/bin/grub-mkconfig -o /boot/grub/grub.cfg -install -d -m0700 mnt/root/.ssh -install -m0600 /etc/libvirt-executor/id_ed25519.pub mnt/root/.ssh/authorized_keys -rm -f mnt/etc/machine-id - -cp -a mnt/boot/{initramfs-linux-fallback.img,initramfs-linux.img} - -rm -r mnt/etc/pacman.d/gnupg/{openpgp-revocs.d,private-keys-v1.d}/ -arch-chroot mnt pacman-key --delete pacman@localhost -umount mnt -losetup -d "${loopdev}" -loopdev="" - -qemu-img convert -f raw -O qcow2 Arch-Linux-x86_64-basic.img Arch-Linux-x86_64-basic.qcow2 -printf -v image_path '%s/runner-base-%(%s)T.qcow2' "${libvirt_default_pool_path}" -cp Arch-Linux-x86_64-basic.qcow2 "${image_path}.tmp" -mv "${image_path}"{.tmp,} - -# Keep one week of base images -compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | head -n -7 | xargs --no-run-if-empty rm -vf diff --git a/roles/gitlab_runner/files/libvirt-executor-update-base-image.service b/roles/gitlab_runner/files/libvirt-executor-update-base-image.service deleted file mode 100644 index b28f92fc30128aae1e57ab4772d43fddbf6e2b3a..0000000000000000000000000000000000000000 --- a/roles/gitlab_runner/files/libvirt-executor-update-base-image.service +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Update libvirt-executor base image -Wants=network-online.target -After=network-online.target nss-lookup.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/libvirt-executor-update-base-image diff --git a/roles/gitlab_runner/files/libvirt-executor-update-base-image.timer b/roles/gitlab_runner/files/libvirt-executor-update-base-image.timer deleted file mode 100644 index 4cb8f17bb7c8ca7c133adc5c297e1bd27ab2f9ea..0000000000000000000000000000000000000000 --- a/roles/gitlab_runner/files/libvirt-executor-update-base-image.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Run libvirt-executor-update-base-image.service daily - -[Timer] -OnCalendar=daily -Persistent=true -RandomizedDelaySec=1d - -[Install] -WantedBy=timers.target diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml index 08938213bc0eb2413bcc530f754e9e71640a8b40..60e468c84fa90fbeea032ba6acbd97bbd1fe2540 100644 --- a/roles/gitlab_runner/tasks/main.yml +++ b/roles/gitlab_runner/tasks/main.yml @@ -66,33 +66,27 @@ - name: Setup libvirt-executor when: "'gitlab_vm_runners' in group_names" block: - - name: Install libvirt-executor-update-base-image dependencies - pacman: name=arch-install-scripts,rsop state=present + - name: Install libvirt-executor-fetch-image dependencies + pacman: name=rsop state=present - - name: Create libvirt-executor configuration and data directories - file: path={{ item }} state=directory owner=root group=root mode=0755 - loop: - - /etc/libvirt-executor - - /usr/local/lib/libvirt-executor + - name: Create libvirt-executor data directory + file: path=/usr/local/lib/libvirt-executor state=directory owner=root group=root mode=0755 - name: Install libvirt-executor copy: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode={{ item.mode }} loop: - {src: arch-boxes.asc, dest: /usr/local/lib/libvirt-executor/, mode: 644} - - {src: domain_template.xml, dest: /usr/local/lib/libvirt-executor/, mode: 755} - {src: libvirt-executor, dest: /usr/local/bin/, mode: 755} - - {src: libvirt-executor-update-base-image, dest: /usr/local/bin/, mode: 755} + - {src: libvirt-executor-fetch-image, dest: /usr/local/bin/, mode: 755} - - name: Create SSH keys for libvirt-executor - command: ssh-keygen -N "" -f /etc/libvirt-executor/id_ed25519 -t ed25519 - args: - creates: /etc/libvirt-executor/id_ed25519 + - name: Install libvirt-executor domain template + template: src=domain_template.xml.j2 dest=/usr/local/lib/libvirt-executor/domain_template.xml owner=root group=root mode=0644 - - name: Install libvirt-executor-update-base-image.{service,timer} + - name: Install libvirt-executor-fetch-image.{service,timer} copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644 loop: - - libvirt-executor-update-base-image.service - - libvirt-executor-update-base-image.timer + - libvirt-executor-fetch-image.service + - libvirt-executor-fetch-image.timer - - name: Enable and start libvirt-executor-update-base-image.timer - systemd: name=libvirt-executor-update-base-image.timer state=started enabled=yes daemon_reload=yes + - name: Enable and start libvirt-executor-fetch-image.timer + systemd: name=libvirt-executor-fetch-image.timer state=started enabled=yes daemon_reload=yes diff --git a/roles/gitlab_runner/templates/config.toml.j2 b/roles/gitlab_runner/templates/config.toml.j2 index 6f8fee71356da7743c7145f181c65f71fcbab712..03cc16f4c3aa95134484b0c17f588b64d86a5a3e 100644 --- a/roles/gitlab_runner/templates/config.toml.j2 +++ b/roles/gitlab_runner/templates/config.toml.j2 @@ -32,7 +32,7 @@ listen_address = ":9252" executor = "custom" builds_dir = "/builds" cache_dir = "/cache" - limit = {{ (ansible_memtotal_mb * 0.9 / 2048) | round | int }} + limit = {{ (ansible_memtotal_mb * 0.9 / gitlab_runner_libvirt_vm_memory) | round | int }} environment = ["ARCHIVER_STAGING_DIR=/var/tmp"] [runners.custom] prepare_exec = "/usr/local/bin/libvirt-executor" diff --git a/roles/gitlab_runner/files/domain_template.xml b/roles/gitlab_runner/templates/domain_template.xml.j2 similarity index 70% rename from roles/gitlab_runner/files/domain_template.xml rename to roles/gitlab_runner/templates/domain_template.xml.j2 index 80ca498284c1da0eff126febd10bf3d7666f878b..a7f6b7bacbd848c64f800a029c5a27a11df0f8ef 100644 --- a/roles/gitlab_runner/files/domain_template.xml +++ b/roles/gitlab_runner/templates/domain_template.xml.j2 @@ -1,9 +1,16 @@ <domain type='kvm'> <name>$vm_name</name> - <memory unit='MiB'>2048</memory> + <memory unit='MiB'>{{ gitlab_runner_libvirt_vm_memory }}</memory> <vcpu>4</vcpu> - <os> + <sysinfo type='smbios'> + <oemStrings> + <entry>io.systemd.credential:system.hostname=$vm_name</entry> + <entry>io.systemd.credential.binary:ssh.authorized_keys.root=$ssh_authorized_keys_root</entry> + </oemStrings> + </sysinfo> + <os firmware='efi'> <type arch='x86_64' machine='q35'>hvm</type> + <smbios mode='sysinfo'/> </os> <features> <acpi/> @@ -18,7 +25,7 @@ </clock> <devices> <disk type='file' device='disk'> - <driver name='qemu' type='qcow2'/> + <driver name='qemu' type='qcow2' discard='unmap'/> <source file='/var/lib/libvirt/images/$vm_name.qcow2'/> <target dev='sdb' bus='scsi'/> </disk> @@ -30,6 +37,7 @@ <model type='virtio'/> <filterref filter='clean-traffic'/> </interface> + <memballoon model='virtio' freePageReporting='on'/> <rng model='virtio'> <backend model='random'>/dev/urandom</backend> </rng> diff --git a/roles/libvirt/files/images.xml b/roles/libvirt/files/images.xml new file mode 100644 index 0000000000000000000000000000000000000000..6222186e178294ab0bddb1ac024bd62a0c12756c --- /dev/null +++ b/roles/libvirt/files/images.xml @@ -0,0 +1,6 @@ +<pool type='dir'> + <name>images</name> + <target> + <path>/var/lib/libvirt/images</path> + </target> +</pool> diff --git a/roles/libvirt/tasks/main.yml b/roles/libvirt/tasks/main.yml index ae8863b51d2d10525c6c8904a81c875277728db9..9307576fbedb1e454a7f65a363c54eddfa4c0737 100644 --- a/roles/libvirt/tasks/main.yml +++ b/roles/libvirt/tasks/main.yml @@ -12,6 +12,8 @@ - qemu-base - qemu-hw-display-virtio-gpu - qemu-hw-display-virtio-vga + - libvirt-python + - python-lxml register: result - name: Reload firewalld @@ -23,3 +25,19 @@ - name: Start and enable libvirtd systemd: name=libvirtd enabled=yes state=started daemon_reload=yes + +- name: Define the images storage pool + community.libvirt.virt_pool: + command: define + name: images + xml: "{{ lookup('file', 'images.xml') }}" + +- name: Start the image storage pool + community.libvirt.virt_pool: + state: active + name: images + +- name: Start the image storage pool at boot + community.libvirt.virt_pool: + autostart: true + name: images