Merge branch 'libvirt-executor-improvements' into 'master'

libvirt-executor improvements

See merge request !881

roles/gitlab_runner/defaults/main.yml

+gitlab_runner_libvirt_vm_memory: 2048

roles/gitlab_runner/files/libvirt-executor

 #!/usr/bin/env bash
 set -o nounset -o errexit -o pipefail
-readonly libvirt_default_pool_path="/var/lib/libvirt/images"
+readonly libvirt_pool="images"
 
 ssh() {
   command ssh \
-    -i "/etc/libvirt-executor/id_ed25519" \
+    -i "/run/libvirt-executor/id_ed25519_$(vm_name)" \
     -F /dev/null \
     -o ServerAliveCountMax=2 \
     -o ServerAliveInterval=15 \
     -o UserKnownHostsFile=/dev/null \
     -o StrictHostKeyChecking=off \
+    -o LogLevel=error \
     "root@${1}" "${@:2}"
 }
 
@@ -39,7 +40,6 @@ wait_for_ssh() {
       sleep 1
       continue
     fi
-    printf "%s" "${ip}"
     return 0
   done
   echo 'Waited 30 seconds for VM to start, exiting...'
@@ -50,16 +50,23 @@ wait_for_ssh() {
 prepare() {
   # shellcheck disable=SC2064
   trap "exit ${SYSTEM_FAILURE_EXIT_CODE:-1}" ERR
-  local base_image
-  base_image="$(compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | tail -n 1)"
 
-  if [[ -z ${base_image} ]]; then
-    echo 'Base image not found...'
+  if [[ ! -f /usr/local/lib/libvirt-executor/backing-vol-name ]]; then
+    echo 'Backing volume not found...'
     exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
   fi
+  local backing_volume
+  backing_volume="$(</usr/local/lib/libvirt-executor/backing-vol-name)"
 
-  qemu-img create -f qcow2 -b "${base_image}" -F qcow2 "${libvirt_default_pool_path}/$(vm_name).qcow2"
-  virsh define <(sed "s/\$vm_name/$(vm_name)/" /usr/local/lib/libvirt-executor/domain_template.xml)
+  mkdir -p /run/libvirt-executor
+  chmod 700 /run/libvirt-executor
+  ssh-keygen -q -N "" -f /run/libvirt-executor/id_ed25519_$(vm_name) -t ed25519
+
+  local ssh_authorized_keys_root
+  ssh_authorized_keys_root="$(base64 -w 0 /run/libvirt-executor/id_ed25519_$(vm_name).pub)"
+
+  virsh vol-create-as "${libvirt_pool}" "$(vm_name).qcow2" 0 --format qcow2 --backing-vol "${backing_volume}" --backing-vol-format qcow2
+  virsh define <(sed -e "s/\$vm_name/$(vm_name)/" -e "s/\$ssh_authorized_keys_root/${ssh_authorized_keys_root}/" /usr/local/lib/libvirt-executor/domain_template.xml)
   virsh start "$(vm_name)"
 
   wait_for_ssh "$(vm_name)"
@@ -68,15 +75,21 @@ prepare() {
 # https://docs.gitlab.com/runner/executors/custom.html#run
 run() {
   local ip
-  ip="$(wait_for_ssh "$(vm_name)")"
-  ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  ip="$(vm_ip "$(vm_name)")"
+  if [[ ${2} == prepare_script ]]; then
+    # TODO: Get this fixed upstream or perhaps we should just install inetutils?
+    # https://gitlab.com/gitlab-org/gitlab-runner/-/blob/v17.5.2/shells/bash.go?ref_type=tags#L452-L456
+    ssh "${ip}" bash < <(sed 's/$(hostname)/$(hostnamectl hostname)/' "${1}") || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  else
+    ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  fi
 }
 
 # https://docs.gitlab.com/runner/executors/custom.html#cleanup
 cleanup() {
+  rm /run/libvirt-executor/id_ed25519_$(vm_name){,.pub}
   virsh destroy "$(vm_name)" || true
-  rm "${libvirt_default_pool_path}/$(vm_name).qcow2"
-  virsh undefine "$(vm_name)"
+  virsh undefine --nvram --remove-all-storage "$(vm_name)"
 }
 
 case "${1:-}" in

roles/gitlab_runner/files/libvirt-executor-fetch-image

+#!/bin/bash
+set -o nounset -o errexit -o pipefail
+readonly libvirt_pool="images"
+readonly arch_boxes_signing_key=/usr/local/lib/libvirt-executor/arch-boxes.asc
+readonly arch_boxes_fingerprint=1B9A16984A4E8CB448712D2AE0B78BF4326C6F8F
+
+cleanup() {
+  rm -r "${tmpdir}"
+}
+
+tmpdir="$(mktemp --directory --tmpdir="/var/tmp")"
+trap cleanup EXIT
+
+cd "${tmpdir}"
+
+version="$(curl -sSfL 'https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/artifacts/master/raw/build.env?job=build:secure' | awk -F= '$1=="BUILD_VERSION" {print $2}')"
+image_name="Arch-Linux-x86_64-libvirt-executor-${version}.qcow2"
+
+if cmp --quiet <(echo "${image_name}") /usr/local/lib/libvirt-executor/backing-vol-name; then
+  echo "Nothing to do"
+  exit
+fi
+
+curl -sSfL --remote-name-all https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/artifacts/master/raw/output/${image_name}{,.sig}?job=build:secure
+rsop verify "${image_name}.sig" "${arch_boxes_signing_key}" < "${image_name}"
+
+virsh vol-create-as "${libvirt_pool}" "${image_name}" 0 --format qcow2
+virsh vol-upload "${image_name}" "${image_name}" "${libvirt_pool}"
+
+echo "${image_name}" > /usr/local/lib/libvirt-executor/backing-vol-name.tmp
+mv /usr/local/lib/libvirt-executor/backing-vol-name{.tmp,}
+
+# Keep one week of images
+virsh vol-list "${libvirt_pool}" | awk '$1~"Arch-Linux-x86_64-libvirt-executor-[0-9]*\\.[0-9]*\\.qcow2" {print $1}' | sort -n -t - -k6,6 | head -n -7 | xargs -I{} --no-run-if-empty virsh vol-delete {} "${libvirt_pool}"

roles/gitlab_runner/files/libvirt-executor-update-base-image.service → roles/gitlab_runner/files/libvirt-executor-fetch-image.service

 [Unit]
-Description=Update libvirt-executor base image
+Description=Fetch libvirt-executor image
 Wants=network-online.target
 After=network-online.target nss-lookup.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/libvirt-executor-update-base-image
+ExecStart=/usr/local/bin/libvirt-executor-fetch-image

roles/gitlab_runner/files/libvirt-executor-fetch-image.timer

+[Unit]
+Description=Run libvirt-executor-fetch-image.service daily
+
+[Timer]
+# One hour after the "Nightly build" pipeline
+# https://gitlab.archlinux.org/archlinux/arch-boxes/-/pipeline_schedules
+OnCalendar=06:00 UTC
+Persistent=true
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target

roles/gitlab_runner/files/libvirt-executor-update-base-image

-#!/bin/bash
-set -o nounset -o errexit
-readonly libvirt_default_pool_path="/var/lib/libvirt/images"
-readonly arch_boxes_signing_key=/usr/local/lib/libvirt-executor/arch-boxes.asc
-readonly arch_boxes_fingerprint=1B9A16984A4E8CB448712D2AE0B78BF4326C6F8F
-
-loopdev=""
-
-cleanup() {
-  set +o errexit
-
-  if mountpoint -q mnt; then
-    umount -R mnt
-  fi
-  if [[ -n ${loopdev} ]]; then
-    losetup -d "${loopdev}"
-  fi
-  rm -r "${tmpdir}"
-}
-
-tmpdir="$(mktemp --directory --tmpdir="/var/tmp")"
-trap cleanup EXIT
-
-cd "${tmpdir}"
-curl -sSf --remote-name-all https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2{,.sig}
-rsop verify Arch-Linux-x86_64-basic.qcow2.sig "${arch_boxes_signing_key}" < Arch-Linux-x86_64-basic.qcow2
-
-image=Arch-Linux-x86_64-basic.img
-qemu-img convert -f qcow2 -O raw Arch-Linux-x86_64-basic.qcow2 Arch-Linux-x86_64-basic.img
-
-loopdev="$(losetup --find --partscan --show "${image}")"
-mount --mkdir "${loopdev}p3" mnt
-
-arch-chroot mnt pacman-key --init
-arch-chroot mnt pacman-key --populate
-
-arch-chroot mnt systemctl disable systemd-time-wait-sync
-arch-chroot mnt pacman -Sy --noconfirm --needed archlinux-keyring
-arch-chroot mnt pacman -Syu --noconfirm --needed git git-lfs gitlab-runner
-sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i mnt/etc/pacman.conf
-arch-chroot mnt userdel -r arch
-sed 's/^\(GRUB_CMDLINE_LINUX=".*\)"$/\1 lockdown=confidentiality"/' -i mnt/etc/default/grub
-arch-chroot mnt /usr/bin/grub-mkconfig -o /boot/grub/grub.cfg
-install -d -m0700 mnt/root/.ssh
-install -m0600 /etc/libvirt-executor/id_ed25519.pub mnt/root/.ssh/authorized_keys
-rm -f mnt/etc/machine-id
-
-cp -a mnt/boot/{initramfs-linux-fallback.img,initramfs-linux.img}
-
-rm -r mnt/etc/pacman.d/gnupg/{openpgp-revocs.d,private-keys-v1.d}/
-arch-chroot mnt pacman-key --delete pacman@localhost
-umount mnt
-losetup -d "${loopdev}"
-loopdev=""
-
-qemu-img convert -f raw -O qcow2 Arch-Linux-x86_64-basic.img Arch-Linux-x86_64-basic.qcow2
-printf -v image_path '%s/runner-base-%(%s)T.qcow2' "${libvirt_default_pool_path}"
-cp Arch-Linux-x86_64-basic.qcow2 "${image_path}.tmp"
-mv "${image_path}"{.tmp,}
-
-# Keep one week of base images
-compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | head -n -7 | xargs --no-run-if-empty rm -vf

roles/gitlab_runner/files/libvirt-executor-update-base-image.timer

-[Unit]
-Description=Run libvirt-executor-update-base-image.service daily
-
-[Timer]
-OnCalendar=daily
-Persistent=true
-RandomizedDelaySec=1d
-
-[Install]
-WantedBy=timers.target

roles/gitlab_runner/tasks/main.yml

@@ -66,33 +66,27 @@
 - name: Setup libvirt-executor
   when: "'gitlab_vm_runners' in group_names"
   block:
-    - name: Install libvirt-executor-update-base-image dependencies
-      pacman: name=arch-install-scripts,rsop state=present
+    - name: Install libvirt-executor-fetch-image dependencies
+      pacman: name=rsop state=present
 
-    - name: Create libvirt-executor configuration and data directories
-      file: path={{ item }} state=directory owner=root group=root mode=0755
-      loop:
-        - /etc/libvirt-executor
-        - /usr/local/lib/libvirt-executor
+    - name: Create libvirt-executor data directory
+      file: path=/usr/local/lib/libvirt-executor state=directory owner=root group=root mode=0755
 
     - name: Install libvirt-executor
       copy: src={{ item.src }} dest={{ item.dest }} owner=root group=root mode={{ item.mode }}
       loop:
         - {src: arch-boxes.asc, dest: /usr/local/lib/libvirt-executor/, mode: 644}
-        - {src: domain_template.xml, dest: /usr/local/lib/libvirt-executor/, mode: 755}
         - {src: libvirt-executor, dest: /usr/local/bin/, mode: 755}
-        - {src: libvirt-executor-update-base-image, dest: /usr/local/bin/, mode: 755}
+        - {src: libvirt-executor-fetch-image, dest: /usr/local/bin/, mode: 755}
 
-    - name: Create SSH keys for libvirt-executor
-      command: ssh-keygen -N "" -f /etc/libvirt-executor/id_ed25519 -t ed25519
-      args:
-        creates: /etc/libvirt-executor/id_ed25519
+    - name: Install libvirt-executor domain template
+      template: src=domain_template.xml.j2 dest=/usr/local/lib/libvirt-executor/domain_template.xml owner=root group=root mode=0644
 
-    - name: Install libvirt-executor-update-base-image.{service,timer}
+    - name: Install libvirt-executor-fetch-image.{service,timer}
       copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
       loop:
-        - libvirt-executor-update-base-image.service
-        - libvirt-executor-update-base-image.timer
+        - libvirt-executor-fetch-image.service
+        - libvirt-executor-fetch-image.timer
 
-    - name: Enable and start libvirt-executor-update-base-image.timer
-      systemd: name=libvirt-executor-update-base-image.timer state=started enabled=yes daemon_reload=yes
+    - name: Enable and start libvirt-executor-fetch-image.timer
+      systemd: name=libvirt-executor-fetch-image.timer state=started enabled=yes daemon_reload=yes

roles/gitlab_runner/templates/config.toml.j2

@@ -32,7 +32,7 @@ listen_address = ":9252"
   executor = "custom"
   builds_dir = "/builds"
   cache_dir = "/cache"
-  limit = {{ (ansible_memtotal_mb * 0.9 / 2048) | round | int }}
+  limit = {{ (ansible_memtotal_mb * 0.9 / gitlab_runner_libvirt_vm_memory) | round | int }}
   environment = ["ARCHIVER_STAGING_DIR=/var/tmp"]
   [runners.custom]
     prepare_exec = "/usr/local/bin/libvirt-executor"

roles/gitlab_runner/files/domain_template.xml → roles/gitlab_runner/templates/domain_template.xml.j2

 <domain type='kvm'>
   <name>$vm_name</name>
-  <memory unit='MiB'>2048</memory>
+  <memory unit='MiB'>{{ gitlab_runner_libvirt_vm_memory }}</memory>
   <vcpu>4</vcpu>
-  <os>
+  <sysinfo type='smbios'>
+    <oemStrings>
+      <entry>io.systemd.credential:system.hostname=$vm_name</entry>
+      <entry>io.systemd.credential.binary:ssh.authorized_keys.root=$ssh_authorized_keys_root</entry>
+    </oemStrings>
+  </sysinfo>
+  <os firmware='efi'>
     <type arch='x86_64' machine='q35'>hvm</type>
+    <smbios mode='sysinfo'/>
   </os>
   <features>
     <acpi/>
@@ -18,7 +25,7 @@
   </clock>
   <devices>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='qcow2'/>
+      <driver name='qemu' type='qcow2' discard='unmap'/>
       <source file='/var/lib/libvirt/images/$vm_name.qcow2'/>
       <target dev='sdb' bus='scsi'/>
     </disk>
@@ -30,6 +37,7 @@
       <model type='virtio'/>
       <filterref filter='clean-traffic'/>
     </interface>
+    <memballoon model='virtio' freePageReporting='on'/>
     <rng model='virtio'>
       <backend model='random'>/dev/urandom</backend>
     </rng>

roles/libvirt/files/images.xml

+<pool type='dir'>
+  <name>images</name>
+  <target>
+    <path>/var/lib/libvirt/images</path>
+  </target>
+</pool>

roles/libvirt/tasks/main.yml

@@ -12,6 +12,8 @@
       - qemu-base
       - qemu-hw-display-virtio-gpu
       - qemu-hw-display-virtio-vga
+      - libvirt-python
+      - python-lxml
   register: result
 
 - name: Reload firewalld
@@ -23,3 +25,19 @@
 
 - name: Start and enable libvirtd
   systemd: name=libvirtd enabled=yes state=started daemon_reload=yes
+
+- name: Define the images storage pool
+  community.libvirt.virt_pool:
+    command: define
+    name: images
+    xml: "{{ lookup('file', 'images.xml') }}"
+
+- name: Start the image storage pool
+  community.libvirt.virt_pool:
+    state: active
+    name: images
+
+- name: Start the image storage pool at boot
+  community.libvirt.virt_pool:
+    autostart: true
+    name: images