libvirt-executor improvements

A bunch of improvements to the libvirt-executor, with the major one being that the image is now built in the arch-boxes projects (arch-boxes!200 (merged)), instead of on each runner.

Please see the commit messages for more details.

---

Deployed on runner1.archlinux.org and secure-runner1.archlinux.org.

roles/gitlab_runner/files/libvirt-executor

 #!/usr/bin/env bash
 set -o nounset -o errexit -o pipefail
-readonly libvirt_default_pool_path="/var/lib/libvirt/images"
+readonly libvirt_pool="images"
 
 ssh() {
   command ssh \
-    -i "/etc/libvirt-executor/id_ed25519" \
+    -i "/run/libvirt-executor/id_ed25519_$(vm_name)" \
     -F /dev/null \
     -o ServerAliveCountMax=2 \
     -o ServerAliveInterval=15 \
     -o UserKnownHostsFile=/dev/null \
     -o StrictHostKeyChecking=off \
+    -o LogLevel=error \
     "root@${1}" "${@:2}"
 }
 
@@ -39,7 +40,6 @@ wait_for_ssh() {
       sleep 1
       continue
     fi
-    printf "%s" "${ip}"
     return 0
   done
   echo 'Waited 30 seconds for VM to start, exiting...'
@@ -50,16 +50,23 @@ wait_for_ssh() {
 prepare() {
   # shellcheck disable=SC2064
   trap "exit ${SYSTEM_FAILURE_EXIT_CODE:-1}" ERR
-  local base_image
-  base_image="$(compgen -G "${libvirt_default_pool_path}/runner-base-*.qcow2" | sort -n -t - -k3,3 | tail -n 1)"
 
-  if [[ -z ${base_image} ]]; then
-    echo 'Base image not found...'
+  if [[ ! -f /usr/local/lib/libvirt-executor/backing-vol-name ]]; then
+    echo 'Backing volume not found...'
     exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
   fi
+  local backing_volume
+  backing_volume="$(</usr/local/lib/libvirt-executor/backing-vol-name)"
 
-  qemu-img create -f qcow2 -b "${base_image}" -F qcow2 "${libvirt_default_pool_path}/$(vm_name).qcow2"
-  virsh define <(sed "s/\$vm_name/$(vm_name)/" /usr/local/lib/libvirt-executor/domain_template.xml)
+  mkdir -p /run/libvirt-executor
+  chmod 700 /run/libvirt-executor
+  ssh-keygen -q -N "" -f /run/libvirt-executor/id_ed25519_$(vm_name) -t ed25519
+
+  local ssh_authorized_keys_root
+  ssh_authorized_keys_root="$(base64 -w 0 /run/libvirt-executor/id_ed25519_$(vm_name).pub)"
+
+  virsh vol-create-as "${libvirt_pool}" "$(vm_name).qcow2" 0 --format qcow2 --backing-vol "${backing_volume}" --backing-vol-format qcow2
+  virsh define <(sed -e "s/\$vm_name/$(vm_name)/" -e "s/\$ssh_authorized_keys_root/${ssh_authorized_keys_root}/" /usr/local/lib/libvirt-executor/domain_template.xml)
   virsh start "$(vm_name)"
 
   wait_for_ssh "$(vm_name)"
@@ -68,15 +75,21 @@ prepare() {
 # https://docs.gitlab.com/runner/executors/custom.html#run
 run() {
   local ip
-  ip="$(wait_for_ssh "$(vm_name)")"
-  ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  ip="$(vm_ip "$(vm_name)")"
+  if [[ ${2} == prepare_script ]]; then
+    # TODO: Get this fixed upstream or perhaps we should just install inetutils?
+    # https://gitlab.com/gitlab-org/gitlab-runner/-/blob/v17.5.2/shells/bash.go?ref_type=tags#L452-L456
+    ssh "${ip}" bash < <(sed 's/$(hostname)/$(hostnamectl hostname)/' "${1}") || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  else
+    ssh "${ip}" bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  fi
 }
 
 # https://docs.gitlab.com/runner/executors/custom.html#cleanup
 cleanup() {
+  rm /run/libvirt-executor/id_ed25519_$(vm_name){,.pub}
   virsh destroy "$(vm_name)" || true
-  rm "${libvirt_default_pool_path}/$(vm_name).qcow2"
-  virsh undefine "$(vm_name)"
+  virsh undefine --nvram --remove-all-storage "$(vm_name)"
 }
 
 case "${1:-}" in