gitlab_runner: Add VM based executor (libvirt-executor)

For some workloads running in a container is too restrictive, ex:
arch-boxes (loop device, filesystem mount, pacstrap) and archiso
(pacstrap). Currently they both run a TCG accelerated QEMU VM, which is
very slow and painful to work with. We should provide a better option to
our users!

This adds a hardware accelerated VM for this kinds of workloads, which
is way faster and you can do whatever you like (mostly)!

Fix #283

---

TODO:

• Enable KSM?
• Get archiso to build
  • Done: https://gitlab.archlinux.org/klausenbusk/archiso/-/pipelines/10976
• Use BUILD_FAILURE_EXIT_CODE and SYSTEM_FAILURE_EXIT_CODE: https://docs.gitlab.com/runner/executors/custom.html#error-handling
• Fix existing users to use the secure-vm tag

roles/gitlab_runner/files/libvirt-executor

@@ -19,6 +19,18 @@ get_vm_name() {
   printf 'libvirt_executor_runner_%s_project-%s_concurrent_%s\n' "${CUSTOM_ENV_CI_RUNNER_SHORT_TOKEN}" "${CUSTOM_ENV_CI_PROJECT_ID}" "${CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID}"
 }
 
+clone_vm() {
+  for _ in {1..10}; do
+    # --reflink sadly doesn't work with non-raw formts:
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1324006
+    if virt-clone -o "${1}" -n "${2}" --auto-clone; then
+      return 0
+    fi
+    sleep 1
+  done
+  return 1
+}
+
 wait_for_ssh() {
   for _ in {1..90}; do
     if ! get_vm_ip "${1}"; then
@@ -54,10 +66,11 @@ create_vm_template() {
   local vm_name
   printf -v vm_name 'libvirt_executor_vm_template_%(%s)T_tmp'
   local image_path="${LIBVIRT_DEFAULT_POOL_PATH}/${vm_name}.qcow2"
-  trap "rm -f -- ${image_path}" EXIT
+  trap 'rm -f -- "${image_path}"' EXIT
   curl -sSf "${MIRROR}/images/v20210815.31636/Arch-Linux-x86_64-cloudimg-20210815.31636.qcow2" --output "${image_path}"
   qemu-img resize "${image_path}" 10G
-  local tmp_user_data="$(mktemp -u)"
+  local tmp_user_data
+  tmp_user_data="$(mktemp -u)"
   trap 'rm -f -- "$tmp_user_data"; virsh destroy "${vm_name}"; virsh undefine "${vm_name}" --remove-all-storage; exit 1' EXIT
   sed "s:PUBLIC_SSH_KEY:$(< "${STATE_DIR}/id_rsa.pub"):" "${STATE_DIR}/user-data" > "${tmp_user_data}"
   virt-install --name "${vm_name}" \
@@ -77,7 +90,6 @@ create_vm_template() {
   ssh pacman -Sy --noconfirm --needed archlinux-keyring
   ssh pacman -Syu --noconfirm git git-lfs gitlab-runner
   ssh "sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i /etc/pacman.conf"
-  ssh rm /etc/machine-id /var/lib/dbus/machine-id
 
   # Reboot to be sure the network is working
   virsh shutdown "${vm_name}"
@@ -85,6 +97,7 @@ create_vm_template() {
   virsh start "${vm_name}"
   vm_ip=""
   wait_for_ssh "${vm_name}"
+  ssh rm /etc/machine-id /var/lib/dbus/machine-id
 
   virsh shutdown "${vm_name}"
   wait_for_vm_shutdown "${vm_name}"
@@ -100,9 +113,7 @@ prepare() {
     exit 1
   fi
   vm_name="$(get_vm_name)"
-  # --reflink sadly doesn't work with non-raw formts:
-  # https://bugzilla.redhat.com/show_bug.cgi?id=1324006
-  virt-clone -o "${vm_template}" -n "${vm_name}" --auto-clone
+  clone_vm "${vm_template}" "${vm_name}"
   virsh start "${vm_name}"
   wait_for_ssh "${vm_name}"
 }
@@ -111,7 +122,11 @@ prepare() {
 run() {
   vm_name="$(get_vm_name)"
   wait_for_ssh "${vm_name}"
-  ssh bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  if [[ "${2:-}" == 'upload_artifacts_'* ]]; then
+    ssh 'TMPDIR=/var/tmp bash' < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  else
+    ssh bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+  fi
 }
 
 # https://docs.gitlab.com/runner/executors/custom.html#cleanup
@@ -129,7 +144,7 @@ case "${1:-}" in
     prepare
     ;;
   run)
-    run "${2}"
+    run "${2}" "${3}"
     ;;
   cleanup)
     cleanup