gitlab_runner: Add VM based executor (libvirt-executor)

For some workloads running in a container is too restrictive, ex:
arch-boxes (loop device, filesystem mount, pacstrap) and archiso
(pacstrap). Currently they both run a TCG accelerated QEMU VM, which is
very slow and painful to work with. We should provide a better option to
our users!

This adds a hardware accelerated VM for this kinds of workloads, which
is way faster and you can do whatever you like (mostly)!

Fix #283

---

TODO:

• Enable KSM?
• Get archiso to build
  • Done: https://gitlab.archlinux.org/klausenbusk/archiso/-/pipelines/10976
• Use BUILD_FAILURE_EXIT_CODE and SYSTEM_FAILURE_EXIT_CODE: https://docs.gitlab.com/runner/executors/custom.html#error-handling
• Fix existing users to use the secure-vm tag

roles/gitlab_runner/files/libvirt-executor

+#!/bin/bash
+set -o nounset -o errexit -o pipefail
+readonly MIRROR="https://mirror.pkgbuild.com"
+readonly LIBVIRT_DEFAULT_POOL_PATH="/var/lib/libvirt/images"
+
+ssh() {
+    command ssh -i foo_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=off "root@${vm_ip}"  "${@}"
+}
+
+get_vm_ip() {
+    if [[ -z ${vm_ip-} ]]; then
+        vm_ip="$(virsh -q domifaddr "${1}" | awk -F'[ /]+' '{print $5}')"
+        if [[ -z ${vm_ip} ]]; then
+            return 1
+        fi
+    fi
+    echo $vm_ip
+}
+
+get_vm_name() {
+    return "libvirt_executor_runner_${CUSTOM_ENV_CI_RUNNER_SHORT_TOKEN}_project-${CUSTOM_ENV_CI_PROJECT_ID}-concurrent_${CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID}"
+}
+
+wait_for_ssh() {
+  for i in {1..60}; do
+    if [ "${i}" -eq 60 ]; then
+      echo 'Waited 60 seconds for VM to start, exiting...'
+      exit "${SYSTEM_FAILURE_EXIT_CODE:-1}"
+    fi
+#    if ! { VM_IP="$(virsh -q domifaddr "${1}" | awk -F'[ /]+' '{print $5}')" && [ -n "${VM_IP}" ]; }; then
+    if ! get_vm_ip "${1}"; then
+      echo "Waiting for network"
+      sleep 1
+      continue
+    fi
+    if ! ssh true; then
+      echo "Waiting for SSH to be ready"
+      sleep 1
+      continue
+    fi
+    break
+  done
+}
+
+wait_for_vm_shutdown() {
+    for i in {1..10}; do
+        if [ "${i}" -eq 60 ]; then
+            return 1
+        fi
+        if LANG="" virsh domstate "${1}" | grep "shut off"; then
+            break
+        fi
+        sleep 1
+    done
+}
+
+# Create a updated VM image with the required tools
+create_vm_template() {
+    # Download from arch-boxes when
+    # https://gitlab.archlinux.org/archlinux/arch-boxes/-/merge_requests/172 is in
+    local vm_name="libvirt_executor_vm_template_$(date +%s)_tmp"
+    cp Arch-Linux-x86_64-cloudimg-20210815.31636.qcow2 "${LIBVIRT_DEFAULT_POOL_PATH}/${vm_name}.qcow2"
+    qemu-img resize "${LIBVIRT_DEFAULT_POOL_PATH}/${vm_name}.qcow2" 10G
+    # TODO remove exit 1 andre steder
+    trap "virsh destroy ${vm_name}; virsh undefine ${vm_name} --remove-all-storage; exit 1" EXIT
+    virt-install --name "${vm_name}" \
+                 --cloud-init user-data=$PWD/user-data \
+                 --disk path="${LIBVIRT_DEFAULT_POOL_PATH}/${vm_name}.qcow2",device=disk \
+                 --os-type Linux \
+                 --os-variant archlinux \
+                 --network network=default,filterref.filter=clean-traffic \
+                 --noautoconsole
+    wait_for_ssh "${vm_name}"
+    # Reboot to be sure the network is working
+    virsh shutdown "${vm_name}"
+    wait_for_vm_shutdown "${vm_name}"
+    virsh start "${vm_name}"
+    vm_ip=""
+    wait_for_ssh "${vm_name}"
+
+    ssh "cat > /etc/pacman.d/mirrorlist" <<< "Server = ${MIRROR}/\$repo/os/\$arch"
+    ssh "cat > /etc/systemd/network/20-wired.network" <<< $'[Match]\nName=eth0\n[Network]\nDHCP=yes'
+    ssh pacman -Sy --noconfirm --needed archlinux-keyring
+    ssh pacman -Syu --noconfirm git git-lfs gitlab-runner
+    ssh "sed -E 's/^#(IgnorePkg *=)/\1 linux/' -i /etc/pacman.conf"
+
+    virsh shutdown "${vm_name}"
+    wait_for_vm_shutdown "${vm_name}"
+    virsh domrename "${vm_name}" "${vm_name%%_tmp}"
+    # TODO: delete machine-id
+    # 192.168.122.170
+    trap - EXIT
+}
+
+# https://docs.gitlab.com/runner/executors/custom.html#prepare
+prepare() {
+    vm_template="$(virsh list --state-shutoff --name | grep "^libvirt_executor_vm_template_[0-9]*$" | sort -r | head -n 1)"
+    if [[ -z "${vm_template}" ]]; then
+        echo "Error no VM template found"
+        exit 1
+    fi
+    vm_name="foo"
+    if ! virt-clone -o "${vm_template}" -n "${vm_name}" --auto-clone --reflink; then
+        virt-clone -o "${vm_template}" -n "${vm_name}" --auto-clone
+    fi
+    virsh start "${vm_name}"
+    wait_for_ssh "${vm_name}"
+}
+
+# https://docs.gitlab.com/runner/executors/custom.html#run
+run() {
+    vm_name="foo"
+    wait_for_ssh "${vm_name}"
+    ssh bash < "${1}" || exit "${BUILD_FAILURE_EXIT_CODE:-1}"
+}
+
+# https://docs.gitlab.com/runner/executors/custom.html#cleanup
+cleanup() {
+    vm_name="foo"
+    virsh destroy "${vm_name}" || true
+    virsh undefine "${vm_name}" --remove-all-storage
+}
+
+case "${1:-}" in
+    create-vm-template)
+        create_vm_template
+        ;;
+    prepare)
+        prepare
+        ;;
+    run)
+        run "${2}"
+        ;;
+    cleanup)
+        cleanup
+        ;;
+    *)
+        echo "Error invalid command: ${1:-}"
+        exit 1;
+esac