Split build-inside-vm.sh per-image logic into separate files

.gitlab-ci.yml

@@ -12,14 +12,16 @@ shellcheck:
   before_script:
     - pacman -Syu --needed --noconfirm shellcheck
   script:
-    - shellcheck *.sh
+    - shopt -s globstar
+    - shellcheck **/*.sh
 
 shfmt:
   stage: lint
   before_script:
     - pacman -Syu --needed --noconfirm shfmt
   script:
-    - shfmt -i 2 -ci -d *.sh
+    - shopt -s globstar
+    - shfmt -i 2 -ci -d **/*.sh
 
 .build:
   stage: build

build-host.sh

@@ -118,7 +118,7 @@ function main() {
   expect "# "
   send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n"
   expect "# "
-  send "cp -a /mnt/arch-boxes/{box.ovf,build-inside-vm.sh} .\n"
+  send "cp -a /mnt/arch-boxes/{box.ovf,build-inside-vm.sh,images} .\n"
   expect "# "
   send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n"
   expect "# "

build-inside-vm.sh

@@ -4,6 +4,7 @@
 # nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion."
 # errexit: "Exit immediately if [...] command exits with a non-zero status."
 set -o nounset -o errexit
+shopt -s extglob
 readonly DISK_SIZE="20G"
 readonly IMAGE="image.img"
 # shellcheck disable=SC2016
@@ -77,86 +78,6 @@ EOF
   cp mirrorlist "${MOUNT}/etc/pacman.d/"
 }
 
-# Misc "tweaks" done after bootstrapping
-function postinstall() {
-  # Remove machine-id see:
-  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/25
-  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/117
-  rm "${MOUNT}/etc/machine-id"
-
-  arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
-  chattr +C "${MOUNT}/swap"
-  chmod 0700 "${MOUNT}/swap"
-  fallocate -l 512M "${MOUNT}/swap/swapfile"
-  mkswap "${MOUNT}/swap/swapfile"
-  echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
-
-  echo "COMPRESSION=\"zstd\"" >>"${MOUNT}/etc/mkinitcpio.conf"
-  arch-chroot "${MOUNT}" /usr/bin/mkinitcpio -p linux
-
-  sed -i -e 's/^#\(en_US.UTF-8\)/\1/' "${MOUNT}/etc/locale.gen"
-  arch-chroot "${MOUNT}" /usr/bin/locale-gen
-  arch-chroot "${MOUNT}" /usr/bin/systemd-firstboot --locale=en_US.UTF-8 --timezone=UTC --hostname=archlinux --keymap=us
-  ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT}/etc/resolv.conf"
-
-  # Setup pacman-init.service for clean pacman keyring initialization
-  cat <<EOF >"${MOUNT}/etc/systemd/system/pacman-init.service"
-[Unit]
-Description=Initializes Pacman keyring
-Wants=haveged.service
-After=haveged.service
-Before=sshd.service cloud-final.service
-ConditionFirstBoot=yes
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/bin/pacman-key --init
-ExecStart=/usr/bin/pacman-key --populate archlinux
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-  # Add service for running reflector on first boot
-  cat <<EOF >"${MOUNT}/etc/systemd/system/reflector-init.service"
-[Unit]
-Description=Initializes mirrors for the VM
-After=network-online.target
-Wants=network-online.target
-Before=sshd.service cloud-final.service
-ConditionFirstBoot=yes
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=reflector --latest 20 --protocol https --sort rate --save /etc/pacman.d/mirrorlist
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-  # enabling important services
-  arch-chroot "${MOUNT}" /bin/bash -e <<EOF
-. /etc/profile
-systemctl enable sshd
-systemctl enable haveged
-systemctl enable systemd-networkd
-systemctl enable systemd-resolved
-systemctl enable systemd-timesyncd
-systemctl enable pacman-init.service
-systemctl enable reflector-init.service
-EOF
-
-  # GRUB
-  arch-chroot "${MOUNT}" /usr/bin/grub-install --target=i386-pc "${LOOPDEV}"
-  sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' "${MOUNT}/etc/default/grub"
-  # setup unpredictable kernel names
-  sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' "${MOUNT}/etc/default/grub"
-  sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' "${MOUNT}/etc/default/grub"
-  arch-chroot "${MOUNT}" /usr/bin/grub-mkconfig -o /boot/grub/grub.cfg
-}
-
 # Cleanup the image and trim it
 function image_cleanup() {
   # Remove pacman key ring for re-initialization
@@ -217,6 +138,12 @@ function create_image() {
   local tmp_image
   tmp_image="$(basename "$(mktemp -u)")"
   copy_and_mount_image "${tmp_image}"
+  if [ 0 -lt "${#PACKAGES[@]}" ]; then
+    arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm "${PACKAGES[@]}"
+  fi
+  if [ 0 -lt "${#SERVICES[@]}" ]; then
+    arch-chroot "${MOUNT}" /usr/bin/systemctl enable "${SERVICES[@]}"
+  fi
   "${2}"
   image_cleanup
   unmount_image
@@ -224,112 +151,6 @@ function create_image() {
   mv_to_output "${1}"
 }
 
-function cloud_image() {
-  # The growpart module[1] requires the growpart program, provided by the
-  # cloud-guest-utils package
-  # [1] https://cloudinit.readthedocs.io/en/latest/topics/modules.html#growpart
-  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm cloud-init cloud-guest-utils
-  arch-chroot "${MOUNT}" /usr/bin/systemctl enable cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service
-}
-
-function cloud_image_post() {
-  qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
-  rm "${1}"
-}
-
-function vagrant_common() {
-  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm netctl polkit
-
-  NEWUSER="vagrant"
-  # setting the user credentials
-  arch-chroot "${MOUNT}" /usr/bin/useradd -m -U "${NEWUSER}"
-  echo -e "${NEWUSER}\n${NEWUSER}" | arch-chroot "${MOUNT}" /usr/bin/passwd "${NEWUSER}"
-
-  # setting sudo for the user
-  cat <<EOF >"${MOUNT}/etc/sudoers.d/${NEWUSER}"
-Defaults:${NEWUSER} !requiretty
-${NEWUSER} ALL=(ALL) NOPASSWD: ALL
-EOF
-  chmod 440 "${MOUNT}/etc/sudoers.d/${NEWUSER}"
-
-  # setup network
-  cat <<EOF >"${MOUNT}/etc/systemd/network/eth0.network"
-[Match]
-Name=eth0
-
-[Network]
-DHCP=ipv4
-EOF
-
-  # install vagrant ssh key
-  arch-chroot "${MOUNT}" /bin/bash -e <<EOF
-install --directory --owner=vagrant --group=vagrant --mode=0700 /home/vagrant/.ssh
-curl --output /home/vagrant/.ssh/authorized_keys --location https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub
-chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
-chmod 0600 /home/vagrant/.ssh/authorized_keys
-EOF
-
-  # setting automatic authentication for any action requiring admin rights via Polkit
-  cat <<EOF >"${MOUNT}/etc/polkit-1/rules.d/49-nopasswd_global.rules"
-polkit.addRule(function(action, subject) {
-    if (subject.isInGroup("vagrant")) {
-        return polkit.Result.YES;
-    }
-});
-EOF
-}
-
-function vagrant_qemu_post() {
-  # Create vagrant box
-  cat <<EOF >Vagrantfile
-Vagrant.configure("2") do |config|
-  config.vm.provider :libvirt do |libvirt|
-    libvirt.driver = "kvm"
-  end
-end
-EOF
-  local virtual_size
-  virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
-  echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
-  qemu-img convert -f raw -O qcow2 "${1}" box.img
-  rm "${1}"
-
-  tar -czf "${2}" Vagrantfile metadata.json box.img
-  rm Vagrantfile metadata.json box.img
-}
-
-function vagrant_virtualbox() {
-  vagrant_common
-  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm virtualbox-guest-utils-nox
-  arch-chroot "${MOUNT}" /usr/bin/systemctl enable vboxservice
-}
-
-function vagrant_virtualbox_post() {
-  # Create vagrant box
-  # VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
-  local mac_address
-  mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
-  cat <<EOF >Vagrantfile
-Vagrant.configure("2") do |config|
-  config.vm.base_mac = "${mac_address}"
-end
-EOF
-  echo '{"provider":"virtualbox"}' >metadata.json
-  qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
-  rm "${1}"
-
-  cp "${ORIG_PWD}/box.ovf" .
-  sed -e "s/MACHINE_UUID/$(uuidgen)/" \
-    -e "s/DISK_UUID/$(uuidgen)/" \
-    -e "s/DISK_CAPACITY/$(qemu-img info --output=json "packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
-    -e "s/UNIX/$(date +%s)/" \
-    -e "s/MAC_ADDRESS/${mac_address}/" \
-    -i box.ovf
-
-  tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
-  rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
-}
-
 # ${1} - Optional build version. If not set, will generate a default based on date.
 function main() {
   if [ "$(id -u)" -ne 0 ]; then
@@ -340,7 +161,9 @@ function main() {
 
   setup_disk
   bootstrap
-  postinstall
+  # shellcheck source=images/base.sh
+  . "${ORIG_PWD}/images/base.sh"
+  pre
   unmount_image
 
   local build_version
@@ -351,8 +174,13 @@ function main() {
   else
     build_version="${1}"
   fi
-  create_image "Arch-Linux-x86_64-cloudimg-${build_version}.qcow2" cloud_image cloud_image_post
-  create_image "Arch-Linux-x86_64-libvirt-${build_version}.box" vagrant_common vagrant_qemu_post
-  create_image "Arch-Linux-x86_64-virtualbox-${build_version}.box" vagrant_virtualbox vagrant_virtualbox_post
+
+  # shellcheck source=images/common.sh
+  . "${ORIG_PWD}/images/common.sh"
+  for image in "${ORIG_PWD}/images/"!(base|common).sh; do
+    # shellcheck source=/dev/null
+    . "${image}"
+    create_image "${IMAGE_NAME}" pre post
+  done
 }
 main "$@"

images/base.sh

+#!/bin/bash
+
+# Misc "tweaks" done after bootstrapping
+function pre() {
+  # Remove machine-id see:
+  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/25
+  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/117
+  rm "${MOUNT}/etc/machine-id"
+
+  arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
+  chattr +C "${MOUNT}/swap"
+  chmod 0700 "${MOUNT}/swap"
+  fallocate -l 512M "${MOUNT}/swap/swapfile"
+  mkswap "${MOUNT}/swap/swapfile"
+  echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
+
+  echo "COMPRESSION=\"zstd\"" >>"${MOUNT}/etc/mkinitcpio.conf"
+  arch-chroot "${MOUNT}" /usr/bin/mkinitcpio -p linux
+
+  sed -i -e 's/^#\(en_US.UTF-8\)/\1/' "${MOUNT}/etc/locale.gen"
+  arch-chroot "${MOUNT}" /usr/bin/locale-gen
+  arch-chroot "${MOUNT}" /usr/bin/systemd-firstboot --locale=en_US.UTF-8 --timezone=UTC --hostname=archlinux --keymap=us
+  ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT}/etc/resolv.conf"
+
+  # Setup pacman-init.service for clean pacman keyring initialization
+  cat <<EOF >"${MOUNT}/etc/systemd/system/pacman-init.service"
+[Unit]
+Description=Initializes Pacman keyring
+Wants=haveged.service
+After=haveged.service
+Before=sshd.service cloud-final.service
+ConditionFirstBoot=yes
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/bin/pacman-key --init
+ExecStart=/usr/bin/pacman-key --populate archlinux
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+  # Add service for running reflector on first boot
+  cat <<EOF >"${MOUNT}/etc/systemd/system/reflector-init.service"
+[Unit]
+Description=Initializes mirrors for the VM
+After=network-online.target
+Wants=network-online.target
+Before=sshd.service cloud-final.service
+ConditionFirstBoot=yes
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=reflector --latest 20 --protocol https --sort rate --save /etc/pacman.d/mirrorlist
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+  # enabling important services
+  arch-chroot "${MOUNT}" /bin/bash -e <<EOF
+. /etc/profile
+systemctl enable sshd
+systemctl enable haveged
+systemctl enable systemd-networkd
+systemctl enable systemd-resolved
+systemctl enable systemd-timesyncd
+systemctl enable pacman-init.service
+systemctl enable reflector-init.service
+EOF
+
+  # GRUB
+  arch-chroot "${MOUNT}" /usr/bin/grub-install --target=i386-pc "${LOOPDEV}"
+  sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' "${MOUNT}/etc/default/grub"
+  # setup unpredictable kernel names
+  sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' "${MOUNT}/etc/default/grub"
+  sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' "${MOUNT}/etc/default/grub"
+  arch-chroot "${MOUNT}" /usr/bin/grub-mkconfig -o /boot/grub/grub.cfg
+}

images/cloud-image.sh

+#!/bin/bash
+# shellcheck disable=SC2034,SC2154
+IMAGE_NAME="Arch-Linux-x86_64-cloudimg-${build_version}.qcow2"
+# The growpart module[1] requires the growpart program, provided by the
+# cloud-guest-utils package
+# [1] https://cloudinit.readthedocs.io/en/latest/topics/modules.html#growpart
+PACKAGES=(cloud-init cloud-guest-utils)
+SERVICES=(cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service)
+
+function pre() {
+  true
+}
+
+function post() {
+  qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
+  rm "${1}"
+}

images/common.sh

+#!/bin/bash
+
+function vagrant_common() {
+  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm netctl polkit
+
+  local NEWUSER="vagrant"
+  # setting the user credentials
+  arch-chroot "${MOUNT}" /usr/bin/useradd -m -U "${NEWUSER}"
+  echo -e "${NEWUSER}\n${NEWUSER}" | arch-chroot "${MOUNT}" /usr/bin/passwd "${NEWUSER}"
+
+  # setting sudo for the user
+  cat <<EOF >"${MOUNT}/etc/sudoers.d/${NEWUSER}"
+Defaults:${NEWUSER} !requiretty
+${NEWUSER} ALL=(ALL) NOPASSWD: ALL
+EOF
+  chmod 440 "${MOUNT}/etc/sudoers.d/${NEWUSER}"
+
+  # setup network
+  cat <<EOF >"${MOUNT}/etc/systemd/network/eth0.network"
+[Match]
+Name=eth0
+
+[Network]
+DHCP=ipv4
+EOF
+
+  # install vagrant ssh key
+  arch-chroot "${MOUNT}" /bin/bash -e <<EOF
+install --directory --owner=vagrant --group=vagrant --mode=0700 /home/vagrant/.ssh
+curl --output /home/vagrant/.ssh/authorized_keys --location https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub
+chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
+chmod 0600 /home/vagrant/.ssh/authorized_keys
+EOF
+
+  # setting automatic authentication for any action requiring admin rights via Polkit
+  cat <<EOF >"${MOUNT}/etc/polkit-1/rules.d/49-nopasswd_global.rules"
+polkit.addRule(function(action, subject) {
+    if (subject.isInGroup("vagrant")) {
+        return polkit.Result.YES;
+    }
+});
+EOF
+}

images/vagrant-libvirt.sh

+#!/bin/bash
+# shellcheck disable=SC2034,SC2154
+IMAGE_NAME="Arch-Linux-x86_64-libvirt-${build_version}.box"
+PACKAGES=()
+SERVICES=()
+
+function pre() {
+  vagrant_common
+}
+
+function post() {
+  # Create vagrant box
+  cat <<EOF >Vagrantfile
+Vagrant.configure("2") do |config|
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.driver = "kvm"
+  end
+end
+EOF
+  local virtual_size
+  virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
+  echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
+  qemu-img convert -f raw -O qcow2 "${1}" box.img
+  rm "${1}"
+
+  tar -czf "${2}" Vagrantfile metadata.json box.img
+  rm Vagrantfile metadata.json box.img
+}

images/vagrant-virtualbox.sh

+#!/bin/bash
+# shellcheck disable=SC2034,SC2154
+IMAGE_NAME="Arch-Linux-x86_64-virtualbox-${build_version}.box"
+PACKAGES=(virtualbox-guest-utils-nox)
+SERVICES=(vboxservice)
+
+function pre() {
+  vagrant_common
+}
+
+function post() {
+  # Create vagrant box
+  # VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
+  local mac_address
+  mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
+  cat <<EOF >Vagrantfile
+Vagrant.configure("2") do |config|
+  config.vm.base_mac = "${mac_address}"
+end
+EOF
+  echo '{"provider":"virtualbox"}' >metadata.json
+  qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
+  rm "${1}"
+
+  cp "${ORIG_PWD}/box.ovf" .
+  sed -e "s/MACHINE_UUID/$(uuidgen)/" \
+    -e "s/DISK_UUID/$(uuidgen)/" \
+    -e "s/DISK_CAPACITY/$(qemu-img info --output=json "packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
+    -e "s/UNIX/$(date +%s)/" \
+    -e "s/MAC_ADDRESS/${mac_address}/" \
+    -i box.ovf
+
+  tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
+  rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
+}