Merge branch 'http-nuke' into 'master'

Merge http/* into build-inside-vm.sh

See merge request archlinux/arch-boxes!146

.gitlab-ci.yml

@@ -12,6 +12,7 @@ shellcheck:
   before_script:
     - pacman -Syu --needed --noconfirm shellcheck
   script:
+    - shopt -s globstar
     - shellcheck **/*.sh
 
 shfmt:
@@ -19,6 +20,7 @@ shfmt:
   before_script:
     - pacman -Syu --needed --noconfirm shfmt
   script:
+    - shopt -s globstar
     - shfmt -i 2 -ci -d **/*.sh
 
 .build:

build-host.sh

@@ -118,7 +118,7 @@ function main() {
   expect "# "
   send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n"
   expect "# "
-  send "cp -a /mnt/arch-boxes/{box.ovf,build-inside-vm.sh,http} .\n"
+  send "cp -a /mnt/arch-boxes/{box.ovf,build-inside-vm.sh,images} .\n"
   expect "# "
   send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n"
   expect "# "

build-inside-vm.sh

@@ -4,6 +4,7 @@
 # nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion."
 # errexit: "Exit immediately if [...] command exits with a non-zero status."
 set -o nounset -o errexit
+shopt -s extglob
 readonly DISK_SIZE="20G"
 readonly IMAGE="image.img"
 # shellcheck disable=SC2016
@@ -77,29 +78,6 @@ EOF
   cp mirrorlist "${MOUNT}/etc/pacman.d/"
 }
 
-# Misc "tweaks" done after bootstrapping
-function postinstall() {
-  # Remove machine-id see:
-  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/25
-  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/117
-  rm "${MOUNT}/etc/machine-id"
-
-  arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
-  chattr +C "${MOUNT}/swap"
-  chmod 0700 "${MOUNT}/swap"
-  fallocate -l 512M "${MOUNT}/swap/swapfile"
-  mkswap "${MOUNT}/swap/swapfile"
-  echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
-
-  echo "COMPRESSION=\"zstd\"" >>"${MOUNT}/etc/mkinitcpio.conf"
-  arch-chroot "${MOUNT}" /usr/bin/mkinitcpio -p linux
-
-  sed -i -e 's/^#\(en_US.UTF-8\)/\1/' "${MOUNT}/etc/locale.gen"
-  arch-chroot "${MOUNT}" /usr/bin/locale-gen
-  arch-chroot "${MOUNT}" /usr/bin/systemd-firstboot --locale=en_US.UTF-8 --timezone=UTC --hostname=archlinux --keymap=us
-  ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT}/etc/resolv.conf"
-}
-
 # Cleanup the image and trim it
 function image_cleanup() {
   # Remove pacman key ring for re-initialization
@@ -113,10 +91,10 @@ function image_cleanup() {
 # ${1} - loop device
 function wait_until_settled() {
   udevadm settle
-  blockdev --flushbufs --rereadpt ${1}
+  blockdev --flushbufs --rereadpt "${1}"
   until test -e "${1}p2"; do
-      echo "${1}p2 doesn't exist yet..."
-      sleep 1
+    echo "${1}p2 doesn't exist yet..."
+    sleep 1
   done
 }
 
@@ -124,7 +102,7 @@ function wait_until_settled() {
 function mount_image() {
   LOOPDEV=$(losetup --find --partscan --show "${1:-${IMAGE}}")
   # Partscan is racy
-  wait_until_settled ${LOOPDEV}
+  wait_until_settled "${LOOPDEV}"
   mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}"
   # Setup bind mount to package cache
   mount --bind "/var/cache/pacman/pkg" "${MOUNT}/var/cache/pacman/pkg"
@@ -157,8 +135,15 @@ function mv_to_output() {
 # ${2} - pre
 # ${3} - post
 function create_image() {
-  local tmp_image="$(basename "$(mktemp -u)")"
+  local tmp_image
+  tmp_image="$(basename "$(mktemp -u)")"
   copy_and_mount_image "${tmp_image}"
+  if [ 0 -lt "${#PACKAGES[@]}" ]; then
+    arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm "${PACKAGES[@]}"
+  fi
+  if [ 0 -lt "${#SERVICES[@]}" ]; then
+    arch-chroot "${MOUNT}" /usr/bin/systemctl enable "${SERVICES[@]}"
+  fi
   "${2}"
   image_cleanup
   unmount_image
@@ -166,84 +151,6 @@ function create_image() {
   mv_to_output "${1}"
 }
 
-function cloud_image() {
-  arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{cloud,common}.sh)
-  # The growpart module[1] requires the growpart program, provided by the
-  # cloud-guest-utils package
-  # [1] https://cloudinit.readthedocs.io/en/latest/topics/modules.html#growpart
-  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm cloud-init cloud-guest-utils
-  arch-chroot "${MOUNT}" /usr/bin/systemctl enable cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service
-}
-
-function cloud_image_post() {
-  qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
-  rm "${1}"
-}
-
-function vagrant_common() {
-  arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{vagrant,common}.sh)
-  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm netctl polkit
-  # setting automatic authentication for any action requiring admin rights via Polkit
-  cat <<EOF >"${MOUNT}/etc/polkit-1/rules.d/49-nopasswd_global.rules"
-polkit.addRule(function(action, subject) {
-    if (subject.isInGroup("vagrant")) {
-        return polkit.Result.YES;
-    }
-});
-EOF
-}
-
-function vagrant_qemu_post() {
-  # Create vagrant box
-  cat <<EOF >Vagrantfile
-Vagrant.configure("2") do |config|
-  config.vm.provider :libvirt do |libvirt|
-    libvirt.driver = "kvm"
-  end
-end
-EOF
-  local virtual_size
-  virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
-  echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
-  qemu-img convert -f raw -O qcow2 "${1}" box.img
-  rm "${1}"
-
-  tar -czf "${2}" Vagrantfile metadata.json box.img
-  rm Vagrantfile metadata.json box.img
-}
-
-function vagrant_virtualbox() {
-  vagrant_common
-  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm virtualbox-guest-utils-nox
-  arch-chroot "${MOUNT}" /usr/bin/systemctl enable vboxservice
-}
-
-function vagrant_virtualbox_post() {
-  # Create vagrant box
-  # VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
-  local mac_address
-  mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
-  cat <<EOF >Vagrantfile
-Vagrant.configure("2") do |config|
-  config.vm.base_mac = "${mac_address}"
-end
-EOF
-  echo '{"provider":"virtualbox"}' >metadata.json
-  qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
-  rm "${1}"
-
-  cp "${ORIG_PWD}/box.ovf" .
-  sed -e "s/MACHINE_UUID/$(uuidgen)/" \
-    -e "s/DISK_UUID/$(uuidgen)/" \
-    -e "s/DISK_CAPACITY/$(qemu-img info --output=json "packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
-    -e "s/UNIX/$(date +%s)/" \
-    -e "s/MAC_ADDRESS/${mac_address}/" \
-    -i box.ovf
-
-  tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
-  rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
-}
-
 # ${1} - Optional build version. If not set, will generate a default based on date.
 function main() {
   if [ "$(id -u)" -ne 0 ]; then
@@ -254,9 +161,9 @@ function main() {
 
   setup_disk
   bootstrap
-  postinstall
-  # We run it here as it is the easiest solution and we do not want anything to go wrong!
-  arch-chroot "${MOUNT}" grub-install --target=i386-pc "${LOOPDEV}"
+  # shellcheck source=images/base.sh
+  . "${ORIG_PWD}/images/base.sh"
+  pre
   unmount_image
 
   local build_version
@@ -267,8 +174,13 @@ function main() {
   else
     build_version="${1}"
   fi
-  create_image "Arch-Linux-x86_64-cloudimg-${build_version}.qcow2" cloud_image cloud_image_post
-  create_image "Arch-Linux-x86_64-libvirt-${build_version}.box" vagrant_common vagrant_qemu_post
-  create_image "Arch-Linux-x86_64-virtualbox-${build_version}.box" vagrant_virtualbox vagrant_virtualbox_post
+
+  # shellcheck source=images/common.sh
+  . "${ORIG_PWD}/images/common.sh"
+  for image in "${ORIG_PWD}/images/"!(base|common).sh; do
+    # shellcheck source=/dev/null
+    . "${image}"
+    create_image "${IMAGE_NAME}" pre post
+  done
 }
 main "$@"

http/install-cloud.sh

-#!/bin/bash
-# shellcheck disable=SC2034
-NEWUSER="arch"

http/install-vagrant.sh

-#!/bin/bash
-# shellcheck disable=SC2034
-NEWUSER="vagrant"
-
-post() {
-  # setting the user credentials
-  useradd -m -U "${NEWUSER}"
-  echo -e "${NEWUSER}\n${NEWUSER}" | passwd "${NEWUSER}"
-
-  # setting sudo for the user
-  cat <<EOF >"/etc/sudoers.d/${NEWUSER}"
-Defaults:${NEWUSER} !requiretty
-${NEWUSER} ALL=(ALL) NOPASSWD: ALL
-EOF
-  chmod 440 "/etc/sudoers.d/${NEWUSER}"
-
-  # setup network
-  cat <<EOF >/etc/systemd/network/eth0.network
-[Match]
-Name=eth0
-
-[Network]
-DHCP=ipv4
-EOF
-
-  # install vagrant ssh key
-  install --directory --owner=vagrant --group=vagrant --mode=0700 /home/vagrant/.ssh
-  curl --output /home/vagrant/.ssh/authorized_keys --location https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub
-  chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
-  chmod 0600 /home/vagrant/.ssh/authorized_keys
-}

http/install-common.sh → images/base.sh

 #!/bin/bash
 
-set -e
-set -x
+# Misc "tweaks" done after bootstrapping
+function pre() {
+  # Remove machine-id see:
+  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/25
+  # https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/117
+  rm "${MOUNT}/etc/machine-id"
 
-# Setup pacman-init.service for clean pacman keyring initialization
-cat <<EOF >/etc/systemd/system/pacman-init.service
+  arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
+  chattr +C "${MOUNT}/swap"
+  chmod 0700 "${MOUNT}/swap"
+  fallocate -l 512M "${MOUNT}/swap/swapfile"
+  mkswap "${MOUNT}/swap/swapfile"
+  echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
+
+  echo "COMPRESSION=\"zstd\"" >>"${MOUNT}/etc/mkinitcpio.conf"
+  arch-chroot "${MOUNT}" /usr/bin/mkinitcpio -p linux
+
+  sed -i -e 's/^#\(en_US.UTF-8\)/\1/' "${MOUNT}/etc/locale.gen"
+  arch-chroot "${MOUNT}" /usr/bin/locale-gen
+  arch-chroot "${MOUNT}" /usr/bin/systemd-firstboot --locale=en_US.UTF-8 --timezone=UTC --hostname=archlinux --keymap=us
+  ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT}/etc/resolv.conf"
+
+  # Setup pacman-init.service for clean pacman keyring initialization
+  cat <<EOF >"${MOUNT}/etc/systemd/system/pacman-init.service"
 [Unit]
 Description=Initializes Pacman keyring
 Wants=haveged.service
@@ -22,8 +41,8 @@ ExecStart=/usr/bin/pacman-key --populate archlinux
 WantedBy=multi-user.target
 EOF
 
-# Add service for running reflector on first boot
-cat <<EOF >/etc/systemd/system/reflector-init.service
+  # Add service for running reflector on first boot
+  cat <<EOF >"${MOUNT}/etc/systemd/system/reflector-init.service"
 [Unit]
 Description=Initializes mirrors for the VM
 After=network-online.target
@@ -40,7 +59,9 @@ ExecStart=reflector --latest 20 --protocol https --sort rate --save /etc/pacman.
 WantedBy=multi-user.target
 EOF
 
-# enabling important services
+  # enabling important services
+  arch-chroot "${MOUNT}" /bin/bash -e <<EOF
+. /etc/profile
 systemctl enable sshd
 systemctl enable haveged
 systemctl enable systemd-networkd
@@ -48,13 +69,13 @@ systemctl enable systemd-resolved
 systemctl enable systemd-timesyncd
 systemctl enable pacman-init.service
 systemctl enable reflector-init.service
+EOF
 
-sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' /etc/default/grub
-# setup unpredictable kernel names
-sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' /etc/default/grub
-sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' /etc/default/grub
-grub-mkconfig -o /boot/grub/grub.cfg
-
-if declare -f post >/dev/null; then
-  post
-fi
+  # GRUB
+  arch-chroot "${MOUNT}" /usr/bin/grub-install --target=i386-pc "${LOOPDEV}"
+  sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' "${MOUNT}/etc/default/grub"
+  # setup unpredictable kernel names
+  sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' "${MOUNT}/etc/default/grub"
+  sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' "${MOUNT}/etc/default/grub"
+  arch-chroot "${MOUNT}" /usr/bin/grub-mkconfig -o /boot/grub/grub.cfg
+}

images/cloud-image.sh

+#!/bin/bash
+# shellcheck disable=SC2034,SC2154
+IMAGE_NAME="Arch-Linux-x86_64-cloudimg-${build_version}.qcow2"
+# The growpart module[1] requires the growpart program, provided by the
+# cloud-guest-utils package
+# [1] https://cloudinit.readthedocs.io/en/latest/topics/modules.html#growpart
+PACKAGES=(cloud-init cloud-guest-utils)
+SERVICES=(cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service)
+
+function pre() {
+  true
+}
+
+function post() {
+  qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
+  rm "${1}"
+}

images/common.sh

+#!/bin/bash
+
+function vagrant_common() {
+  arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm netctl polkit
+
+  local NEWUSER="vagrant"
+  # setting the user credentials
+  arch-chroot "${MOUNT}" /usr/bin/useradd -m -U "${NEWUSER}"
+  echo -e "${NEWUSER}\n${NEWUSER}" | arch-chroot "${MOUNT}" /usr/bin/passwd "${NEWUSER}"
+
+  # setting sudo for the user
+  cat <<EOF >"${MOUNT}/etc/sudoers.d/${NEWUSER}"
+Defaults:${NEWUSER} !requiretty
+${NEWUSER} ALL=(ALL) NOPASSWD: ALL
+EOF
+  chmod 440 "${MOUNT}/etc/sudoers.d/${NEWUSER}"
+
+  # setup network
+  cat <<EOF >"${MOUNT}/etc/systemd/network/eth0.network"
+[Match]
+Name=eth0
+
+[Network]
+DHCP=ipv4
+EOF
+
+  # install vagrant ssh key
+  arch-chroot "${MOUNT}" /bin/bash -e <<EOF
+install --directory --owner=vagrant --group=vagrant --mode=0700 /home/vagrant/.ssh
+curl --output /home/vagrant/.ssh/authorized_keys --location https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub
+chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
+chmod 0600 /home/vagrant/.ssh/authorized_keys
+EOF
+
+  # setting automatic authentication for any action requiring admin rights via Polkit
+  cat <<EOF >"${MOUNT}/etc/polkit-1/rules.d/49-nopasswd_global.rules"
+polkit.addRule(function(action, subject) {
+    if (subject.isInGroup("vagrant")) {
+        return polkit.Result.YES;
+    }
+});
+EOF
+}

images/vagrant-libvirt.sh

+#!/bin/bash
+# shellcheck disable=SC2034,SC2154
+IMAGE_NAME="Arch-Linux-x86_64-libvirt-${build_version}.box"
+PACKAGES=()
+SERVICES=()
+
+function pre() {
+  vagrant_common
+}
+
+function post() {
+  # Create vagrant box
+  cat <<EOF >Vagrantfile
+Vagrant.configure("2") do |config|
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.driver = "kvm"
+  end
+end
+EOF
+  local virtual_size
+  virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
+  echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
+  qemu-img convert -f raw -O qcow2 "${1}" box.img
+  rm "${1}"
+
+  tar -czf "${2}" Vagrantfile metadata.json box.img
+  rm Vagrantfile metadata.json box.img
+}

images/vagrant-virtualbox.sh

+#!/bin/bash
+# shellcheck disable=SC2034,SC2154
+IMAGE_NAME="Arch-Linux-x86_64-virtualbox-${build_version}.box"
+PACKAGES=(virtualbox-guest-utils-nox)
+SERVICES=(vboxservice)
+
+function pre() {
+  vagrant_common
+}
+
+function post() {
+  # Create vagrant box
+  # VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
+  local mac_address
+  mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
+  cat <<EOF >Vagrantfile
+Vagrant.configure("2") do |config|
+  config.vm.base_mac = "${mac_address}"
+end
+EOF
+  echo '{"provider":"virtualbox"}' >metadata.json
+  qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
+  rm "${1}"
+
+  cp "${ORIG_PWD}/box.ovf" .
+  sed -e "s/MACHINE_UUID/$(uuidgen)/" \
+    -e "s/DISK_UUID/$(uuidgen)/" \
+    -e "s/DISK_CAPACITY/$(qemu-img info --output=json "packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
+    -e "s/UNIX/$(date +%s)/" \
+    -e "s/MAC_ADDRESS/${mac_address}/" \
+    -i box.ovf
+
+  tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
+  rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
+}