Commit 26cd9cc5 authored by Kristian Klausen's avatar Kristian Klausen 🎉
Browse files

Replace packer with two custom shell scripts

Using actual VMs to build VMs is slow and error-prone (you need to use
VNC to see what is going on, and booting takes over +110 seconds as we
wait to be sure Arch Linux is ready).

build.sh can build all three images in ~135 seconds (assuming all the
packages is cached), we still need to use a VM for the actually building
in GitLab CI (as that is the only safe way it can be done at the
moment), which is a bit slower (~22 min vs ~13 min (Packer)), but that
isn't really a big issue.

In the future we can hopefully switch to Kate Containers[1] and reduce
the build time significantly.

[1] infrastructure#108
parent 97129b35
...@@ -5,3 +5,5 @@ output-* ...@@ -5,3 +5,5 @@ output-*
.vscode .vscode
*.SHA256 *.SHA256
*.qcow2 *.qcow2
/tmp/
/output/
default: default:
image: "archlinux:latest" image: "archlinux:latest"
variables:
PACKER_LOG: 1
stages: stages:
- lint - lint
- build - build
...@@ -23,55 +20,20 @@ shfmt: ...@@ -23,55 +20,20 @@ shfmt:
script: script:
- find . -iname "*.sh" -exec shfmt -i 2 -ci -d {} + - find . -iname "*.sh" -exec shfmt -i 2 -ci -d {} +
validate-packer:
stage: lint
before_script:
- pacman -Syu --needed --noconfirm packer
script:
- packer validate config.json
# Note: We explicitly need the `ipv6` tag here because otherwise we'd get random # Note: We explicitly need the `ipv6` tag here because otherwise we'd get random
# gpg/pacman-key issues. # gpg/pacman-key issues.
build:cloud-qemu: build:
stage: build
tags:
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
script:
- packer build -only=cloud -except=sign config.json
artifacts:
name: "archlinux_x86_64_qcow2"
paths:
- "Arch-Linux-x86_64-cloudimg-*.*"
expire_in: 2d
build:vagrant-virtualbox:
stage: build
tags:
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
script:
- packer build -only=virtualbox config.json
artifacts:
name: "archlinux_x86_64_virtualbox"
paths:
- "*.box"
expire_in: 2d
build:vagrant-qemu:
stage: build stage: build
tags: tags:
- ipv6 - ipv6
before_script: before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless - pacman -Syu --needed --noconfirm qemu-headless libisoburn
script: script:
- packer build -only=libvirt config.json - ./build-in-qemu.sh
artifacts: artifacts:
name: "archlinux_x86_64_libvirt" name: "output"
paths: paths:
- "*.box" - "output/*"
expire_in: 2d expire_in: 2d
publish: publish:
...@@ -85,10 +47,10 @@ publish: ...@@ -85,10 +47,10 @@ publish:
- vagrant cloud auth login --token $VAGRANT_API_TOKEN - vagrant cloud auth login --token $VAGRANT_API_TOKEN
- vagrant cloud auth login --check - vagrant cloud auth login --check
- vagrant cloud box show archlinux/archlinux - vagrant cloud box show archlinux/archlinux
- LIBVIRT_RELEASE=`ls Arch-Linux-x86_64-libvirt-*.box | awk -F "." '{print $1}' | awk -F "-" '{print $5"."$6"."$7}'` - LIBVIRT_RELEASE=`ls output/Arch-Linux-x86_64-libvirt-*.box | awk -F "." '{print $1}' | awk -F "-" '{print $5"."$6"."$7}'`
- VIRTUALBOX_RELEASE=`ls Arch-Linux-x86_64-virtualbox-*.box | awk -F "." '{print $1}' | awk -F "-" '{print $5"."$6"."$7}'` - VIRTUALBOX_RELEASE=`ls output/Arch-Linux-x86_64-virtualbox-*.box | awk -F "." '{print $1}' | awk -F "-" '{print $5"."$6"."$7}'`
- vagrant cloud publish archlinux/archlinux $LIBVIRT_RELEASE libvirt Arch-Linux-x86_64-libvirt-*.box --release -f - vagrant cloud publish archlinux/archlinux $LIBVIRT_RELEASE libvirt output/Arch-Linux-x86_64-libvirt-*.box --release -f
- vagrant cloud publish archlinux/archlinux $VIRTUALBOX_RELEASE virtualbox Arch-Linux-x86_64-virtualbox-*.box --release -f - vagrant cloud publish archlinux/archlinux $VIRTUALBOX_RELEASE virtualbox output/Arch-Linux-x86_64-virtualbox-*.box --release -f
only: only:
variables: variables:
- $SCHEDULED_PUBLISH == "TRUE" - $SCHEDULED_PUBLISH == "TRUE"
......
#!/bin/bash
set -o nounset -o errexit
MIRROR="https://mirror.pkgbuild.com"
ORIG_PWD="${PWD}"
OUTPUT="${PWD}/output"
mkdir -p "tmp" "${OUTPUT}"
TMPDIR="$(mktemp --directory --tmpdir="${PWD}/tmp")"
cd "${TMPDIR}"
function cleanup() {
rm -rf "${TMPDIR}"
jobs -p | xargs --no-run-if-empty kill
}
trap cleanup EXIT
function prepare_boot() {
if LOCAL_ISO="$(ls "${ORIG_PWD}/"archlinux-*-x86_64.iso 2>/dev/null)"; then
echo "Using local iso: ${LOCAL_ISO}"
ISO="${LOCAL_ISO}"
fi
if [ -z "${LOCAL_ISO}" ]; then
LATEST_ISO="$(curl -fs "${MIRROR}/iso/latest/" | grep -Eo 'archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64.iso' | head -n 1)"
if [ -z "${LATEST_ISO}" ]; then
echo "Error: Couldn't find latest iso'"
exit 1
fi
curl -fO "${MIRROR}/iso/latest/${LATEST_ISO}"
ISO="${PWD}/${LATEST_ISO}"
fi
xorriso -osirrox on -indev "${ISO}" -extract arch/boot/x86_64 .
ISO_VOLUME_ID="$(xorriso -indev "${ISO}" |& awk -F : '$1 ~ "Volume id" {print $2}' | tr -d "' ")"
}
function start_qemu() {
mkfifo guest.out guest.in
# We could use a sparse file but we want to fail early
fallocate -l 4G scratch-disk.img
{ qemu-system-x86_64 \
-machine accel=kvm:tcg \
-m 768 \
-net nic \
-net user \
-kernel vmlinuz-linux \
-initrd archiso.img \
-append "archisobasedir=arch archisolabel=${ISO_VOLUME_ID} ip=dhcp net.ifnames=0 console=ttyS0" \
-drive file=scratch-disk.img,format=raw,if=virtio \
-drive file="${ISO}",format=raw,if=virtio,media=cdrom,read-only \
-virtfs "local,path=${ORIG_PWD},mount_tag=host,security_model=none" \
-monitor none \
-serial pipe:guest \
-nographic || kill "${$}"; } &
# Send guest.out to fd1 (stdout) and fd10
exec 3>&1 10< <(tee /dev/fd/3 <guest.out)
}
function expect() {
length="${#1}"
i=0
while IFS= read -r -u 10 -n 1 c; do
if [ "${1:${i}:1}" = "${c}" ]; then
i="$((i + 1))"
if [ "${length}" -eq "${i}" ]; then
break
fi
else
i=0
fi
done
}
function send() {
echo -en "${1}" >guest.in
}
prepare_boot
start_qemu
expect "archiso login:"
send "root\n"
expect "# "
send "bash\n"
expect "# "
send "trap \"shutdown now\" ERR\n"
expect "# "
send "mkdir /mnt/arch-boxes && mount -t 9p -o trans=virtio host /mnt/arch-boxes -oversion=9p2000.L\n"
expect "# "
send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n"
expect "# "
send "cp -a /mnt/arch-boxes/{box.ovf,build.sh,http} .\n"
expect "# "
send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n"
expect "# "
# Wait for pacman-init
send "until systemctl is-active pacman-init; do sleep 1; done\n"
expect "# "
send "pacman -Sy --noconfirm qemu-headless jq\n"
expect "# "
send "bash -x ./build.sh\n"
expect "# "
send "cp -r --preserve=mode,timestamps output /mnt/arch-boxes/tmp/$(basename "${TMPDIR}")/\n"
expect "# "
mv output/* "${OUTPUT}/"
send "shutdown now\n"
wait
#!/bin/bash
set -o nounset -o errexit
DISK_SIZE="2G"
IMAGE="image.img"
# shellcheck disable=SC2016
MIRROR='https://mirror.pkgbuild.com/$repo/os/$arch'
if [ "$(id -u)" -ne 0 ]; then
echo "root is required"
exit 1
fi
ORIG_PWD="${PWD}"
OUTPUT="${PWD}/output"
mkdir -p "tmp" "${OUTPUT}"
if [ -n "${SUDO_UID:-}" ]; then
chown "${SUDO_UID}:${SUDO_GID}" "tmp" "${OUTPUT}"
fi
TMPDIR="$(mktemp --directory --tmpdir="${PWD}/tmp")"
cd "${TMPDIR}"
MOUNT="${PWD}/mount"
mkdir "${MOUNT}"
function cleanup() {
set +o errexit
if [ -n "${LOOPDEV:-}" ]; then
losetup -d "${LOOPDEV}"
fi
if [ -n "${MOUNT:-}" ] && mountpoint -q "${MOUNT}"; then
# We do not want risking deleting ex: the package cache
umount --recursive "${MOUNT}" || exit 1
fi
if [ -n "${TMPDIR:-}" ]; then
rm -rf "${TMPDIR}"
fi
}
trap cleanup EXIT
function setup_disk() {
truncate -s "${DISK_SIZE}" "${IMAGE}"
sgdisk --clear \
--new 1::+1M --typecode=1:ef02 \
--new 2::-0 --typecode=2:8300 \
"${IMAGE}"
LOOPDEV=$(losetup --find --partscan --show "${IMAGE}")
mkfs.btrfs "${LOOPDEV}p2"
mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}"
}
function bootstrap() {
cat <<EOF >pacman.conf
[options]
Architecture = auto
[core]
Include = mirrorlist
[extra]
Include = mirrorlist
[community]
Include = mirrorlist
EOF
echo "Server = ${MIRROR}" >mirrorlist
# We use the hosts package cache
pacstrap -c -C pacman.conf -M "${MOUNT}" base linux grub openssh sudo polkit haveged netctl python btrfs-progs reflector
cp mirrorlist "${MOUNT}/etc/pacman.d/"
}
function postinstall() {
echo "archlinux" >"${MOUNT}/etc/hostname"
echo "KEYMAP=us" >"${MOUNT}/etc/vconsole.conf"
ln -sf /var/run/systemd/resolve/resolv.conf "${MOUNT}/etc/resolv.conf"
}
function image_cleanup() {
# Remove machine-id: see https://github.com/archlinux/arch-boxes/issues/25
rm "${MOUNT}/etc/machine-id"
# Remove pacman key ring for re-initialization
rm -rf "${MOUNT}/etc/pacman.d/gnupg/"
sync -f "${MOUNT}/etc/os-release"
fstrim --verbose "${MOUNT}"
}
function mount_image() {
LOOPDEV=$(losetup --find --partscan --show "${1:-${IMAGE}}")
mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}"
# Setup bind mount to package cache
mount --bind "/var/cache/pacman/pkg" "${MOUNT}/var/cache/pacman/pkg"
}
function unmount_image() {
umount --recursive "${MOUNT}"
losetup -d "${LOOPDEV}"
LOOPDEV=""
}
function copy_and_mount_image() {
cp -a "${IMAGE}" "${1}"
mount_image "${1}"
}
function mv_to_output() {
sha256sum "${1}" >"${1}.SHA256"
if [ -n "${SUDO_UID:-}" ]; then
chown "${SUDO_UID}:${SUDO_GID}" "${1}"{,.SHA256}
fi
mv "${1}"{,.SHA256} "${OUTPUT}/"
}
# ${1} - new image file
# ${2} - final file
# ${3} - pre
# ${4} - post
function create_image() {
copy_and_mount_image "${1}"
"${3}"
image_cleanup
unmount_image
"${4}" "${1}" "${2}"
mv_to_output "${2}"
}
function cloud_image() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{cloud,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm linux-headers qemu-guest-agent cloud-init
arch-chroot "${MOUNT}" /usr/bin/systemctl enable cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service
}
function cloud_image_post() {
qemu-img convert -f raw -O qcow2 "${1}" "${2}"
rm "${1}"
}
function create_box() {
TYPE="${1}"
IMAGE_FILE="${2}"
OUTPUT_FILE="${3}"
mkdir box
case "${TYPE}" in
qemu)
cat <<EOF >box/Vagrantfile
Vagrant.configure("2") do |config|
config.vm.provider :libvirt do |libvirt|
libvirt.driver = "kvm"
end
end
EOF
VIRTUAL_SIZE="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${VIRTUAL_SIZE}"'}' >box/metadata.json
qemu-img convert -f raw -O qcow2 "${IMAGE_FILE}" "box/box.img"
;;
virtualbox)
# VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
MAC_ADDRESS="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
cat <<EOF >box/Vagrantfile
Vagrant.configure("2") do |config|
config.vm.base_mac = "${MAC_ADDRESS}"
end
EOF
echo '{"provider":"virtualbox"}' >box/metadata.json
cp "${ORIG_PWD}/box.ovf" box/
qemu-img convert -f raw -O vmdk "${IMAGE_FILE}" "box/packer-virtualbox.vmdk"
sed -e "s/MACHINE_UUID/$(uuidgen)/" \
-e "s/DISK_UUID/$(uuidgen)/" \
-e "s/DISK_CAPACITY/$(qemu-img info --output=json "box/packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
-e "s/UNIX/$(date +%s)/" \
-e "s/MAC_ADDRESS/${MAC_ADDRESS}/" \
-i box/box.ovf
;;
*)
echo "Unknown box type: ${TYPE}"
exit 1
;;
esac
rm "${IMAGE_FILE}"
tar --xform 's:^box/::' -czf "${OUTPUT_FILE}" box/*
rm -r box
}
function vagrant_qemu() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{chroot,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm linux-headers qemu-guest-agent
}
function vagrant_qemu_post() {
create_box "qemu" "${1}" "${2}"
}
function vagrant_virtualbox() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{chroot,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm virtualbox-guest-utils-nox
arch-chroot "${MOUNT}" /usr/bin/systemctl enable vboxservice
}
function vagrant_virtualbox_post() {
create_box "virtualbox" "${1}" "${2}"
}
setup_disk
bootstrap
postinstall
# We run it here as it is the easiest solution and we do not want anything to go wrong!
arch-chroot "${MOUNT}" grub-install --target=i386-pc "${LOOPDEV}"
unmount_image
DATE="$(date -I)"
create_image "cloud-img.img" "Arch-Linux-x86_64-cloudimg-${DATE}.qcow2" cloud_image cloud_image_post
create_image "vagrant-qemu.img" "Arch-Linux-x86_64-libvirt-${DATE}.box" vagrant_qemu vagrant_qemu_post
create_image "vagrant-virtualbox.img" "Arch-Linux-x86_64-virtualbox-${DATE}.box" vagrant_qemu vagrant_virtualbox_post
{
"variables": {
"iso_url": "https://mirror.pkgbuild.com/iso/latest/archlinux-{{isotime \"2006.01\"}}.01-x86_64.iso",
"iso_checksum_url": "https://mirror.pkgbuild.com/iso/latest/sha1sums.txt",
"disk_size": "20480",
"headless": "true",
"boot_wait": "60s",
"accelerator": "",
"mirror": "https://mirror.pkgbuild.com/$repo/os/$arch"
},
"builders": [
{
"type": "qemu",
"name": "virtualbox",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_discard": "unmap",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "qemu",
"name": "libvirt",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_discard": "unmap",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "qemu",
"name": "cloud",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"disk_discard": "unmap",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "arch",
"ssh_password": "arch",
"ssh_port": 22,
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"vm_name": "Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2",
"headless": "{{user `headless`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-cloud}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{cloud,common}.sh) && systemctl reboot<enter>"
]
}
],
"provisioners": [
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/virtualbox.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"virtualbox"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/qemu.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"libvirt"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/qemu.sh",
"provision/cloud-init.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'arch'|sudo -S sh '{{.Path}}'",
"only": [
"cloud"
]
}
],
"post-processors": [
[
{
"type": "shell-local",
"only": [
"virtualbox"
],
"environment_vars": [
"OUTPUT=output-{{build_name}}",
"VM_NAME=packer-{{build_name}}"
],
"script": "post-processor/virtualbox.sh"
},
{
"type": "artifice",
"only": [
"virtualbox"
],
"files": [
"output-{{build_name}}/packer-{{build_name}}.vmdk",
"output-{{build_name}}/box.ovf"
]
},
{
"type": "vagrant",
"except": [
"cloud"
],
"keep_input_artifact": false,
"output": "Arch-Linux-x86_64-{{ .Provider }}-{{isotime \"2006-01-02\"}}.box",
"provider_override": "{{build_name}}"
},
{
"name": "cleanup",
"type": "shell-local",
"only": [
"virtualbox"
],
"inline": [
"rm -r output-virtualbox"
]
},
{
"type": "checksum",
"only": [
"cloud"
],
"checksum_types": [
"sha256"
],
"output": "Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.SHA256"
},
{
"name": "rename",
"type": "shell-local",
"only": [
"cloud"
],
"inline": [
"mv output-cloud/Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2 Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2 && rm -r output-cloud"
]
},
{