Commit a8e162bd authored by Sven-Hendrik Haase's avatar Sven-Hendrik Haase
Browse files

Merge branch 'losetup' into 'master'

Replace packer with two custom shell scripts

See merge request archlinux/arch-boxes!118
parents 97129b35 a8af1b0b
Pipeline #1823 canceled with stages
in 29 seconds
......@@ -5,3 +5,5 @@ output-*
.vscode
*.SHA256
*.qcow2
/tmp/
/output/
default:
image: "archlinux:latest"
variables:
PACKER_LOG: 1
stages:
- lint
- build
......@@ -23,55 +20,23 @@ shfmt:
script:
- find . -iname "*.sh" -exec shfmt -i 2 -ci -d {} +
validate-packer:
stage: lint
before_script:
- pacman -Syu --needed --noconfirm packer
script:
- packer validate config.json
# Note: We explicitly need the `ipv6` tag here because otherwise we'd get random
# gpg/pacman-key issues.
build:cloud-qemu:
stage: build
tags:
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
script:
- packer build -only=cloud -except=sign config.json
artifacts:
name: "archlinux_x86_64_qcow2"
paths:
- "Arch-Linux-x86_64-cloudimg-*.*"
expire_in: 2d
build:vagrant-virtualbox:
stage: build
tags:
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
script:
- packer build -only=virtualbox config.json
artifacts:
name: "archlinux_x86_64_virtualbox"
paths:
- "*.box"
expire_in: 2d
build:vagrant-qemu:
build:
stage: build
tags:
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
- pacman -Syu --needed --noconfirm qemu-headless libisoburn
script:
- packer build -only=libvirt config.json
- echo "BUILD_DATE=$(date -I)" > build.env
- . build.env
- ./build-in-qemu.sh
- mv build.env output/
artifacts:
name: "archlinux_x86_64_libvirt"
name: "output"
paths:
- "*.box"
- "output/*"
expire_in: 2d
publish:
......@@ -82,13 +47,12 @@ publish:
before_script:
- pacman -Syu --needed --noconfirm vagrant
script:
- vagrant cloud auth login --token $VAGRANT_API_TOKEN
- . output/build.env
- vagrant cloud auth login --token "${VAGRANT_API_TOKEN}"
- vagrant cloud auth login --check
- vagrant cloud box show archlinux/archlinux
- LIBVIRT_RELEASE=`ls Arch-Linux-x86_64-libvirt-*.box | awk -F "." '{print $1}' | awk -F "-" '{print $5"."$6"."$7}'`
- VIRTUALBOX_RELEASE=`ls Arch-Linux-x86_64-virtualbox-*.box | awk -F "." '{print $1}' | awk -F "-" '{print $5"."$6"."$7}'`
- vagrant cloud publish archlinux/archlinux $LIBVIRT_RELEASE libvirt Arch-Linux-x86_64-libvirt-*.box --release -f
- vagrant cloud publish archlinux/archlinux $VIRTUALBOX_RELEASE virtualbox Arch-Linux-x86_64-virtualbox-*.box --release -f
- vagrant cloud publish archlinux/archlinux "v${BUILD_DATE}" libvirt output/Arch-Linux-x86_64-libvirt-*.box --release -f
- vagrant cloud publish archlinux/archlinux "v${BUILD_DATE}" virtualbox output/Arch-Linux-x86_64-virtualbox-*.box --release -f
only:
variables:
- $SCHEDULED_PUBLISH == "TRUE"
......
#!/bin/bash
# build-in qemu.sh runs build.sh in a qemu VM running the latest Arch installer iso
#
# nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion."
# errexit: "Exit immediately if [...] command exits with a non-zero status."
set -o nounset -o errexit
readonly MIRROR="https://mirror.pkgbuild.com"
function init() {
readonly ORIG_PWD="${PWD}"
readonly OUTPUT="${PWD}/output"
readonly TMPDIR="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")"
mkdir -p "${OUTPUT}" "${TMPDIR}"
cd "${TMPDIR}"
}
# Do some cleanup when the script exits
function cleanup() {
rm -rf "${TMPDIR}"
jobs -p | xargs --no-run-if-empty kill
}
trap cleanup EXIT
# Use local Arch iso or download the latest iso and extract the relevant files
function prepare_boot() {
if LOCAL_ISO="$(ls "${ORIG_PWD}/"archlinux-*-x86_64.iso 2>/dev/null)"; then
echo "Using local iso: ${LOCAL_ISO}"
ISO="${LOCAL_ISO}"
fi
if [ -z "${LOCAL_ISO}" ]; then
LATEST_ISO="$(curl -fs "${MIRROR}/iso/latest/" | grep -Eo 'archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64.iso' | head -n 1)"
if [ -z "${LATEST_ISO}" ]; then
echo "Error: Couldn't find latest iso'"
exit 1
fi
curl -fO "${MIRROR}/iso/latest/${LATEST_ISO}"
ISO="${PWD}/${LATEST_ISO}"
fi
# We need to extract the kernel and initrd so we can set a custom cmdline:
# console=ttyS0, so the kernel and systemd sends output to the serial.
xorriso -osirrox on -indev "${ISO}" -extract arch/boot/x86_64 .
ISO_VOLUME_ID="$(xorriso -indev "${ISO}" |& awk -F : '$1 ~ "Volume id" {print $2}' | tr -d "' ")"
}
function start_qemu() {
# Used to communicate with qemu
mkfifo guest.out guest.in
# We could use a sparse file but we want to fail early
fallocate -l 4G scratch-disk.img
{ qemu-system-x86_64 \
-machine accel=kvm:tcg \
-m 768 \
-net nic \
-net user \
-kernel vmlinuz-linux \
-initrd archiso.img \
-append "archisobasedir=arch archisolabel=${ISO_VOLUME_ID} ip=dhcp net.ifnames=0 console=ttyS0 mirror=${MIRROR}" \
-drive file=scratch-disk.img,format=raw,if=virtio \
-drive file="${ISO}",format=raw,if=virtio,media=cdrom,read-only \
-virtfs "local,path=${ORIG_PWD},mount_tag=host,security_model=none" \
-monitor none \
-serial pipe:guest \
-nographic || kill "${$}"; } &
# We want to send the output to both stdout (fd1) and fd10 (used by the expect function)
exec 3>&1 10< <(tee /dev/fd/3 <guest.out)
}
# Wait for a specific string from qemu
function expect() {
local length="${#1}"
local i=0
local timeout="${2:-30}"
# We can't use ex: grep as we could end blocking forever, if the string isn't followed by a newline
while true; do
# read should never exit with a non-zero exit code,
# but it can happen if the fd is EOF or it times out
IFS= read -r -u 10 -n 1 -t "${timeout}" c
if [ "${1:${i}:1}" = "${c}" ]; then
i="$((i + 1))"
if [ "${length}" -eq "${i}" ]; then
break
fi
else
i=0
fi
done
}
# Send string to qemu
function send() {
echo -en "${1}" >guest.in
}
function main() {
init
prepare_boot
start_qemu
# Login
expect "archiso login:"
send "root\n"
expect "# "
# Switch to bash and shutdown on error
send "bash\n"
expect "# "
send "trap \"shutdown now\" ERR\n"
expect "# "
# Prepare environment
send "mkdir /mnt/arch-boxes && mount -t 9p -o trans=virtio host /mnt/arch-boxes -oversion=9p2000.L\n"
expect "# "
send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n"
expect "# "
send "cp -a /mnt/arch-boxes/{box.ovf,build.sh,http} .\n"
expect "# "
send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n"
expect "# "
# Wait for pacman-init
send "until systemctl is-active pacman-init; do sleep 1; done\n"
expect "# "
# Install required packages
send "pacman -Sy --noconfirm qemu-headless jq\n"
expect "# "
## Start build and copy output to local disk
send "bash -x ./build.sh\n"
expect "# " 240 # qemu-img convert can take a long time
send "cp -r --preserve=mode,timestamps output /mnt/arch-boxes/tmp/$(basename "${TMPDIR}")/\n"
expect "# " 60
mv output/* "${OUTPUT}/"
# Shutdown the VM
send "shutdown now\n"
wait
}
main
#!/bin/bash
# build.sh builds the images (cloud image, vagrant boxes)
# nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion."
# errexit: "Exit immediately if [...] command exits with a non-zero status."
set -o nounset -o errexit
readonly DISK_SIZE="2G"
readonly IMAGE="image.img"
# shellcheck disable=SC2016
readonly MIRROR='https://mirror.pkgbuild.com/$repo/os/$arch'
function init() {
readonly ORIG_PWD="${PWD}"
readonly OUTPUT="${PWD}/output"
readonly TMPDIR="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")"
mkdir -p "${OUTPUT}" "${TMPDIR}"
if [ -n "${SUDO_UID:-}" ]; then
chown "${SUDO_UID}:${SUDO_GID}" "${OUTPUT}" "${TMPDIR}"
fi
cd "${TMPDIR}"
readonly MOUNT="${PWD}/mount"
mkdir "${MOUNT}"
}
# Do some cleanup when the script exits
function cleanup() {
# We want all the commands to run, even if one of them fails.
set +o errexit
if [ -n "${LOOPDEV:-}" ]; then
losetup -d "${LOOPDEV}"
fi
if [ -n "${MOUNT:-}" ] && mountpoint -q "${MOUNT}"; then
# We do not want risking deleting ex: the package cache
umount --recursive "${MOUNT}" || exit 1
fi
if [ -n "${TMPDIR:-}" ]; then
rm -rf "${TMPDIR}"
fi
}
trap cleanup EXIT
# Create the disk, partitions it, format the partition and mount the filesystem
function setup_disk() {
truncate -s "${DISK_SIZE}" "${IMAGE}"
sgdisk --clear \
--new 1::+1M --typecode=1:ef02 \
--new 2::-0 --typecode=2:8300 \
"${IMAGE}"
LOOPDEV=$(losetup --find --partscan --show "${IMAGE}")
mkfs.btrfs "${LOOPDEV}p2"
mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}"
}
# Install Arch Linux to the filesystem (bootstrap)
function bootstrap() {
cat <<EOF >pacman.conf
[options]
Architecture = auto
[core]
Include = mirrorlist
[extra]
Include = mirrorlist
[community]
Include = mirrorlist
EOF
echo "Server = ${MIRROR}" >mirrorlist
# We use the hosts package cache
pacstrap -c -C pacman.conf -M "${MOUNT}" base linux grub openssh sudo polkit haveged netctl python btrfs-progs reflector
cp mirrorlist "${MOUNT}/etc/pacman.d/"
}
# Misc "tweaks" done after bootstrapping
function postinstall() {
arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
chattr +C "${MOUNT}/swap"
chmod 0700 "${MOUNT}/swap"
fallocate -l 512M "${MOUNT}/swap/swapfile"
mkswap "${MOUNT}/swap/swapfile"
echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
echo "archlinux" >"${MOUNT}/etc/hostname"
echo "KEYMAP=us" >"${MOUNT}/etc/vconsole.conf"
ln -sf /var/run/systemd/resolve/resolv.conf "${MOUNT}/etc/resolv.conf"
}
# Cleanup the image and trim it
function image_cleanup() {
# Remove pacman key ring for re-initialization
rm -rf "${MOUNT}/etc/pacman.d/gnupg/"
sync -f "${MOUNT}/etc/os-release"
fstrim --verbose "${MOUNT}"
}
# Mount image helper (loop device + mount)
function mount_image() {
LOOPDEV=$(losetup --find --partscan --show "${1:-${IMAGE}}")
mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}"
# Setup bind mount to package cache
mount --bind "/var/cache/pacman/pkg" "${MOUNT}/var/cache/pacman/pkg"
}
# Unmount image helper (umount + detach loop device)
function unmount_image() {
umount --recursive "${MOUNT}"
losetup -d "${LOOPDEV}"
LOOPDEV=""
}
# Copy image and mount the copied image
function copy_and_mount_image() {
cp -a "${IMAGE}" "${1}"
mount_image "${1}"
}
# Compute SHA256, adjust owner to $SUDO_UID:$SUDO_UID and move to output/
function mv_to_output() {
sha256sum "${1}" >"${1}.SHA256"
if [ -n "${SUDO_UID:-}" ]; then
chown "${SUDO_UID}:${SUDO_GID}" "${1}"{,.SHA256}
fi
mv "${1}"{,.SHA256} "${OUTPUT}/"
}
# Helper function: create a new image from the "base" image
# ${1} - new image file
# ${2} - final file
# ${3} - pre
# ${4} - post
function create_image() {
copy_and_mount_image "${1}"
"${3}"
image_cleanup
unmount_image
"${4}" "${1}" "${2}"
mv_to_output "${2}"
}
function cloud_image() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{cloud,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm linux-headers qemu-guest-agent cloud-init
arch-chroot "${MOUNT}" /usr/bin/systemctl enable cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service
}
function cloud_image_post() {
qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
rm "${1}"
}
function vagrant_qemu() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{chroot,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm linux-headers qemu-guest-agent
}
function vagrant_qemu_post() {
# Create vagrant box
cat <<EOF >Vagrantfile
Vagrant.configure("2") do |config|
config.vm.provider :libvirt do |libvirt|
libvirt.driver = "kvm"
end
end
EOF
local virtual_size
virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
qemu-img convert -f raw -O qcow2 "${1}" box.img
rm "${1}"
tar -czf "${2}" Vagrantfile metadata.json box.img
rm Vagrantfile metadata.json box.img
}
function vagrant_virtualbox() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{chroot,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm virtualbox-guest-utils-nox
arch-chroot "${MOUNT}" /usr/bin/systemctl enable vboxservice
}
function vagrant_virtualbox_post() {
# Create vagrant box
# VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
local mac_address
mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
cat <<EOF >Vagrantfile
Vagrant.configure("2") do |config|
config.vm.base_mac = "${mac_address}"
end
EOF
echo '{"provider":"virtualbox"}' >metadata.json
qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
rm "${1}"
cp "${ORIG_PWD}/box.ovf" .
sed -e "s/MACHINE_UUID/$(uuidgen)/" \
-e "s/DISK_UUID/$(uuidgen)/" \
-e "s/DISK_CAPACITY/$(qemu-img info --output=json "box/packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
-e "s/UNIX/$(date +%s)/" \
-e "s/MAC_ADDRESS/${mac_address}/" \
-i box.ovf
tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
}
function main() {
if [ "$(id -u)" -ne 0 ]; then
echo "root is required"
exit 1
fi
init
setup_disk
bootstrap
postinstall
# We run it here as it is the easiest solution and we do not want anything to go wrong!
arch-chroot "${MOUNT}" grub-install --target=i386-pc "${LOOPDEV}"
unmount_image
if [ -z "${BUILD_DATE:-}" ]; then
BUILD_DATE="$(date -I)"
fi
create_image "cloud-img.img" "Arch-Linux-x86_64-cloudimg-${BUILD_DATE}.qcow2" cloud_image cloud_image_post
create_image "vagrant-qemu.img" "Arch-Linux-x86_64-libvirt-${BUILD_DATE}.box" vagrant_qemu vagrant_qemu_post
create_image "vagrant-virtualbox.img" "Arch-Linux-x86_64-virtualbox-${BUILD_DATE}.box" vagrant_qemu vagrant_virtualbox_post
}
main
{
"variables": {
"iso_url": "https://mirror.pkgbuild.com/iso/latest/archlinux-{{isotime \"2006.01\"}}.01-x86_64.iso",
"iso_checksum_url": "https://mirror.pkgbuild.com/iso/latest/sha1sums.txt",
"disk_size": "20480",
"headless": "true",
"boot_wait": "60s",
"accelerator": "",
"mirror": "https://mirror.pkgbuild.com/$repo/os/$arch"
},
"builders": [
{
"type": "qemu",
"name": "virtualbox",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_discard": "unmap",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "qemu",
"name": "libvirt",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_discard": "unmap",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "qemu",
"name": "cloud",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"disk_discard": "unmap",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "arch",
"ssh_password": "arch",
"ssh_port": 22,
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"vm_name": "Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2",
"headless": "{{user `headless`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-cloud}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{cloud,common}.sh) && systemctl reboot<enter>"
]
}
],
"provisioners": [
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/virtualbox.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"virtualbox"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/qemu.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"libvirt"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/qemu.sh",
"provision/cloud-init.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'arch'|sudo -S sh '{{.Path}}'",
"only": [
"cloud"
]
}
],
"post-processors": [
[
{
"type": "shell-local",
"only": [
"virtualbox"
],
"environment_vars": [
"OUTPUT=output-{{build_name}}",
"VM_NAME=packer-{{build_name}}"
],
"script": "post-processor/virtualbox.sh"
},
{
"type": "artifice",
"only": [
"virtualbox"
],
"files": [
"output-{{build_name}}/packer-{{build_name}}.vmdk",
"output-{{build_name}}/box.ovf"
]
},
{
"type": "vagrant",
"except": [
"cloud"