Commit 7c277938 authored by Sven-Hendrik Haase's avatar Sven-Hendrik Haase
Browse files

Merge branch 'http-nuke' into 'master'

Merge http/* into build-inside-vm.sh

See merge request !146
parents 1bfbf2d4 fdb4bd89
Pipeline #3538 passed with stages
in 36 minutes and 20 seconds
...@@ -12,6 +12,7 @@ shellcheck: ...@@ -12,6 +12,7 @@ shellcheck:
before_script: before_script:
- pacman -Syu --needed --noconfirm shellcheck - pacman -Syu --needed --noconfirm shellcheck
script: script:
- shopt -s globstar
- shellcheck **/*.sh - shellcheck **/*.sh
shfmt: shfmt:
...@@ -19,6 +20,7 @@ shfmt: ...@@ -19,6 +20,7 @@ shfmt:
before_script: before_script:
- pacman -Syu --needed --noconfirm shfmt - pacman -Syu --needed --noconfirm shfmt
script: script:
- shopt -s globstar
- shfmt -i 2 -ci -d **/*.sh - shfmt -i 2 -ci -d **/*.sh
.build: .build:
......
...@@ -118,7 +118,7 @@ function main() { ...@@ -118,7 +118,7 @@ function main() {
expect "# " expect "# "
send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n" send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n"
expect "# " expect "# "
send "cp -a /mnt/arch-boxes/{box.ovf,build-inside-vm.sh,http} .\n" send "cp -a /mnt/arch-boxes/{box.ovf,build-inside-vm.sh,images} .\n"
expect "# " expect "# "
send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n" send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n"
expect "# " expect "# "
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
# nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion." # nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion."
# errexit: "Exit immediately if [...] command exits with a non-zero status." # errexit: "Exit immediately if [...] command exits with a non-zero status."
set -o nounset -o errexit set -o nounset -o errexit
shopt -s extglob
readonly DISK_SIZE="20G" readonly DISK_SIZE="20G"
readonly IMAGE="image.img" readonly IMAGE="image.img"
# shellcheck disable=SC2016 # shellcheck disable=SC2016
...@@ -77,29 +78,6 @@ EOF ...@@ -77,29 +78,6 @@ EOF
cp mirrorlist "${MOUNT}/etc/pacman.d/" cp mirrorlist "${MOUNT}/etc/pacman.d/"
} }
# Misc "tweaks" done after bootstrapping
function postinstall() {
# Remove machine-id see:
# https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/25
# https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/117
rm "${MOUNT}/etc/machine-id"
arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
chattr +C "${MOUNT}/swap"
chmod 0700 "${MOUNT}/swap"
fallocate -l 512M "${MOUNT}/swap/swapfile"
mkswap "${MOUNT}/swap/swapfile"
echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
echo "COMPRESSION=\"zstd\"" >>"${MOUNT}/etc/mkinitcpio.conf"
arch-chroot "${MOUNT}" /usr/bin/mkinitcpio -p linux
sed -i -e 's/^#\(en_US.UTF-8\)/\1/' "${MOUNT}/etc/locale.gen"
arch-chroot "${MOUNT}" /usr/bin/locale-gen
arch-chroot "${MOUNT}" /usr/bin/systemd-firstboot --locale=en_US.UTF-8 --timezone=UTC --hostname=archlinux --keymap=us
ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT}/etc/resolv.conf"
}
# Cleanup the image and trim it # Cleanup the image and trim it
function image_cleanup() { function image_cleanup() {
# Remove pacman key ring for re-initialization # Remove pacman key ring for re-initialization
...@@ -113,10 +91,10 @@ function image_cleanup() { ...@@ -113,10 +91,10 @@ function image_cleanup() {
# ${1} - loop device # ${1} - loop device
function wait_until_settled() { function wait_until_settled() {
udevadm settle udevadm settle
blockdev --flushbufs --rereadpt ${1} blockdev --flushbufs --rereadpt "${1}"
until test -e "${1}p2"; do until test -e "${1}p2"; do
echo "${1}p2 doesn't exist yet..." echo "${1}p2 doesn't exist yet..."
sleep 1 sleep 1
done done
} }
...@@ -124,7 +102,7 @@ function wait_until_settled() { ...@@ -124,7 +102,7 @@ function wait_until_settled() {
function mount_image() { function mount_image() {
LOOPDEV=$(losetup --find --partscan --show "${1:-${IMAGE}}") LOOPDEV=$(losetup --find --partscan --show "${1:-${IMAGE}}")
# Partscan is racy # Partscan is racy
wait_until_settled ${LOOPDEV} wait_until_settled "${LOOPDEV}"
mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}" mount -o compress-force=zstd "${LOOPDEV}p2" "${MOUNT}"
# Setup bind mount to package cache # Setup bind mount to package cache
mount --bind "/var/cache/pacman/pkg" "${MOUNT}/var/cache/pacman/pkg" mount --bind "/var/cache/pacman/pkg" "${MOUNT}/var/cache/pacman/pkg"
...@@ -157,8 +135,15 @@ function mv_to_output() { ...@@ -157,8 +135,15 @@ function mv_to_output() {
# ${2} - pre # ${2} - pre
# ${3} - post # ${3} - post
function create_image() { function create_image() {
local tmp_image="$(basename "$(mktemp -u)")" local tmp_image
tmp_image="$(basename "$(mktemp -u)")"
copy_and_mount_image "${tmp_image}" copy_and_mount_image "${tmp_image}"
if [ 0 -lt "${#PACKAGES[@]}" ]; then
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm "${PACKAGES[@]}"
fi
if [ 0 -lt "${#SERVICES[@]}" ]; then
arch-chroot "${MOUNT}" /usr/bin/systemctl enable "${SERVICES[@]}"
fi
"${2}" "${2}"
image_cleanup image_cleanup
unmount_image unmount_image
...@@ -166,84 +151,6 @@ function create_image() { ...@@ -166,84 +151,6 @@ function create_image() {
mv_to_output "${1}" mv_to_output "${1}"
} }
function cloud_image() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{cloud,common}.sh)
# The growpart module[1] requires the growpart program, provided by the
# cloud-guest-utils package
# [1] https://cloudinit.readthedocs.io/en/latest/topics/modules.html#growpart
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm cloud-init cloud-guest-utils
arch-chroot "${MOUNT}" /usr/bin/systemctl enable cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service
}
function cloud_image_post() {
qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
rm "${1}"
}
function vagrant_common() {
arch-chroot "${MOUNT}" /bin/bash < <(cat "${ORIG_PWD}"/http/install-{vagrant,common}.sh)
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm netctl polkit
# setting automatic authentication for any action requiring admin rights via Polkit
cat <<EOF >"${MOUNT}/etc/polkit-1/rules.d/49-nopasswd_global.rules"
polkit.addRule(function(action, subject) {
if (subject.isInGroup("vagrant")) {
return polkit.Result.YES;
}
});
EOF
}
function vagrant_qemu_post() {
# Create vagrant box
cat <<EOF >Vagrantfile
Vagrant.configure("2") do |config|
config.vm.provider :libvirt do |libvirt|
libvirt.driver = "kvm"
end
end
EOF
local virtual_size
virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
qemu-img convert -f raw -O qcow2 "${1}" box.img
rm "${1}"
tar -czf "${2}" Vagrantfile metadata.json box.img
rm Vagrantfile metadata.json box.img
}
function vagrant_virtualbox() {
vagrant_common
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm virtualbox-guest-utils-nox
arch-chroot "${MOUNT}" /usr/bin/systemctl enable vboxservice
}
function vagrant_virtualbox_post() {
# Create vagrant box
# VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
local mac_address
mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
cat <<EOF >Vagrantfile
Vagrant.configure("2") do |config|
config.vm.base_mac = "${mac_address}"
end
EOF
echo '{"provider":"virtualbox"}' >metadata.json
qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
rm "${1}"
cp "${ORIG_PWD}/box.ovf" .
sed -e "s/MACHINE_UUID/$(uuidgen)/" \
-e "s/DISK_UUID/$(uuidgen)/" \
-e "s/DISK_CAPACITY/$(qemu-img info --output=json "packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
-e "s/UNIX/$(date +%s)/" \
-e "s/MAC_ADDRESS/${mac_address}/" \
-i box.ovf
tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
}
# ${1} - Optional build version. If not set, will generate a default based on date. # ${1} - Optional build version. If not set, will generate a default based on date.
function main() { function main() {
if [ "$(id -u)" -ne 0 ]; then if [ "$(id -u)" -ne 0 ]; then
...@@ -254,9 +161,9 @@ function main() { ...@@ -254,9 +161,9 @@ function main() {
setup_disk setup_disk
bootstrap bootstrap
postinstall # shellcheck source=images/base.sh
# We run it here as it is the easiest solution and we do not want anything to go wrong! . "${ORIG_PWD}/images/base.sh"
arch-chroot "${MOUNT}" grub-install --target=i386-pc "${LOOPDEV}" pre
unmount_image unmount_image
local build_version local build_version
...@@ -267,8 +174,13 @@ function main() { ...@@ -267,8 +174,13 @@ function main() {
else else
build_version="${1}" build_version="${1}"
fi fi
create_image "Arch-Linux-x86_64-cloudimg-${build_version}.qcow2" cloud_image cloud_image_post
create_image "Arch-Linux-x86_64-libvirt-${build_version}.box" vagrant_common vagrant_qemu_post # shellcheck source=images/common.sh
create_image "Arch-Linux-x86_64-virtualbox-${build_version}.box" vagrant_virtualbox vagrant_virtualbox_post . "${ORIG_PWD}/images/common.sh"
for image in "${ORIG_PWD}/images/"!(base|common).sh; do
# shellcheck source=/dev/null
. "${image}"
create_image "${IMAGE_NAME}" pre post
done
} }
main "$@" main "$@"
#!/bin/bash
# shellcheck disable=SC2034
NEWUSER="arch"
#!/bin/bash
# shellcheck disable=SC2034
NEWUSER="vagrant"
post() {
# setting the user credentials
useradd -m -U "${NEWUSER}"
echo -e "${NEWUSER}\n${NEWUSER}" | passwd "${NEWUSER}"
# setting sudo for the user
cat <<EOF >"/etc/sudoers.d/${NEWUSER}"
Defaults:${NEWUSER} !requiretty
${NEWUSER} ALL=(ALL) NOPASSWD: ALL
EOF
chmod 440 "/etc/sudoers.d/${NEWUSER}"
# setup network
cat <<EOF >/etc/systemd/network/eth0.network
[Match]
Name=eth0
[Network]
DHCP=ipv4
EOF
# install vagrant ssh key
install --directory --owner=vagrant --group=vagrant --mode=0700 /home/vagrant/.ssh
curl --output /home/vagrant/.ssh/authorized_keys --location https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub
chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
chmod 0600 /home/vagrant/.ssh/authorized_keys
}
#!/bin/bash #!/bin/bash
set -e # Misc "tweaks" done after bootstrapping
set -x function pre() {
# Remove machine-id see:
# https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/25
# https://gitlab.archlinux.org/archlinux/arch-boxes/-/issues/117
rm "${MOUNT}/etc/machine-id"
# Setup pacman-init.service for clean pacman keyring initialization arch-chroot "${MOUNT}" /usr/bin/btrfs subvolume create /swap
cat <<EOF >/etc/systemd/system/pacman-init.service chattr +C "${MOUNT}/swap"
chmod 0700 "${MOUNT}/swap"
fallocate -l 512M "${MOUNT}/swap/swapfile"
mkswap "${MOUNT}/swap/swapfile"
echo -e "/swap/swapfile none swap defaults 0 0" >>"${MOUNT}/etc/fstab"
echo "COMPRESSION=\"zstd\"" >>"${MOUNT}/etc/mkinitcpio.conf"
arch-chroot "${MOUNT}" /usr/bin/mkinitcpio -p linux
sed -i -e 's/^#\(en_US.UTF-8\)/\1/' "${MOUNT}/etc/locale.gen"
arch-chroot "${MOUNT}" /usr/bin/locale-gen
arch-chroot "${MOUNT}" /usr/bin/systemd-firstboot --locale=en_US.UTF-8 --timezone=UTC --hostname=archlinux --keymap=us
ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT}/etc/resolv.conf"
# Setup pacman-init.service for clean pacman keyring initialization
cat <<EOF >"${MOUNT}/etc/systemd/system/pacman-init.service"
[Unit] [Unit]
Description=Initializes Pacman keyring Description=Initializes Pacman keyring
Wants=haveged.service Wants=haveged.service
...@@ -22,8 +41,8 @@ ExecStart=/usr/bin/pacman-key --populate archlinux ...@@ -22,8 +41,8 @@ ExecStart=/usr/bin/pacman-key --populate archlinux
WantedBy=multi-user.target WantedBy=multi-user.target
EOF EOF
# Add service for running reflector on first boot # Add service for running reflector on first boot
cat <<EOF >/etc/systemd/system/reflector-init.service cat <<EOF >"${MOUNT}/etc/systemd/system/reflector-init.service"
[Unit] [Unit]
Description=Initializes mirrors for the VM Description=Initializes mirrors for the VM
After=network-online.target After=network-online.target
...@@ -40,7 +59,9 @@ ExecStart=reflector --latest 20 --protocol https --sort rate --save /etc/pacman. ...@@ -40,7 +59,9 @@ ExecStart=reflector --latest 20 --protocol https --sort rate --save /etc/pacman.
WantedBy=multi-user.target WantedBy=multi-user.target
EOF EOF
# enabling important services # enabling important services
arch-chroot "${MOUNT}" /bin/bash -e <<EOF
. /etc/profile
systemctl enable sshd systemctl enable sshd
systemctl enable haveged systemctl enable haveged
systemctl enable systemd-networkd systemctl enable systemd-networkd
...@@ -48,13 +69,13 @@ systemctl enable systemd-resolved ...@@ -48,13 +69,13 @@ systemctl enable systemd-resolved
systemctl enable systemd-timesyncd systemctl enable systemd-timesyncd
systemctl enable pacman-init.service systemctl enable pacman-init.service
systemctl enable reflector-init.service systemctl enable reflector-init.service
EOF
sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' /etc/default/grub # GRUB
# setup unpredictable kernel names arch-chroot "${MOUNT}" /usr/bin/grub-install --target=i386-pc "${LOOPDEV}"
sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' /etc/default/grub sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' "${MOUNT}/etc/default/grub"
sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' /etc/default/grub # setup unpredictable kernel names
grub-mkconfig -o /boot/grub/grub.cfg sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' "${MOUNT}/etc/default/grub"
sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' "${MOUNT}/etc/default/grub"
if declare -f post >/dev/null; then arch-chroot "${MOUNT}" /usr/bin/grub-mkconfig -o /boot/grub/grub.cfg
post }
fi
#!/bin/bash
# shellcheck disable=SC2034,SC2154
IMAGE_NAME="Arch-Linux-x86_64-cloudimg-${build_version}.qcow2"
# The growpart module[1] requires the growpart program, provided by the
# cloud-guest-utils package
# [1] https://cloudinit.readthedocs.io/en/latest/topics/modules.html#growpart
PACKAGES=(cloud-init cloud-guest-utils)
SERVICES=(cloud-init-local.service cloud-init.service cloud-config.service cloud-final.service)
function pre() {
true
}
function post() {
qemu-img convert -c -f raw -O qcow2 "${1}" "${2}"
rm "${1}"
}
#!/bin/bash
function vagrant_common() {
arch-chroot "${MOUNT}" /usr/bin/pacman -S --noconfirm netctl polkit
local NEWUSER="vagrant"
# setting the user credentials
arch-chroot "${MOUNT}" /usr/bin/useradd -m -U "${NEWUSER}"
echo -e "${NEWUSER}\n${NEWUSER}" | arch-chroot "${MOUNT}" /usr/bin/passwd "${NEWUSER}"
# setting sudo for the user
cat <<EOF >"${MOUNT}/etc/sudoers.d/${NEWUSER}"
Defaults:${NEWUSER} !requiretty
${NEWUSER} ALL=(ALL) NOPASSWD: ALL
EOF
chmod 440 "${MOUNT}/etc/sudoers.d/${NEWUSER}"
# setup network
cat <<EOF >"${MOUNT}/etc/systemd/network/eth0.network"
[Match]
Name=eth0
[Network]
DHCP=ipv4
EOF
# install vagrant ssh key
arch-chroot "${MOUNT}" /bin/bash -e <<EOF
install --directory --owner=vagrant --group=vagrant --mode=0700 /home/vagrant/.ssh
curl --output /home/vagrant/.ssh/authorized_keys --location https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub
chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
chmod 0600 /home/vagrant/.ssh/authorized_keys
EOF
# setting automatic authentication for any action requiring admin rights via Polkit
cat <<EOF >"${MOUNT}/etc/polkit-1/rules.d/49-nopasswd_global.rules"
polkit.addRule(function(action, subject) {
if (subject.isInGroup("vagrant")) {
return polkit.Result.YES;
}
});
EOF
}
#!/bin/bash
# shellcheck disable=SC2034,SC2154
IMAGE_NAME="Arch-Linux-x86_64-libvirt-${build_version}.box"
PACKAGES=()
SERVICES=()
function pre() {
vagrant_common
}
function post() {
# Create vagrant box
cat <<EOF >Vagrantfile
Vagrant.configure("2") do |config|
config.vm.provider :libvirt do |libvirt|
libvirt.driver = "kvm"
end
end
EOF
local virtual_size
virtual_size="$(grep -o "^[0-9]*" <<<"${DISK_SIZE}")"
echo '{"format":"qcow2","provider":"libvirt","virtual_size":'"${virtual_size}"'}' >metadata.json
qemu-img convert -f raw -O qcow2 "${1}" box.img
rm "${1}"
tar -czf "${2}" Vagrantfile metadata.json box.img
rm Vagrantfile metadata.json box.img
}
#!/bin/bash
# shellcheck disable=SC2034,SC2154
IMAGE_NAME="Arch-Linux-x86_64-virtualbox-${build_version}.box"
PACKAGES=(virtualbox-guest-utils-nox)
SERVICES=(vboxservice)
function pre() {
vagrant_common
}
function post() {
# Create vagrant box
# VirtualBox-6.1.12 src/VBox/NetworkServices/Dhcpd/Config.cpp line 276
local mac_address
mac_address="080027$(openssl rand -hex 3 | tr '[:lower:]' '[:upper:]')"
cat <<EOF >Vagrantfile
Vagrant.configure("2") do |config|
config.vm.base_mac = "${mac_address}"
end
EOF
echo '{"provider":"virtualbox"}' >metadata.json
qemu-img convert -f raw -O vmdk "${1}" "packer-virtualbox.vmdk"
rm "${1}"
cp "${ORIG_PWD}/box.ovf" .
sed -e "s/MACHINE_UUID/$(uuidgen)/" \
-e "s/DISK_UUID/$(uuidgen)/" \
-e "s/DISK_CAPACITY/$(qemu-img info --output=json "packer-virtualbox.vmdk" | jq '."virtual-size"')/" \
-e "s/UNIX/$(date +%s)/" \
-e "s/MAC_ADDRESS/${mac_address}/" \
-i box.ovf
tar -czf "${2}" Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
rm Vagrantfile metadata.json packer-virtualbox.vmdk box.ovf
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment