Commit 96248bd8 authored by Sven-Hendrik Haase's avatar Sven-Hendrik Haase
Browse files

Merge branch 'try-software-mode' into 'master'

Attempt to use tcg instead of kvm as qemu accelerator

Closes #106

See merge request !115
parents 97c2d3eb deb40297
Pipeline #1032 passed with stages
in 16 minutes and 37 seconds
......@@ -3,3 +3,5 @@ packer_cache/
*.swp
output-*
.vscode
*.SHA256
*.qcow2
default:
image: "archlinux:latest"
variables:
PACKER_LOG: 1
stages:
- lint
- build
......@@ -25,65 +28,44 @@ validate-packer:
before_script:
- pacman -Syu --needed --noconfirm packer
script:
- packer validate -var "iso_checksum_url=https://mirror.pkgbuild.com/iso/latest/sha1sums.txt" -except=vagrant-cloud vagrant.json
- packer validate local.json
- packer validate -var "iso_checksum_url=https://mirror.pkgbuild.com/iso/latest/sha1sums.txt" -except=publish vagrant.json
- packer validate cloud.json
# Note: We explicitly need the `ipv6` tag here because otherwise we'd get random
# gpg/pacman-key issues.
build:cloud-qemu:
stage: build
tags:
- secure-kvm
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
script:
- packer build -parallel-builds=1 -var 'headless=true' -var 'write_zeroes=yes' -except=sign cloud.json
- packer build -except=sign cloud.json
artifacts:
name: "cloud-qemu"
name: "qcow2"
paths:
- "release/Arch-Linux-x86_64-cloudimg-*.img"
- "Arch-Linux-x86_64-cloudimg-*.*"
expire_in: 2d
resource_group: vm-build
build:vagrant-qemu:
stage: build
tags:
- secure-kvm
- ipv6
before_script:
- pacman -Syu --needed --noconfirm packer qemu-headless
script:
- packer build -parallel-builds=1 -var 'headless=true' -var 'write_zeroes=yes' -only=qemu local.json
artifacts:
paths:
- "Arch-Linux-x86_64-libvirt-*.box"
expire_in: 3h
resource_group: vm-build
build:vagrant-virtualbox:
stage: build
tags:
- secure-virtualbox
before_script:
- pacman -Syu --needed --noconfirm packer virtualbox
script:
- packer build -parallel-builds=1 -var 'headless=true' -var 'write_zeroes=yes' -only=virtualbox-iso local.json
artifacts:
paths:
- "Arch-Linux-x86_64-virtualbox-*.box"
expire_in: 3h
resource_group: vm-build
- packer build -only=qemu -except publish vagrant.json
publish:
stage: publish
tags:
- secure-kvm
- secure-virtualbox
- ipv6
- secure
before_script:
- pacman -Syu --needed --noconfirm qemu-headless virtualbox packer
script:
- packer build -force -parallel-builds=1 -var "vagrant_cloud_token=$VAGRANT_API_TOKEN" -var 'headless=true' -var 'write_zeroes=yes' -except=vmware-iso vagrant.json
- packer build -var "vagrant_cloud_token=$VAGRANT_API_TOKEN" vagrant.json
only:
refs:
- release
variables:
- $SCHEDULED_PUBLISH == "TRUE"
resource_group: vm-build
......@@ -17,11 +17,10 @@ You'll need the following dependencies:
* vagrant (for vagrant images)
* qemu (for libvirt provider support)
* virtualbox (for virtualbox support)
* VMware Workstation Pro (for vmware support)
## Variables
Here is an overview over all variables you can set in `vagrant.json` or
`local.json`:
`cloud.json`:
* `iso_url`: the url to the ISO. This can be an url or a filepath
beginning with `file://`
......@@ -30,29 +29,24 @@ Here is an overview over all variables you can set in `vagrant.json` or
* `iso_checksum_type`: this specifies the hashing algorithm for the
checksum.
* `disk_size`: this specifices the disk size in bytes.
* `memory`: this specifies the size of the RAM in bytes.
* `cpus`: this specifies the number of cores for your VM.
* `headless`: this sets GUI on or off.
* `vagrant_cloud_token`: here you can specify the vagrant cloud token for
uploading your box to the vagrantcloud. If you don't have a vagrant cloud
token you can ignore this variable. Without a token the boxes will be
built, but the upload step step will fail.
* `write_zeroes`: this variable is empty. if you set any string in this
variable it will fill the box with zeros to reduce the size. *NOTE: This
may overly tax your SSD's lifetime.*
* `boot_wait`: this specifies the time packer should wait for booting up
the ISO before entering any command.
## How to start the build process locally
If you want to build the boxes locally without uploading them to the Vagrant
cloud you need to edit the `local.json` before you start the build. set the
cloud you need to edit the `vagrant.json` before you start the build. set the
right `iso_url` and the right `iso_checksum_url`. Then you can start the build
for virtualbox only with the following command:
`packer build -only=virtualbox-iso local.json`
`packer build -only=qemu -except=publish vagrant.json`
## How to start the build process for official builds
The official builds are done on our Arch Linux Buildserver.
The official builds are done in our Arch Linux GitLab CI.
`packer build vagrant.json`
......@@ -60,7 +54,6 @@ The official builds are done on our Arch Linux Buildserver.
* virtualbox-iso
* qemu/libvirt
* vmware-iso
## Post-processors
......@@ -106,7 +99,7 @@ Start `packer` with `-parallel=false`:
```bash
$ packer build cloud.json
$ cp release/Arch-Linux-cloudimg-amd64-2020-02-24.img disk.img
$ cp release/Arch-Linux-cloudimg-amd64-2020-02-24.qcow2 disk.qcow2
# Copied from (with minor changes): https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html
$ { echo instance-id: iid-local01; echo local-hostname: cloudimg; } > meta-data
......@@ -117,13 +110,13 @@ $ printf "#cloud-config\npassword: passw0rd\nchpasswd: { expire: False }\nssh_pw
$ genisoimage -output seed.iso -volid cidata -joliet -rock user-data meta-data
## create a new qcow image to boot, backed by your original image
$ qemu-img create -f qcow2 -b disk.img boot-disk.img
$ qemu-img create -f qcow2 -b disk.qcow2 boot-disk.qcow2
## boot the image and login as 'arch' with password 'passw0rd'
## note, passw0rd was set as password through the user-data above,
## there is no password set on these images.
$ qemu-system-x86_64 -m 256 \
-net nic -net user,hostfwd=tcp::2222-:22 \
-drive file=boot-disk.img,if=virtio \
-drive file=boot-disk.qcow2,if=virtio \
-drive file=seed.iso,if=virtio
```
......@@ -3,39 +3,31 @@
"iso_url": "https://mirror.pkgbuild.com/iso/latest/archlinux-{{isotime \"2006.01\"}}.01-x86_64.iso",
"iso_checksum_url": "https://mirror.pkgbuild.com/iso/latest/sha1sums.txt",
"disk_size": "20480",
"memory": "1024",
"cpus": "2",
"headless": "true",
"write_zeroes": "",
"boot_wait": "60s",
"mirror": ""
"accelerator": "",
"mirror": "https://mirror.pkgbuild.com/$repo/os/$arch"
},
"builders": [
{
"name": "Arch-Linux-cloudimg-x86_64-{{isotime \"2006-01-02\"}}.img",
"type": "qemu",
"output_directory": "release",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"disk_discard": "unmap",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "arch",
"ssh_password": "arch",
"ssh_port": 22,
"ssh_timeout": "10000s",
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"vm_name": "Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2",
"headless": "{{user `headless`}}",
"qemuargs": [
[
"-m",
"{{user `memory`}}"
],
[
"-smp",
"{{user `cpus`}}"
]
],
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-cloud}.sh'<enter><wait>",
......@@ -53,37 +45,29 @@
"provision/cleanup.sh"
],
"execute_command": "echo 'arch'|sudo -S sh '{{.Path}}'"
},
{
"type": "shell",
"scripts": [
"provision/write_zeroes.sh"
],
"execute_command": "if [ ! -z \"{{user `write_zeroes`}}\" ]; then echo 'arch'|sudo -S sh '{{.Path}}'; fi"
}
],
"post-processors": [
[
{
"type": "checksum",
"checksum_types": [
"sha256"
],
"output": "release/Arch-Linux-cloudimg-x86_64-{{isotime \"2006-01-02\"}}.SHA256"
},
{
"name": "rename",
"type": "shell-local",
"inline": [
"mv release/packer-Arch-Linux-cloudimg-x86_64-{{isotime \"2006-01-02\"}}.img release/Arch-Linux-cloudimg-x86_64-{{isotime \"2006-01-02\"}}.img",
"sed -i 's/packer-//' release/Arch-Linux-cloudimg-x86_64-{{isotime \"2006-01-02\"}}.SHA256"
"mv output-qemu/Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2 Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.qcow2"
]
},
{
"type": "checksum",
"checksum_types": [
"sha256"
],
"output": "Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.SHA256"
},
{
"name": "sign",
"type": "shell-local",
"inline": [
"gpg --sign --detach-sign Arch-Linux-cloudimg-x86_64-{{isotime \"2006-01-02\"}}.SHA256"
"gpg --sign --detach-sign Arch-Linux-x86_64-cloudimg-{{isotime \"2006-01-02\"}}.SHA256"
]
}
]
......
......@@ -89,9 +89,10 @@ if [ -b "/dev/sda" ]; then
elif [ -b "/dev/vda" ]; then
grub-install /dev/vda
fi
sed -i -e 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' /etc/default/grub
sed -i 's/^GRUB_TIMEOUT=.*$/GRUB_TIMEOUT=1/' /etc/default/grub
# setup unpredictable kernel names
sed -i -e 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' /etc/default/grub
sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0"/' /etc/default/grub
sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"rootflags=compress-force=zstd\"/' /etc/default/grub
grub-mkconfig -o /boot/grub/grub.cfg
# lock root account
......
......@@ -13,28 +13,19 @@ else
fi
export device
memory_size_in_kilobytes=$(free | awk '/^Mem:/ { print $2 }')
swap_size_in_kilobytes=$((memory_size_in_kilobytes * 2))
sfdisk "$device" <<EOF
label: dos
size=${swap_size_in_kilobytes}KiB, type=82
type=83, bootable
EOF
memory_size_in_mebibytes=$(free -m | awk '/^Mem:/ { print $2 }')
swap_size_in_mebibytes=$((memory_size_in_mebibytes * 2))
mkswap "${device}1"
mkfs.ext4 -L "rootfs" "${device}2"
mount "${device}2" /mnt
sgdisk -g --clear -n 1:0:+10M $device -c 1:boot -t 1:ef02
sgdisk -n 2:0:+${swap_size_in_mebibytes}M $device -c 2:swap -t 2:8200
sgdisk -n 3:0:0 $device -c 3:root
partprobe
if [ -n "${MIRROR}" ]; then
echo "Server = ${MIRROR}" >/etc/pacman.d/mirrorlist
else
pacman -Sy --noconfirm reflector
reflector --age 12 --protocol https --sort rate --save /etc/pacman.d/mirrorlist
fi
pacstrap -M /mnt base linux grub openssh sudo polkit haveged netctl python reflector
swapon "${device}1"
genfstab -pU /mnt >>/mnt/etc/fstab
swapoff "${device}1"
mkswap /dev/disk/by-partlabel/swap
mkfs.btrfs /dev/disk/by-partlabel/root
mount -o compress-force=zstd PARTLABEL=root /mnt
echo "Server = ${MIRROR}" >/etc/pacman.d/mirrorlist
pacstrap /mnt base linux grub openssh sudo polkit haveged netctl python btrfs-progs reflector
arch-chroot /mnt /usr/bin/sed -i 's/^#Server/Server/' /etc/pacman.d/mirrorlist
arch-chroot /mnt /bin/bash
{
"variables": {
"iso_url": "https://mirror.pkgbuild.com/iso/latest/archlinux-{{isotime \"2006.01\"}}.01-x86_64.iso",
"iso_checksum_url": "https://mirror.pkgbuild.com/iso/latest/sha1sums.txt",
"disk_size": "20480",
"memory": "1024",
"cpus": "2",
"headless": "true",
"write_zeroes": "",
"boot_wait": "60s",
"mirror": ""
},
"builders": [
{
"type": "virtualbox-iso",
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"guest_os_type": "ArchLinux_64",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "10000s",
"shutdown_command": "sudo systemctl poweroff",
"guest_additions_mode": "disable",
"headless": "{{user `headless`}}",
"vboxmanage": [
[
"modifyvm",
"{{.Name}}",
"--memory",
"{{user `memory`}}"
],
[
"modifyvm",
"{{.Name}}",
"--cpus",
"{{user `cpus`}}"
]
],
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "qemu",
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "10000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"qemuargs": [
[
"-m",
"{{user `memory`}}"
],
[
"-smp",
"{{user `cpus`}}"
]
],
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "vmware-iso",
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "10000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"memory": "{{user `memory`}}",
"cpus": "{{user `cpus`}}",
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
}
],
"provisioners": [
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/virtualbox.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"virtualbox-iso"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/qemu.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"qemu"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/vmware.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"vmware-iso"
]
},
{
"type": "shell",
"scripts": [
"provision/write_zeroes.sh"
],
"execute_command": "if [ ! -z \"{{user `write_zeroes`}}\" ]; then echo 'vagrant'|sudo -S sh '{{.Path}}'; fi"
}
],
"post-processors": [
[
{
"type": "vagrant",
"keep_input_artifact": false,
"output": "Arch-Linux-x86_64-{{ .Provider }}-{{isotime \"2006-01-02\"}}.box"
}
]
]
}
......@@ -9,3 +9,6 @@ yes | sudo pacman -Scc
rm /etc/machine-id
# Remove pacman key ring for re-initialization
rm -rf /etc/pacman.d/gnupg/
sync
fstrim --all --verbose
#!/bin/bash
set -e
set -x
sudo pacman -S --noconfirm open-vm-tools
sudo systemctl enable vmtoolsd
#!/bin/bash
set -e
set -x
# Make sure unwritten data has been flushed beforehand
sync
# Write zeros to improve virtual disk compaction.
zerofile=$(/usr/bin/mktemp /zerofile.XXXXX)
dd if=/dev/zero of="$zerofile" bs=1M || true
rm -f "$zerofile"
sync
......@@ -3,44 +3,31 @@
"iso_url": "https://mirror.pkgbuild.com/iso/latest/archlinux-{{isotime \"2006.01\"}}.01-x86_64.iso",
"iso_checksum_url": "https://mirror.pkgbuild.com/iso/latest/sha1sums.txt",
"disk_size": "20480",
"memory": "1024",
"cpus": "2",
"headless": "true",
"vagrant_cloud_token": "PLACEHOLDER",
"write_zeroes": "",
"boot_wait": "60s",
"mirror": ""
"accelerator": "",
"mirror": "https://mirror.pkgbuild.com/$repo/os/$arch"
},
"builders": [
{
"type": "virtualbox-iso",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"hard_drive_discard": "true",
"guest_os_type": "ArchLinux_64",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "10000s",
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"guest_additions_mode": "disable",
"headless": "{{user `headless`}}",
"vboxmanage": [
[
"modifyvm",
"{{.Name}}",
"--memory",
"{{user `memory`}}"
],
[
"modifyvm",
"{{.Name}}",
"--cpus",
"{{user `cpus`}}"
]
],
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
......@@ -49,48 +36,22 @@
},
{
"type": "qemu",
"cpus": 2,
"memory": 1024,
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_discard": "unmap",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "10000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"qemuargs": [
[
"-m",
"{{user `memory`}}"
],
[
"-smp",
"{{user `cpus`}}"
]
],
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
"MIRROR='{{user `mirror`}}' bash install.sh < <(cat install-{chroot,common}.sh) && systemctl reboot<enter>"
]
},
{
"type": "vmware-iso",
"boot_wait": "{{user `boot_wait`}}",
"http_directory": "http",
"disk_size": "{{user `disk_size`}}",
"iso_checksum": "file:{{user `iso_checksum_url`}}",
"iso_url": "{{user `iso_url`}}",
"ssh_username": "vagrant",
"ssh_password": "vagrant",
"ssh_port": 22,
"ssh_timeout": "10000s",
"ssh_timeout": "2000s",
"shutdown_command": "sudo systemctl poweroff",
"headless": "{{user `headless`}}",
"memory": "{{user `memory`}}",
"cpus": "{{user `cpus`}}",
"accelerator": "{{user `accelerator`}}",
"disk_compression": true,
"boot_command": [
"<enter><wait10><wait10><wait10><wait10><wait10><enter><enter>",
"curl -O 'http://{{.HTTPIP}}:{{.HTTPPort}}/install{,-common,-chroot}.sh'<enter><wait>",
......@@ -122,30 +83,12 @@
"only": [
"qemu"
]
},
{
"type": "shell",
"scripts": [
"provision/postinstall.sh",
"provision/vmware.sh",
"provision/cleanup.sh"
],
"execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'",
"only": [
"vmware-iso"
]
},
{
"type": "shell",
"scripts": [
"provision/write_zeroes.sh"
],
"execute_command": "if [ ! -z \"{{user `write_zeroes`}}\" ]; then echo 'vagrant'|sudo -S sh '{{.Path}}'; fi"