Merge branch 'aurweb-review-apps' into 'master'

Misc changes for supporting aurweb's review apps need

See merge request !748

README.md

@@ -45,10 +45,14 @@ tasks.
 We use packer to build snapshots on hcloud to use as server base images.
 In order to use this, you need to install packer and then run
 
-    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_api_key --format env) packer/archlinux.json
+    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_api_key --format env) packer/archlinux.pkr.hcl
 
 This will take some time after which a new snapshot will have been created on the primary hcloud archlinux project.
 
+For the sandbox project please run
+
+    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_sandbox_infrastructure_api_key --format env | sed 's/_sandbox_infrastructure//') -var install_ec2_public_keys_service=true packer/archlinux.pkr.hcl
+
 #### Note about terraform
 
 We use terraform in two ways:

group_vars/all/dyn_dns.yml

@@ -6,3 +6,9 @@ dyn_dns_zones:
     allowed_ipv6: "{{ groups['geo_mirrors'] | map('extract', hostvars, ['ipv6_address']) }}"
     valid_qtypes: [TXT]
   _acme-challenge.riscv.mirror.pkgbuild.com: *acme_challenge
+  sandbox.archlinux.page:
+    key: sandbox
+    allowed_ipv4: "{{ groups['gitlab_runners'] | map('extract', hostvars, ['ipv4_address']) }}"
+    allowed_ipv6: "{{ groups['gitlab_runners'] | map('extract', hostvars, ['ipv6_address']) }}"
+    valid_qtypes: [A, AAAA]
+    subdomains: only

group_vars/all/vault_dyn_dns_keys.yml

 $ANSIBLE_VAULT;1.1;AES256
-61373835393530366133386434373162656332363939656235646235663333633532336435353266
-3364616435323230656233666633353535303436363433610a376133633938663634323932643764
-36656433366566623864636462383861636538363737343861316330306561373965626366363032
-6366373462303839660a653335623261306630623139643630323330633665393030333830653930
-37653166613264643537383734336163313537313334363635653062653832333638356361313461
-62353166393332326534356661653464333266383234396536383633323834333566633861643363
-66316162356566343964623237356264633564646634653834326363386235333361656332386265
-39333463343365393962663637666333376236366638306361316435306537643031346162346464
-33313466353666353136386463353831353365643333613066326136343234343636343833346465
-64343962303766303436613538616165623837383837303230623135623562303664333764323834
-62313864653234653138336134303638666234376631663361396662653863643433313864303330
-63663034353461346562
+62393237353533363738376335336564623464336332393733306465333339376130613338356537
+6166666538303939313238323238616433653036376662360a323663613934636539333365303166
+33343266613234363965363233666165383333343862326436313935636631326266363462613033
+3937393135656534370a663035633362643931653864336336396535373038396165633934366433
+31656663396538376337373762386162386665353639336235363233643139303763333861376339
+62306130363039376431396234333030616235306530343336326237656638636435363038663931
+39356535643265616337306530393962373537336335333764363565313939373565326561613066
+36633931656662393538353836353365386634663736356131323435333265653832656162306230
+64326535353532373137656535386531333536353531643863646135386664333030363564376463
+61386537306235356666353761383237336133376665393365663636386238373534623833306430
+37323336623537613034643763363439643063633433323431623932646465363230316533356337
+34623964653036383766316336373462363562333963663939333431643665643737643164396565
+38396332356630366665666239656562313430363432366639373235343430653236356438643131
+65623438313963356630333939636663393539656463376339326631636263313564636432343635
+39656466323965626264623332393630333035396638653039343536373337643165313564333363
+36626239303836383932336537313061663961636137396162303838356661386636303262653633
+33336665306634363866386237623733643663313136373037376631363364343161373731626637
+30346433666230663564643731616566663339393166343061333033386462366663383839653631
+363865646464333236663262323265376363

misc/vaults/vault_hetzner.yml

 $ANSIBLE_VAULT;1.2;AES256;super
-63616530316632643063333435366135363735333562653039393933666366623930323037643439
-3864643637376534353337626135653830376430333830350a376532353336323230373762303065
-65343939643931336266656337666230303433366432623037393964373737656366643835396430
-3438323638323635620a323132373862316461373930343035646336343466646261363633663263
-37623362643865626264656663653762313632336139623866306537396234303936643366613561
-62333339313638303831643964383633356133343335303130653836353636643431353039636237
-36353130363830346238626433396661346136656532353231663930666130323439636430626334
-39656166386631376363643038636135316164666434363131646231313231303765376533306165
-63303831613061376532333433373132386139363666373936326431393462373632363464623364
-35346533373935363862633536333765366563343538616430396134303937623962303464623538
-64316461366239636138663734613732613437336135356165333435663364376133393966613834
-61343431653939366136323239623836663235653965333364343062373661656433376166393537
-63376136623563633166653765616663306335396637336430646266326233623461343464333130
-31366330306639633265643234623733363533383634633331666430646233363464393662636565
-35666265303034303832326465326437623734623233346564666634386539383938623161326663
-62356562626230386534333438316261333235643166346535666639643166383239343261376236
-34313130383836323366643063353336336534353436303334633632333930373736363035666434
-64663566373531393230663464346265303766616237643834373965383337623065386237656465
-35316265356534616165323566353064633137663132376363616531616265306463656164666331
-38646265636166323736363464343139323566656132353338313663636465316535623131366139
-61353230643061313162383133313031326333383638666239653433383365383030
+61643761613134633964616166333066343935306130303264316637646563376231376466326164
+3063376463383931373961613930313034363164643039620a393033653565363237383236373834
+39303835393230646362656330366238303435393263336430633130643737616332633731656637
+3439616635343133340a623331383938303164663461333135336333336431383936363661313061
+63653133396164353637316162653939303166633135383066636131366337383965383432303665
+65616564633434323761646534323832393162313330343136353138313364333564393731363762
+61626266363062383661623161313631636431613364663761646664626366326566323364636430
+36363736663032633131366136653839383836343034363734353431383362336336363663616166
+62633830663062643133323862346532613139393939626166616534373336626466343761376666
+66623133376137303663636531323839383165643831613065363936376237353438366166646139
+36316337393036303563393263316363306535663337656436386434363033373434373061613530
+39623064623638366465373234356463376364383461643965366438363334336463613530326265
+63376336643831653232373062653431383335326332356335666134336662613131316662386436
+65636230623036393738353937613935343033303562636133653765656239623066353630373964
+61316334363636643234323963616335333637666438366230386534643435326536313932666433
+66653038643966336430333666393731396439333965623531643638353861636664636537623863
+37326562636566386331663334323466306135386232373836373761633734316464346465386237
+62393565663532336335323533313135346233633435346330346161646564326338613262636431
+61623864383032356666333562303734636363643765356137626164383831353836383066653439
+34356233333034376565336539363738303336663532393332313632366235303564653331376561
+32636437643466363534623163353064653231666666373436643735313737353564633965363437
+36623438316366333131363365346233343932653237386133323030366133663439653965316434
+31383164386431383932396531623332653339383133353737303630623036666363646135376261
+36393234303363323866333438313336313631376637633032643337393565353263353337353461
+62656236393263643337356366666539663133623838666437396362393532386230336639613234
+64626139323266653830353431363063323663383565313166373438633734363361616262623234
+35623132646432363361326662633834663966613965313862396261313162386330306637356365
+38383438373135363765643932323931366332613666663331363436613738643136616661666261
+37396166316330646431376335636336613564353432313037343164643530353338343563306535
+64613064316563323038313961323536323665656436326537353131313439666336373764353233
+36313635363833303938333836383063323130353566646639393730303932343830613436313033
+33386263356134356331

packer/archlinux.json

-{
-    "variables": {
-        "hetzner_cloud_api_key": null
-    },
-    "sensitive-variables": ["hetzner_cloud_api_key"],
-    "builders": [{
-        "type": "hcloud",
-        "snapshot_name": "archlinux-{{ isotime \"2006-01-02T15:04\" }}",
-        "snapshot_labels": {
-            "custom_image": "archlinux"
-        },
-        "token": "{{ user `hetzner_cloud_api_key` }}",
-        "image": "ubuntu-22.04",
-        "server_type": "cx11",
-        "ssh_username": "root",
-        "location": "fsn1",
-        "rescue": "linux64"
-    }],
-    "provisioners": [{
-        "type": "ansible",
-        "playbook_file": "playbooks/tasks/install_arch.yml",
-        "host_alias": "packer-base-image",
-        "inventory_directory": ".",
-        "use_proxy": false
-    }]
-}

packer/archlinux.pkr.hcl

+# https://www.packer.io/docs/templates/hcl_templates/blocks/packer
+packer {
+  required_plugins {
+    ansible = {
+      source  = "github.com/hashicorp/ansible"
+      version = ">= 1.1.0"
+    }
+    hcloud = {
+      source  = "github.com/hashicorp/hcloud"
+      version = ">= 1.0.0"
+    }
+  }
+}
+
+# https://www.packer.io/docs/templates/hcl_templates/variables#type-constraints
+variable "hetzner_cloud_api_key" {
+  type      = string
+  sensitive = true
+}
+
+variable "install_ec2_public_keys_service" {
+  type    = bool
+  default = false
+}
+
+# https://www.packer.io/docs/templates/hcl_templates/blocks/source
+source "hcloud" "rescue" {
+  image       = "ubuntu-22.04"
+  location    = "fsn1"
+  rescue      = "linux64"
+  server_type = "cx11"
+  snapshot_labels = {
+    custom_image = "archlinux"
+  }
+  snapshot_name = "archlinux-${timestamp()}"
+  ssh_username  = "root"
+  token         = var.hetzner_cloud_api_key
+}
+
+# https://www.packer.io/docs/templates/hcl_templates/blocks/build
+build {
+  sources = ["source.hcloud.rescue"]
+
+  provisioner "ansible" {
+    host_alias          = "packer-base-image"
+    inventory_directory = "."
+    playbook_file       = "playbooks/tasks/install_arch.yml"
+    extra_arguments = [
+      "--extra-vars", jsonencode({
+        install_ec2_public_keys_service : var.install_ec2_public_keys_service
+      })
+    ]
+    use_proxy = false
+  }
+}

roles/gluebuddy/defaults/main.yml

@@ -5,3 +5,4 @@ gitlab_bots:
   - project_19796_bot
   - project_10177_bot
   - project_57011_bot_12149c4ff92f18dd46696e89f2f3e703
+  - project_57074_bot_9fec0b36c8d721eefc66f25d8cf9acb2

roles/install_arch/files/ec2-public-keys

+#!/usr/bin/python
+
+import os
+from pathlib import Path
+
+import requests
+
+data = requests.get("http://169.254.169.254/2009-04-04/meta-data/public-keys")
+data.raise_for_status()
+
+path = Path("/root/.ssh/authorized_keys")
+path.parent.mkdir(mode=0o700, exist_ok=True)
+os.chmod(path.parent, 0o700)
+
+with open(path, "w") as file:
+    for key in data.json():
+        file.write(f"{key}\n")
+os.chmod(path, 0o600)

roles/install_arch/files/ec2-public-keys.service

+[Unit]
+Description=Fetch SSH public keys from the metadata service
+Before=sshd.service
+After=systemd-networkd-wait-online.service
+ConditionFirstBoot=yes
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/bin/ec2-public-keys
+
+[Install]
+WantedBy=multi-user.target

roles/install_arch/tasks/main.yml

@@ -171,6 +171,13 @@
 - name: Setup pacman-init.service on first boot
   copy: src=pacman-init.service dest=/mnt/etc/systemd/system/ owner=root group=root mode=0644
 
+- name: Setup ec2-public-keys on first boot
+  copy: src={{ item.src }} dest=/mnt/{{ item.dest }} owner=root group=root mode={{ item.mode }}
+  loop:
+    - {src: ec2-public-keys, dest: /usr/local/bin/, mode: 755}
+    - {src: ec2-public-keys.service, dest: /etc/systemd/system/, mode: 644}
+  when: install_ec2_public_keys_service | default(false)
+
 - name: Remove generated keyring in the installation process
   file: path=/mnt/etc/pacman.d/gnupg state=absent
 
@@ -178,7 +185,7 @@
   file: path=/mnt/etc/machine-id state=absent
 
 - name: Enable services inside chroot
-  command: chroot /mnt systemctl enable sshd systemd-networkd systemd-resolved fstrim.timer pacman-init
+  command: chroot /mnt systemctl enable sshd systemd-networkd systemd-resolved fstrim.timer pacman-init {{ 'ec2-public-keys' if install_ec2_public_keys_service | default(false) }}
   register: chroot_systemd_services
   changed_when: "chroot_systemd_services.rc == 0"
 
@@ -187,6 +194,7 @@
     name: root_ssh
   vars:
     root_ssh_directory: /tmp/root.x86_64/mnt/root/.ssh
+  when: not install_ec2_public_keys_service | default(false)
 
 - name: Configure sshd
   template: src=sshd_config.j2 dest=/mnt/etc/ssh/sshd_config owner=root group=root mode=0644

tf-stage1/archlinux.tf

@@ -473,6 +473,14 @@ resource "hetznerdns_record" "archlinux_page_origin_ns1" {
   ttl     = 86400
 }
 
+resource "hetznerdns_record" "archlinux_page_sandbox_ns1" {
+  zone_id = hetznerdns_zone.archlinux_page.id
+  name    = "sandbox"
+  value   = "redirect.archlinux.org."
+  type    = "NS"
+  ttl     = 86400
+}
+
 # TODO: Commented currently as we have no idea how to handle SOA stuff with Terraform:
 # https://github.com/timohirt/terraform-provider-hetznerdns/issues/20
 # https://gitlab.archlinux.org/archlinux/infrastructure/-/merge_requests/62#note_4040