.ansible-lint

@@ -4,13 +4,19 @@ exclude_paths:
   - playbooks/tasks
   - roles/prometheus/files/node.rules.yml
 skip_list:
-  # line too long (x > 80 characters) (line-length)
-  - 'line-length'
-  # yaml: too many spaces inside braces (braces)
-  - 'braces'
+  # yaml: line too long (x > 160 characters) (yaml[line-length])
+  - yaml[line-length]
+  # yaml: too many spaces inside braces (yaml[braces])
+  - yaml[braces]
   # Do not recommend running tasks as handlers
-  - 'no-handler'
+  - no-handler
   # Do not force galaxy info in meta/main.yml
-  - 'meta-no-info'
+  - meta-no-info
   # Allow package versions to be specified as 'latest'
-  - 'package-latest'
+  - package-latest
+  # Don't require fully-qualified collection names
+  - fqcn
+  # Allow free-form module calling syntax
+  - no-free-form
+  # Allow role includes with unprefixed role variables
+  - var-naming[no-role-prefix]

.gitlab-ci.yml

 image: "archlinux:latest"
 
-before_script:
-  - pacman -Syu --needed --noconfirm ansible-lint yamllint terraform
-
 ansible-lint:
+  before_script:
+    - pacman -Syu --needed --noconfirm ansible-lint ansible python-jmespath
   script:
     # Fix weird ansible bug: https://github.com/trailofbits/algo/issues/1637
     # This probably happens due to gitlab-runner mounting the git repo into the container
     - chmod o-w .
     # Fix syntax-check rule (https://github.com/ansible-community/ansible-lint/issues/1350#issuecomment-778764110)
-    - sed "s/,hcloud_inventory.py//" -i ansible.cfg
-    - sed "/^vault_password_file/d" -i ansible.cfg
+    - sed -i "/^vault_identity_list/d" ansible.cfg
+    - sed -i -e "/vars_files:/d" -e "/misc\/vaults\/vault_/d" playbooks/*.yml
     # Fix load-failure: Failed to load or parse file
     - ansible-lint $(printf -- "--exclude %s " */*/vault_*)
 
 terraform-validate:
+  before_script:
+    - pacman -Syu --needed --noconfirm terraform diffutils
   script:
     - cd tf-stage1
     - terraform init -backend=false

.gitlab/issue_templates/New Official Project.md

@@ -28,16 +28,28 @@ If you want to add a new official project, here are some guidelines to follow:
        to a protected branch of the project.
 1. [ ] If a secure runner is used, create an MR to make sure the project's `.gitlab-ci.yml` specifies
        `tags: secure`.
-1. [ ] Make sure that the *Push Rules* in https://gitlab.archlinux.org/archlinux/arch-boxes/-/settings/repository
+1. [ ] Make sure that the *Push Rules* in https://gitlab.archlinux.org/archlinux/my-example/-/settings/repository
        reflect these values:
-   - `Committer restriction`: `on`
+   - `Reject unverified users`: `on`
    - `Reject unsigned commits`: `on`
    - `Do not allow users to remove tags with git push`: `on`
-   - `Check whether author is a gitlab user`: `on`
-   - `Prevent committing secrets to git`: `on`
+   - `Check whether author is a GitLab user`: `on`
+   - `Prevent pushing secret files`: `on`
    - All of these should be activated by default as per group rules but it's good to check.
 1. [ ] The *Protected Branches* in https://gitlab.archlinux.org/archlinux/my-example/-/settings/repository should specify
        `Allowed to merge` and `Allowed to push` as `Developers + Maintainers.`
+1. [ ] Disable unneeded project features under *Visibility, project features, permissions* (https://gitlab.archlinux.org/archlinux/my-example/edit)
+   Always:
+   - `Users can request access`: `off`
+   Often, but not always:
+   - Repository -> Container registry
+   - Repository -> Git Large File Storage (LFS)
+   - Repository -> Packages
+   - Analytics
+   - Requirements
+   - Security & Compliance
+   - Wiki
+   - Operations
 
 ## GitHub.com mirroring checklist
 
@@ -73,6 +85,13 @@ If you want to add a new official project, here are some guidelines to follow:
    - `Wiki`
    - `Issues`
    - `Projects`
+1. [ ] Go to https://github.com/archlinux/my-example/settings/hooks and add a new webhook
+   - `Payload URL`: `$(misc/get_key.py misc/vaults/vault_github.yml github_pull_closer_webhook_url)`
+   - `Content type`: `application/json`
+   - `Which events would you like to trigger this webhook?`
+     - `Let me select individual events.`: `Pull requests`
 1. [ ] In the GitHub description of the mirrored project, append " (read-only mirror)" so that people know it's a mirror.
 1. [ ] Disable `Packages` and `Environments` from being shown on the main page.
 1. [ ] In the website field put the full url to the repository on our GitLab.
+1. [ ] Go to https://github.com/archlinux/my-example/settings/access and remove the GitHub account `archlinux-github`
+1. [ ] Go to https://github.com/orgs/archlinux/teams/read-only-mirrors/repositories and add the repository with `write` permission

.gitlab/issue_templates/Offboarding.md

@@ -7,35 +7,51 @@ This template should be used for offboarding Arch Linux team members.
 ## Details
 
 - **Team member username**:
-- **Currently held roles**: <!-- Add known roles here like TU, DevOps, etc -->
+- **Currently held roles**: <!-- Add known roles here like Package Maintainer, DevOps, etc -->
 - **Removal request**: <!-- Add link to relevant mailing list mail -->
   - **Voting result**: <!-- Add link to relevant mailing list mail -->
 
 ## All roles checklist
 
-- [ ] Remove user email by reverting instructions from `docs/email.md`.
+- [ ] Remove user email by reverting instructions from [`docs/email.md`](docs/email.md).
+  - [ ] Remove entry in [`group_vars/all/archusers.yml`](group_vars/all/archusers.yml).
+  - [ ] Remove SSH pubkey from `pubkeys/<username>.pub`.
+  - [ ] Run `ansible-playbook -t archusers  $(git grep -l archusers playbooks/ | grep -v phrik)`.
+  - [ ] Setup forwarding if requested (please add the current date as a comment above the mail address in Postfix's `users` file).
+    - [ ] Inform the user of the conditions for forwarding.
+      - In most cases we only offer forwarding for 6 months.
+      - We will inform the user prior to disabling the forwarding.
+      - The forwarding can be extended if there are good reasons for doing so.
 - [ ] Set user to inactive in archweb: https://www.archlinux.org/admin/auth/user/
+- [ ] Remove member from [staff mailing list](https://lists.archlinux.org/mailman3/lists/staff.lists.archlinux.org/members/member/).
+- [ ] Moderate email address on [arch-dev-public](https://lists.archlinux.org/mailman3/lists/arch-dev-public.lists.archlinux.org/members/member/) (find member and moderate).
+- [ ] Ask the user to leave `#archlinux-staff` on Libera Chat and forget the password.
+- [ ] Remove staff cloak on Libera Chat ([Group contacts](https://wiki.archlinux.org/title/Arch_IRC_channels#Libera_Chat_group_contacts)). cc @archlinux/teams/irc/group-contacts
+- [ ] Remove the user from relevant staff groups on Keycloak.
+- [ ] Move the user from the public list of their usergroup on archweb ([support staff](https://archlinux.org/people/support-staff/) / [TUs](https://archlinux.org/people/trusted-users/) / [devs](https://archlinux.org/people/developers/)) to the respective fellow site ([fellow support staff](https://archlinux.org/people/support-staff-fellows/) / [fellow TUs](https://archlinux.org/people/trusted-user-fellows/) / [fellow devs](https://archlinux.org/people/developer-fellows/))
+- [ ] Remove the user from the Arch Linux GitHub organisation
 
-## TU/Developer offboarding checklist
+## Main key offboarding checklist
 
-- [ ] Remove entry in `group_vars/all/archusers.yml`.
-- [ ] Remove SSH pubkey from `pubkeys/<username>.pub`.
-- [ ] Run `ansible-playbook -t archusers playbooks/*.yml`.
-- [ ] Remove the user from the `Trusted Users`/`Developers` groups on Keycloak.
-- [ ] Moderate email address on [arch-dev-public](https://lists.archlinux.org/admin/arch-dev-public/members) (find member and moderate)
-- [ ] Remove member from [arch-tu mailing lists](https://lists.archlinux.org/admin/arch-tu/members)
-- [ ] Remove member from [staff mailing lists](https://lists.archlinux.org/admin/staff/members)
+- [ ] Remove user email for the `master-key.archlinux.org` subdomain by reverting instructions from [`docs/email.md`](docs/email.md).
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"Remove Main Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=Remove%20Main%20Key) template.
+
+## Package Maintainer/Developer offboarding checklist
+
+- [ ] Remove member from [arch-tu](https://lists.archlinux.org/mailman3/lists/arch-tu.lists.archlinux.org/members/member/) and/or [arch-dev](https://lists.archlinux.org/mailman3/lists/arch-dev.lists.archlinux.org/members/member/) mailing lists.
+- [ ] Ask the user to leave `#archlinux-tu` and/or `#archlinux-dev` aswell as `#archlinux-packaging` on Libera Chat and forget the password(s).
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"Remove Packager Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=Remove%20Packager%20Key) template.
+- [ ] Remove [stale package relations](https://archlinux.org/packages/stale_relations/) for the now inactive user.
+- [ ] Remove their extended permissions on AURweb
 
 ## DevOps offboarding checklist
 
-- [ ] Remove entries in `group_vars/all/root_access.yml`.
-- [ ] Run `ansible-playbook -t root_ssh playbooks/*.yml`.
+- [ ] Remove entries in [`group_vars/all/root_access.yml`](group_vars/all/root_access.yml).
+- [ ] Run `ansible-playbook -t root_ssh playbooks/all-hosts-basic.yml`.
 - [ ] Run `ansible-playbook playbooks/hetzner_storagebox.yml playbooks/rsync.net.yml`.
-- [ ] Remove the user from the `DevOps` group on Keycloak.
-- [ ] Remove member from [arch-devops-private mailing lists](https://lists.archlinux.org/admin/arch-devops-private/members)
-- [ ] Remove pubkey from [Hetzner's key management](https://robot.your-server.de/key/index)
+- [ ] Remove member from [arch-devops-private mailing lists](https://lists.archlinux.org/mailman3/lists/arch-devops-private.lists.archlinux.org/members/member/).
+- [ ] Remove pubkey from [Hetzner's key management](https://robot.your-server.de/key/index).
 
 ## Wiki Administrator checklist
 
-- [ ] Remove the user from the `Wiki Admins` group on Keycloak.
-- [ ] Remove member from [arch-wiki-admins mailing list](https://lists.archlinux.org/admin/arch-wiki-admins/members).
+- [ ] Remove member from [arch-wiki-admins mailing list](https://lists.archlinux.org/mailman3/lists/arch-wiki-admins.lists.archlinux.org/members/member/).

.gitlab/issue_templates/Onboarding.md

@@ -2,50 +2,80 @@
 This template should be used for onboarding new Arch Linux team members.
 It can also be used as a reference for adding new roles to an existing team member.
 -->
+/confidential
+<!--
+NOTE: Do not remove the above short actions.
+They ensure that the ticket is created confidential and that personal
+information is not publicly visible.
+-->
 
 # Onboarding an Arch Linux team member
 
 ## Details
 
-- **Team member username**:
+- **Team member username**: <!-- Used for SSO account and @archlinux.org e-mail address -->
 - **Application**: <!-- Add link to relevant mailing list mail -->
-  - **Voting result**: <!-- Add link to relevant mailing list mail -->
+- **Voting result**: <!-- Add link to relevant mailing list mail -->
+- **SSH public key**: <!-- Add this when a user's access to machines is added or updated -->
+- **Full Name**: <!-- Relevant for all new users -->
+- **Personal e-mail address**: <!-- Relevant for users who will get a new archweb and/or SSO account -->
+- **PGP key ID used with personal e-mail address**: <!-- Relevant for users who will get a new archweb account -->
+- **Communication e-mail address**: [arch, personal] <!-- Relevant for users who will be signed up to mailing lists. Either choose "arch" or "personal". -->
+
+<!--
+NOTE: When creating this ticket as the sponsor for a new trusted user or
+support staff member, attach the above information as a clearsigned document to
+this ticket.
+https://www.gnupg.org/gph/en/manual/x135.html
+-->
 
 ## All roles checklist
 
-- [ ] Add new user email as per `docs/email.md`.
-- [ ] Create a new user in archweb: https://www.archlinux.org/devel/newuser/
-      This is also linked in the django admin backend at the top
-- [ ] Subscribe user to internal [staff mailing list](https://lists.archlinux.org/admin/staff/members/add)
+- [ ] Add user mail if TU or developer, or support staff and **communication e-mail address** is arch.
+  - [ ] Add new user email as per [`docs/email.md`](docs/email.md).
+  - [ ] Add entry in [`group_vars/all/archusers.yml`](group_vars/all/archusers.yml).
+    - If support staff `hosts` should be set to `mail.archlinux.org`.
+    - `homedir.archlinux.org` is also allowed for support staff, but it is opt-in.
+  - [ ] Add SSH pubkey to `pubkeys/<username>.pub`.
+  - [ ] Run `ansible-playbook -t archusers $(git grep -l archusers playbooks/ | grep -v phrik)`.
+- [ ] Create a new user in [archweb](https://www.archlinux.org/devel/newuser/). Select the appropriate group membership and allowed repos (if applicable).
+- [ ] Subscribe **communication e-mail address** to internal [staff mailing list](https://lists.archlinux.org/mailman3/lists/staff.lists.archlinux.org/mass_subscribe/).
+- [ ] Allow sending from **communication e-mail address** on [arch-dev-public](https://lists.archlinux.org/mailman3/lists/arch-dev-public.lists.archlinux.org/members/member/) (subscribe and/or find address and remove moderation).
+- [ ] Give the user access to `#archlinux-staff` on Libera Chat.
+- [ ] Give the user a link to our [staff services page](https://wiki.archlinux.org/title/DeveloperWiki:Staff_Services).
+- [ ] Replace the **Team member username** with the @-prefixed username on Gitlab.
+- [ ] Remove personal information (such as **Full Name** and **Personal e-mail
+  address**, as well as the clearsigned representation of this data), remove
+  the description history and make the issue non-confidential.
+- [ ] Request staff cloak on Libera Chat ([Group contacts](https://wiki.archlinux.org/title/Arch_IRC_channels#Libera_Chat_group_contacts)) cc @archlinux/teams/irc/group-contacts
+
+## Main key onboarding checklist
+
+- [ ] Add new user email for the `master-key.archlinux.org` subdomain as per [`docs/email.md`](docs/email.md).
+  <!-- The ticket should be created by the developer becoming a new main key holder -->
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"New Main Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=New%20Main%20Key) template.
 
-## Developer onboarding checklist
+## Package Maintainer/Developer onboarding checklist
 
-- [ ] Add entry in `group_vars/all/archusers.yml`.
-- [ ] Add SSH pubkey to `pubkeys/<username>.pub`.
-- [ ] Run `ansible-playbook -t archusers playbooks/*.yml`.
-- [ ] Assign the user to the `Developers` groups on Keycloak.
-- [ ] Subscribe user to internal [arch-dev mailing list](https://lists.archlinux.org/admin/arch-dev/members/add)
-- [ ] Whitelist email address on [arch-dev-public](https://lists.archlinux.org/admin/arch-dev-public/members) (find member and unmoderate)
+<!-- The ticket should be created by a sponsor of the new packager -->
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"New Packager Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=New%20Packager%20Key) template.
+- [ ] Assign the user to the correct group in the `Arch Linux Staff/Package Maintainer Team/` group on Keycloak.
+- [ ] Assign the user to the `Package Maintainers` or `Developers` group on [archlinux.org](https://archlinux.org/admin/auth/user/).
+- [ ] Subscribe **communication e-mail address** to internal [arch-tu](https://lists.archlinux.org/mailman3/lists/arch-tu.lists.archlinux.org/mass_subscribe/) or [arch-dev](https://lists.archlinux.org/mailman3/lists/arch-dev.lists.archlinux.org/mass_subscribe/) mailing list.
+- [ ] Give the user access to `#archlinux-tu` or `#archlinux-dev` aswell as `#archlinux-packaging` on Libera Chat.
 
-## TU onboarding checklist
+## Support staff checklist
 
-- [ ] Add entry in `group_vars/all/archusers.yml`.
-- [ ] Add SSH pubkey to `pubkeys/<username>.pub`.
-- [ ] Run `ansible-playbook -t archusers playbooks/*.yml`.
-- [ ] Assign the user to the `Trusted Users` groups on Keycloak.
-- [ ] Whitelist email address on [arch-dev-public](https://lists.archlinux.org/admin/arch-dev-public/members) (find member and unmoderate)
-- [ ] Subscribe user to internal [arch-tu mailing list](https://lists.archlinux.org/admin/arch-tu/members/add)
+- [ ] Assign the user to the proper support staff group on Keycloak.
 
 ## DevOps onboarding checklist
 
-- [ ] Add entries in `group_vars/all/root_access.yml`.
-- [ ] Run `ansible-playbook -t root_ssh playbooks/*.yml`.
+- [ ] Add entries in [`group_vars/all/root_access.yml`](group_vars/all/root_access.yml).
+- [ ] Run `ansible-playbook -t root_ssh playbooks/all-hosts-basic.yml`.
 - [ ] Run `ansible-playbook playbooks/hetzner_storagebox.yml playbooks/rsync.net.yml`.
-- [ ] Assign the user to the `DevOps` group on Keycloak.
-- [ ] Subscribe user to [arch-devops-private mailing lists](https://lists.archlinux.org/admin/arch-devops-private/members/add)
+- [ ] Subscribe **communication e-mail address** to internal [arch-devops-private](https://lists.archlinux.org/mailman3/lists/arch-devops-private.lists.archlinux.org/mass_subscribe/) mailing list.
 - [ ] Add pubkey to [Hetzner's key management](https://robot.your-server.de/key/index) for Dedicated server rescue system.
 
 ## Wiki Administrator checklist
 
-- [ ] Assign the user to the `Wiki Admins` group on Keycloak.
-- [ ] Subscribe the user to the [arch-wiki-admins mailing list](https://lists.archlinux.org/admin/arch-wiki-admins/members/add).
+- [ ] Subscribe **communication e-mail address** to the [arch-wiki-admins](https://lists.archlinux.org/mailman3/lists/arch-wiki-admins.lists.archlinux.org/mass_subscribe/) mailing list.

README.md

@@ -9,18 +9,18 @@ This repository contains the complete collection of ansible playbooks and roles
 
 Install these packages:
   - terraform
-  - python-typer
+  - python-click
   - python-jmespath
   - moreutils (for playbooks/tasks/reencrypt-vault-key.yml)
 
 ### Instructions
 
 All systems are set up the same way. For the first time setup in the Hetzner rescue system,
-run the provisioning script: `ansible-playbook playbooks/tasks/install-arch.yml -l $host`.
+run the provisioning script: `ansible-playbook playbooks/tasks/install_arch.yml -l $host`.
 The provisioning script configures a sane basic systemd with sshd. By design, it is NOT idempotent.
 After the provisioning script has run, it is safe to reboot.
 
-Once in the new system, run the regular playbook: `HCLOUD_TOKEN=$(misc/get_key.py misc/vault_hetzner.yml hetzner_cloud_api_key) ansible-playbook playbooks/$hostname.yml`.
+Once in the new system, run the regular playbook: `HCLOUD_TOKEN=$(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_api_key) ansible-playbook playbooks/$hostname.yml`.
 This playbook is the one regularity used for administrating the server and is entirely idempotent.
 
 When adding a new machine you should also deploy our SSH known_hosts file and update the SSH hostkeys file in this git repo.
@@ -29,26 +29,30 @@ It will also deploy any new SSH host keys to all our machines.
 
 #### Note about GPG keys
 
-The `root_access.yml` file contains the `root_gpgkeys` variable that determine the users that have access to the vault, as well as the borg backup keys.
-All the keys should be on the local user gpg keyring and at **minimum** be locally signed with `--lsign-key`. This is necessary for running either the reencrypt-vault-key
-or the fetch-borg-keys tasks.
+The `root_access.yml` file contains the `vault_default_pgpkeys` variable which
+determines the users that have access to the `default` vault, as well as the
+borg backup keys. A separate `super` vault exists for storing highly sensitive
+secrets like Hetzner credentials; access to the `super` vault is controlled by
+the `vault_super_pgpkeys` variable.
 
-#### Note about Ansible dynamic inventories
-
-We use a dynamic inventory script in order to automatically get information for
-all servers directly from hcloud. You don't really have to do anything to make
-this work but you should keep in mind to NOT add hcloud servers to `hosts`!
-They'll be available automatically.
+All the keys should be on the local user gpg keyring and at **minimum** be
+locally signed with `--lsign-key` (or if you use TOFU, have `--tofu-policy
+good`). This is necessary for running any of the `reencrypt-vault-default-key`,
+`reencrypt-vault-super-key `or `fetch-borg-keys` tasks.
 
 #### Note about packer
 
 We use packer to build snapshots on hcloud to use as server base images.
 In order to use this, you need to install packer and then run
 
-    packer build -var $(misc/get_key.py misc/vault_hetzner.yml hetzner_cloud_api_key --format env) packer/archlinux.json
+    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_api_key --format env) packer/archlinux.pkr.hcl
 
 This will take some time after which a new snapshot will have been created on the primary hcloud archlinux project.
 
+For the sandbox project please run
+
+    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_sandbox_infrastructure_api_key --format env | sed 's/_sandbox_infrastructure//') -var install_ec2_public_keys_service=true packer/archlinux.pkr.hcl
+
 #### Note about terraform
 
 We use terraform in two ways:
@@ -65,7 +69,7 @@ but for the time being, this is what we're stuck with.
 The very first time you run terraform on your system, you'll have to init it:
 
     cd tf-stage1  # and also tf-stage2
-    terraform init -backend-config="conn_str=postgres://terraform:$(../misc/get_key.py group_vars/all/vault_terraform.yml vault_terraform_db_password)@state.archlinux.org"
+    terraform init -backend-config="conn_str=postgres://terraform:$(../misc/get_key.py ../group_vars/all/vault_terraform.yml vault_terraform_db_password)@state.archlinux.org?sslmode=verify-full"
 
 After making changes to the infrastructure in `tf-stage1/archlinux.tf`, run
 
@@ -119,15 +123,22 @@ Arch-audit can be used to find servers in need of updates for security issues.
 
     ansible all -a "arch-audit -u"
 
-#### Updating servers
+### Semi-automated server upgrades
+
+For updating all servers in a mostly unattented manner, the following playbook
+can be used:
 
-The following steps should be used to update our managed servers:
+    ansible-playbook playbooks/tasks/upgrade-servers.yml [-l SUBSET]
 
-  * pacman -Syu
-  * manually update the kernel, since it is in IgnorePkg by default
-  * sync
-  * checkservices
-  * reboot
+It runs `pacman -Syu` on the targeted hosts in batches and then reboots them.
+If any server fails to reboot successfully, the rolling update stops and
+further batches are cancelled. To display the packages updated on each host,
+you can pass the `--diff` option to ansible-playbook.
+
+Using this update method, `.pacnew` files are left unmerged which is OK for
+most configuration files that are managed by Ansible. However, care must be
+taken with updates that require manual intervention (e.g. major PostgreSQL
+releases).
 
 ## Servers
 
@@ -135,25 +146,19 @@ This section has been moved to [docs/servers.md](docs/servers.md).
 
 ## Ansible repo workflows
 
-### Replace vault password and change vaulted passwords
+### Fetching the borg keys for local storage
 
-  - Generate a new key and save it as ./new-vault-pw: `pwgen -s 64 1 > new-vault-pw`
-  - `for i in $(ag ANSIBLE_VAULT -l); do ansible-vault rekey --new-vault-password-file new-vault-pw $i; done`
-  - Change the key in misc/vault-password.gpg
-  - `rm new-vault-pw`
+  - Make sure you have all the GPG keys **at least** locally signed
+  - Run the `playbooks/tasks/fetch-borg-keys.yml` playbook
+  - Make sure the playbook runs successfully and check the keys under the borg-keys directory
 
-### Re-encrypting the vault after adding or removing a new GPG key
+### Re-encrypting the vaults after adding a new PGP key
 
-  - Make sure you have all the GPG keys **at least** locally signed
-  - Run the `playbooks/tasks/reencrypt-vault-key.yml` playbook and make sure it does not have **any** failed task
-  - Test that the vault is working by running ansible-vault view on any encrypted vault file
-  - Commit and push your changes
+Follow the instructions in [group_vars/all/root_access.yml](group_vars/all/root_access.yml).
 
-### Fetching the borg keys for local storage
+### Changing the vault password on encrypted files
 
-  - Make sure you have all the GPG keys **at least** locally signed
-  - Run the playbooks/tasks/fetch-borg-keys.yml playbook
-  - Make sure the playbook runs successfully and check the keys under the borg-keys directory
+See [docs/vault-rekeying.md](docs/vault-rekeying.md).
 
 ## Backup documentation
 
@@ -166,7 +171,9 @@ See [docs/backups.md](./docs/backups.md) for detailed backup information.
 
 Our Gitlab installation uses [Omnibus](https://docs.gitlab.com/omnibus/) to run Gitlab on Docker. Updating Gitlab is as simple as running the ansible gitlab playbook:
 
-    ansible-playbook playbooks/gitlab.archlinux.org -t gitlab
+    ansible-playbook playbooks/gitlab.archlinux.org.yml --diff -t gitlab
+
+To view the current Gitlab version visit [this url](https://gitlab.archlinux.org/help/)
 
 ## One-shots
 

ansible.cfg

 [defaults]
-inventory = hosts,hcloud_inventory.py
+inventory = hosts
 library = library
 remote_tmp = $HOME/.ansible/tmp
 remote_user = root
 nocows = 1
 roles_path = roles
-vault_password_file = misc/get-vault-pass.sh
+vault_id_match = True
+vault_identity_list = default@misc/vault-keyring-client.sh,super@misc/vault-keyring-client.sh
 retry_files_enabled = False
 callback_plugins = plugins/callback
-callback_whitelist = profile_tasks
+callbacks_enabled = profile_tasks
+max_diff_size = 250000
+stdout_callback = debug
+interpreter_python = /usr/bin/python
 
 [ssh_connection]
 pipelining = True

docs/backups.md

@@ -8,21 +8,21 @@ You'll have to get the correct username from the vault.
 We use two different borg backup hosts: A primary one and an offsite one.
 The URL format for the primary one is
 
-    ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/<hostname>
+    ssh://u236610@u236610.your-storagebox.de:23/~/backup/<hostname>/repo
 
 while for the offsite one it's
 
-    ssh://<rsync_net_username>@prio.ch-s012.rsync.net:22/~/backup/<hostname>
+    ssh://zh1905@zh1905.rsync.net:22/~/backup/<hostname>
 
 In the examples below, we'll just abbreviate the full address as `<backup_address>`.
 If you want to use one of the examples below, you'll have to fill in the
 placeholder with your desired full address to the backup repository. For instance,
 
-    misc/borg.sh list <backup_address>::20191127-084357
+    misc/borg.sh list <backup_address>
 
 becomes
 
-    misc/borg.sh ssh://<hetzner_storagebox_username>@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org::20191127-084357
+    misc/borg.sh list ssh://u236610@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org/repo
 
 A convenience wrapper script is available at `misc/borg.sh` which makes sure you
 use the correct keyfile for the given server.
@@ -72,6 +72,20 @@ or just a sub-directory:
 
     misc/borg.sh extract <backup_address>::<archive_name> backup/srv/gitlab
 
+## Special backups
+
+### Mariadb
+
+For Mariadb backups are made using mariabackup to `backup_mysql_dir`. Backups
+are made and can be restored using the `mariadb-backup` tool. See also
+[official MariaDB docs](https://mariadb.com/kb/en/full-backup-and-restore-with-mariabackup/).
+
+### PostgreSQL
+
+For PostgreSQL backups are made using pg_dump to `backup_postgres_dir`.
+
+Restoring backups can be done with `pg_restore`. See also [official PostgreSQL docs](https://www.postgresql.org/docs/current/app-pgrestore.html).
+
 ## Adding a new server
 
 Adding a new server to be backed up goes as follows:

docs/banning.md

+# Banning IP Addresses for abuse
+
+For banning with an expiry `fail2ban` can be used, the expiry time depends on the configured fail2ban jail:
+
+```
+fail2ban-client set sshd banip 1.1.1.1
+```
+
+To permanently ban an IP address `firewall-cmd` can be used as shown below:
+
+```
+firewall-cmd --add-rich-rule="rule family='ipv4' source address='1.1.1.1' reject" --zone=public
+```
+
+```
+firewall-cmd --add-rich-rule="rule family='ipv6' source address='1:2:3:4:6::' reject" --zone=public
+```
+
+Note that on Gitlab, you must block the ip address for the docker zone:
+
+```
+firewall-cmd --add-rich-rule="rule family='ipv4' source address='1.1.1.1' reject" --zone=docker
+```
+
+To see the bans/rules:
+
+```
+firewall-cmd --list-all
+```
+
+To remove a banned IP Address:
+
+```
+firewall-cmd --remove-rich-rule='rule family="ipv6" source address="1:2:3:4:6::" reject' --zone=public
+```
+
+## Applying your changes
+
+After adding new rules you need to reload the firewall:
+```
+firewall-cmd --reload
+```

docs/becoming-devops.md

@@ -6,17 +6,21 @@ members.
 
 ## Junior DevOps program
 
-In order be able to onboard lesser-known members of the community who still want to help out with
-DevOps topics, we started the Junior DevOps program. This program requires applicants to
+In order to become a full DevOps, the applicant must first join the Junior DevOps program. This
+program requires applicants to
 
 0) have contributed to Arch multiple times in some meaningful ways,
 1) find two sponsors, and
 2) write an application to the arch-devops mailing list.
 
 The idea of Junior DevOps is that they don't get full access to all secrets and machines as opposed
-to full DevOps and have to make operational changes in pairing session with a full DevOps.
+to full DevOps but access within the limited scope on which they have been assigned rights to work
+on. As trust grows the scope on which the Junior DevOps operates may be extended, while their
+sponsors are expected to help them learn and should feel responsible to review any meaningful
+changes.
 
 However, Junior DevOps can already help with many tasks and are expected to take charge of a given
 topic.
 
-After a lot of trust is built up, Junior DevOps may graduate to become full DevOps.
+After a lot of trust is built up, Junior DevOps may graduate to become full DevOps. This usually
+takes 3-9 months.

docs/email.md

 # Configuration for users
 
 SMTP/IMAP server: mail.archlinux.org
-SMTP port: 465 (TLS), [deprecated: 587 STARTTLS]
+
+SMTP port: 465 (TLS)
+
 IMAP port: 993 (TLS)
 
 username: the system account name
+
 password: set by each user themselves with `passwd` on mail.archlinux.org
 
 # Adding new archlinux.org email addresses
@@ -16,6 +19,13 @@ If the user wants to forward email, either enter the destination directly in
 the /etc/postfix/users file or enter a username and then put the destination
 into `~username/.forward` so that they can edit it themselves.
 
+If the user is a new onboarded user the password has to be made empty, so the
+user can login and set a password:
+
+```
+passwd -d $username
+```
+
 # SMTP Architecture
 
 All hosts should be relaying outbound SMTP traffic via our primary MX server
@@ -31,19 +41,16 @@ to the server. This gives us several benefits:
 
 When a new host is provisioned:
 
-- The *postfix* role has a task delegated to 'mail.archlinux.org' to create a local user
+- The *postfix_null* role has a task delegated to 'mail.archlinux.org' to create a local user
   on 'mail.archlinux.org' that is used for the new server to authenticate against. The user
   name is the shortname of the new servers hostname (ie, "foobar.archlinux.org"
   will authenticate with the username "foobar")
 - You will need to run the *postfwd* role against mail.archlinux.org to update the
   rate-limiting it performs (servers are given higher rate-limits than normal
   users - see `/etc/postfwd/postfwd.cf` for exact limits). This *should*
-  happen automatically as the *postfwd* role is a dependency of the *postfix*
+  happen automatically as the *postfwd* role is a dependency of the *postfix_null*
   role (using `delegate_to` to run it against 'mail.archlinux.org' regardless of the target
   host that the postfix role is being run on)
-- Any services on the new host that need to relay mail should relay using SMTP
-  to `localhost` on port 10027 which bypasses any filtering/restrictions that
-  are applied by postfix to port 25 traffic.
 
 # Create new DKIM keys
 
@@ -55,6 +62,32 @@ rspamadm dkim_keygen -s dkim-rsa -b 4096 -d archlinux.org -t rsa -k archlinux.or
 the ouput gives you the DNS entries to add to the terraform files.
 The keys generated need to go to the vault:
 ```
-roles/rspamd/files/archlinux.org.dkim-rsa.key
-roles/rspamd/files/archlinux.org.dkim-ed25519.key
+roles/rspamd/files/archlinux.org.dkim-rsa.key.vault
+roles/rspamd/files/archlinux.org.dkim-ed25519.key.vault
+```
+
+# GitLab Service Desk
+
+GitLab has a [Service Desk
+feature](https://docs.gitlab.com/ee/user/project/service_desk.html) which
+creates issues for incoming emails and allows multiple people to reply via
+GitLab on those issues and assign issues. GitLab generates a default email
+address with the following logic:
+
+```
+gitlab+<group>-<project>-<project-id>-issue-@archlinux.org
+```
+
+As we prefer to use user friendly addresses such as `privacy@archlinux.org` for communication a postfix alias is configured in `/etc/postix/aliases`.
+
+For a new GitLab Service Desk project, add a new alias to `/etc/postfix/aliases` as:
+
+```
+foobar: gitlab+<group>-<project>-<project-id>-issue-@archlinux.org
+```
+
+Then run `postalias`:
+
+```
+postalias /etc/postfix/aliases
 ```

docs/fail2ban.md

@@ -33,3 +33,12 @@ Add `fail2ban_jails` dict with `postfix: true` to the host's `host_vars`.
 The dovecot jail is enabled for our mail server, blocking failed logins. Adding it to a host:
 
 Add `fail2ban_jails` dict with `dovecot: true` to the host's `host_vars`.
+
+### nginx_limit_req
+
+The nginx_limit_req jail is not enabled on any server. This jail bans IPs based repeated errors on nginx error log. Default blocking is 1 hour(s). Adding to a host:
+
+Add `fail2ban_jails` dict with `nginx_limit_req: true` to the host's `host_vars`.
+
+The `rsslimit` zone is whitelisted from being banned with `ignoreregex`, as we
+choose to not ban RSS abusers.

docs/geomirrors.md

+# Geo mirrors
+
+DevOps team maintain a geo mirror across the world. The Geo mirror is public facing on geo.mirror.pkgbuild.com domain and it will resolve the closest to the location of the requester mirror.
+
+## Locations
+
+| Mirror                                    | Location                    |
+| ----------------------------------------- | --------------------------- |
+| https://america.mirror.pkgbuild.com/      | Miami (United States)       |
+| https://asia.mirror.pkgbuild.com/         | Hong Kong                   |
+| https://berlin.mirror.pkgbuild.com/       | Berlin (Germany)            |
+| https://europe.mirror.pkgbuild.com/       | Prague (Czechia)            |
+| https://johannesburg.mirror.pkgbuild.com/ | Johannesburg (South Africa) |
+| https://london.mirror.pkgbuild.com/       | London (United Kingdom)     |
+| https://losangeles.mirror.pkgbuild.com/   | Los Angeles (United States) |
+| https://singapore.mirror.pkgbuild.com/    | Singapore                   |
+| https://sydney.mirror.pkgbuild.com/       | Sydney (Australia)          |
+| https://taipei.mirror.pkgbuild.com/       | Taipei (Taiwan)             |
+
+### Logical split
+The continent mirrors america, asia and europe contain the archive mirrors as well as repository mirrors. The city mirrors have just the repositories hosted.
+
+## Requirements
+- Host with Arch Linux installed
+- root access provided
+- Enough storage to host repos / debugrepos (at least)
+- Bandwidth (depends on location)
+
+## Adding a new mirror box
+- Add new entries in `hosts` file under `mirrors` and `geo_mirrors` sections
+- Adjust terraform `tf-stage1/archlinux.tf` to include the IPv4 and IPv6 entries of the new server
+- Adjust terraform `tf-stage1/templates.tf` to include the IPv4 and IPv6 entries of the new server as a `NS` record for `geo.mirror.pkgbuild.com`
+- Add a new files in `host_vars`
+    - `host_vars/<fqdn>/misc`
+        Containing all the information for the mirror itself
+    - `host_vars/<fqdn>/vault_wireguard.yml`
+        Containing the wireguard private key in encrypted vault
+
+## Ansible Playbooks execution
+
+| Playbook | Roles | Reason | Hosts (limits) |Comments |
+| ----------- | ----------- | ----------- | ----------- |  ----------- |
+| install_arch | All | Install Arch | | Optional if you can |
+| mirrors.yml | All | Setup mirror | `<fqdn>` | |
+| redirect.archlinux.org.yml | dyn_dns | Make TXT records | | |
+| repos.archlinux.org.yml | dbscripts | Allow debug repo syncing | | |
+| mirrors.yml | geo_dns | Add new domain to DNS | All other mirrors from geo.mirror | |
+| monitoring.archlinux.org.yml | wireguard,prometheus | Allow loki and prometheus to fetch data | | |
+| archlinux.org.yml | postgres,wireguard | Allow wireguard IP to connect for Mirror check | | Optional see Check Location below |
+
+### Add mirror in geo.mirror.pkgbuild.com
+
+Add mirror IP and FQDN in archweb admin https://archlinux.org/admin/mirrors/mirror/ under the `geo.mirror.pkgbuild.com` entry.
+
+### Check Location (optional)
+
+If you want the server to check for ping and stats create an entry in:
+ https://archlinux.org/admin/mirrors/checklocation/

docs/grafana.md

 # Grafana
 
-Our Grafana is hosted on https://monitoring.archlinux.org and is accessible for
-all Arch Linux Staff, editing rights are restricted to users with the Devops
-Role.
+Our Grafana is hosted on https://monitoring.archlinux.org and is accessible only to DevOps Staff.
+
+A public accessible instance is hosted on https://dashboards.archlinux.org with selected metrics using prometheus "remote write" feature.
 
 Dashboards and datasources are automatically provisioned by Grafana with Grafana's built-in [provisioning configuration](https://grafana.com/docs/grafana/latest/administration/provisioning/).
 
@@ -13,3 +13,12 @@ A new dashboard can be configured in our Grafana instance to try it out and if s
 * Export the dashboard to json (top left, share dashboard => exporter => save to file).
 * Save the json file in `roles/grafana/files/dashboards'
 * Git add the file and run the grafana playbook
+* If it needs to be available in `dashboards.archlinux.org` create a symlink in `roles/grafana/files/public-dashboards` to the dashboard in `roles/grafana/files/dashboards`
+
+## Adding new metrics to dashboards.archlinux.org
+
+Metrics can be added to the public grafana instance if they are already collected on `monitoring.archlinux.org`
+
+* Verify that the metrics are allowed to be made public and check with another DevOps member.
+* Edit `roles/prometheus/templates/prometheus.yml.j2` and extending the `regex` of the `remote_write` block.
+* Run `ansible-playbook playboks/monitoring.archlinux.org -t prometheus` to update the `remote_write` configuration.

docs/grow-disks.md

 # Growing (partitioned) Disks
 
-Our VPS are provisioned with 20G as CX11 by default. When one is resized the disk size usually changes.
+Our VPS are provisioned with 40G as CX22 by default. When one is resized the disk size usually changes.
 To use the additional space, one needs to grow the partition and the filesystem.
 
 ## Resizing partition

docs/ipmi.md

-# IPMI
-
-One of our servers has IPMI access, to use it install ipmitool and active an
-IPMI remote shell with:
-
-impitool -C3 -I lanplus -H $ip -U $username -P $password sol active

docs/maintenance.md

@@ -27,6 +27,7 @@ The basic configuration looks like this:
     service_name: "<service name>"
     service_domain: "{{ service_domain }}"
     service_alternate_domains: []
+    service_legacy_domains: []
     service_nginx_conf: "{{ service_nginx_conf }}"
   when: maintenance is defined
 ```
@@ -39,7 +40,7 @@ as a variable, to make sure the right file is used.
 - name: set up nginx
   template: src=nginx.d.conf.j2 dest="{{ service_nginx_conf }}" owner=root group=root mode=644
   notify:
-    - reload nginx
+    - Reload nginx
   when: maintenance is not defined
   tags: ['nginx']
 ```

docs/matrix.md

@@ -7,39 +7,51 @@ integrations with third-party services.
 
 ## Signing in
 
-For the initial sign-in you need to use a client that supports OpenID Single-Sign-On, such as [Element
-Web](https://app.element.io/). Enter `@username:archlinux.org` as the username and Element should
-offer to sign into our homeserver.
+For the initial sign-in you need to use a client that supports OpenID Single-Sign-On, such as
+[Element Web](https://app.element.io/). Enter `@username:archlinux.org` as the username and Element
+should offer to sign into our homeserver.
 
-You will be automatically invited to several rooms:
-  - `#archlinux:archlinux.org`: A public room for Arch Linux users.
-  - `#staff:archlinux.org`: Bridged with the `#archlinux-staff` IRC channel on Freenode.
-  - `#internal:archlinux.org`: A staff-only room with end-to-end encryption.
+You will be automatically invited to several spaces and rooms:
+  - `#public-space:archlinux.org`: A public space for Arch Linux users.
+    - `#archlinux:archlinux.org`: A public room for Arch Linux users.
+  - `#staff-space:archlinux.org`: A staff-only space for Arch Linux staff.
+    - `#internal:archlinux.org`: A staff-only room with end-to-end encryption.
 
-After signing in you can use Element's settings to set a password for the account if you want to use
-a client that does not support SSO.
+Password login is currently disabled, which might exclude some clients. It can be re-enabled should
+demand exist.
 
 If you need to provide your client with a homeserver address, use `https://matrix.archlinux.org`.
 
-## IRC bridges
+## Our rooms bridged to IRC
 
-### Our bridge
+We bridge several of our private IRC channels on Libera.Chat to Matrix.
 
-We bridge several of our private IRC channels on Freenode to Matrix, which you need to be invited
-into:
-  - `#developers:archlinux.org`: Bridged with `#archlinux-dev`.
-  - `#trusted-users:archlinux.org`: Bridged with `#archlinux-tu`.
+These rooms are open to all staff-space members:
+  - `#packaging:archlinux.org`: Bridged with `#archlinux-packaging`.
   - `#staff:archlinux.org`: Bridged with `#archlinux-staff`.
 
-### Matrix.org bridge
-
-Channels without keys are available via the "official" Freenode bridge at Matrix.org. For example:
-  - `#freenode_#archlinux-devops:matrix.org`: Bridged with `#archlinux-devops`.
-  - `#freenode_#archlinux-projects:matrix.org`: Bridged with `#archlinux-projects`.
-
-**Please avoid joining large bridged rooms (such as `#freenode_#archlinux:matrix.org`), as these
-slow down the server immensely.**
+The following rooms are not open to all staff, so you need to be invited:
+  - `#developers:archlinux.org`: Bridged with `#archlinux-dev`.
+  - `#trusted-users:archlinux.org`: Bridged with `#archlinux-tu`.
 
-Freenode may require you to have a registered nick to join certain channels. Once
-`@appservice-irc:matrix.org` contacts you, tell it to `!storepass <username>:<password>` with the
-username and the password of your Freenode account and it will reconnect you as registered.
+Please request an invitation in `#internal:archlinux.org` for the rooms you need to be in.
+
+These rooms are bridged to public channels, for which you should log into Libera.Chat via SASL:
+  - `#aurweb:archlinux.org`: Bridged with `#archlinux-aurweb`.
+  - `#bugs:archlinux.org`: Bridged with `#archlinux-bugs`.
+  - `#devops:archlinux.org`: Bridged with `#archlinux-devops`.
+  - `#pacman:archlinux.org`: Bridged with `#archlinux-pacman`.
+  - `#projects:archlinux.org`: Bridged with `#archlinux-projects`.
+  - `#reproducible:archlinux.org`: Bridged with `#archlinux-reproducible`.
+  - `#security:archlinux.org`: Bridged with `#archlinux-security`.
+  - `#testing:archlinux.org`: Bridged with `#archlinux-testing`.
+  - `#wiki:archlinux.org`: Bridged with `#archlinux-wiki`.
+
+If you fail to do so, your bridged IRC user cannot join the channels, meaning your messages won't be
+bridged. See [Libera.Chat's guide](https://libera.chat/guides/registration) on how to register a
+nickname. Afterwards, contact `@irc-bridge:archlinux.org` and send it the folllowing commands:
+  - `!username <username>`, with the primary nickname you registered with, then
+  - `!storepass <password>`, with your password for NickServ, and then
+  - `!reconnect` to reconnect and attempt the SASL login.
+
+If this worked, `@liberachat_SaslServ:archlinux.org` should contact you after the reconnect.

docs/monitoring.md

@@ -5,7 +5,6 @@ To access our monitoring system, go to https://monitoring.archlinux and log in v
 
 ## Adding a new host to monitoring
 
-* Add $host to node_exporters in `hosts`
 * Rollout exporter on host: `ansible-playbook playbooks/host.yml -t prometheus_exporters`
 * Rollout changes on monitoring host: `ansible-playbook playbooks/monitoring.archlinux.org.yml -t prometheus`
 
@@ -23,7 +22,7 @@ For general system performance monitoring [prometheus-node-exporter](https://git
 
 ### Borg
 
-For monitoring our borg backups prometheus-node-exporter's textfile collector feature is used, the textfile is written by a systemd service run periodically by a systemd timer called prometheus-borg-textcollector. Borg's last backup time is recorded for our Hetzner and rsync.net backups. Adding monitoring to a system is as simple as:
+For monitoring our borg backups prometheus-node-exporter's textfile collector feature is used, the textfile is written by a systemd service called prometheus-borg-textcollector. Borg's last backup time is recorded for our Hetzner and rsync.net backups. Adding monitoring to a system is as simple as:
 
 * Add the host to the `borg_clients` group
 * Rollout exporter on host: `ansible-playbook playbooks/host.yml -t prometheus_exporters`
@@ -65,3 +64,15 @@ For http(s)/icmp monitoring [prometheus-black-exporter](https://github.com/prome
 ### Archive monitoring
 
 The [Archive](https://archive.archlinux.org) and its mirrors defined in `archive_mirrors` are monitored using a textcollector which monitors the archive size in bytes.
+
+### Log monitoring
+
+The Nginx access logs/systemd logs are indexed by loki. For non webserver hosts the `promtail` job, for hosts with nginx an extra access_log line needs to be added to log json output which can be scraped by promtail.
+
+### AUR monitoring
+
+Some fun statistics are scraped from aur.archlinux.org using `curl` and `hq` as there is no proper AUR prometheus endpoint as of yet. The statistics are the AUR packages and users and is retrieved every 5 minutes.
+
+### Smart
+
+TODO:

docs/otp.md

@@ -14,7 +14,7 @@ Run
 
     pass otp insert -i GitHub -a archlinux-master-token github.com/archlinux-master-token -s
 
-When asked for a secret, provide the `github_master_seed` from `misc/vault_github.yml`.
+When asked for a secret, provide the `github_master_seed` from `misc/vaults/vault_github.yml`.
 You can then run
 
     pass otp code github.com/archlinux-master-token
@@ -30,13 +30,51 @@ Run
 
     pass otp insert -i Hetzner -a archlinux-master-token Hetzner/archlinux-master-token -s
 
-When asked for a secret, provide the `hetzner_master_seed` from `misc/vault_hetzner.yml`.
+When asked for a secret, provide the `hetzner_master_seed` from `misc/vaults/vault_hetzner.yml`.
 You can then run
 
     pass otp code Hetzner/archlinux-master-token
 
 to generate a token to log in.
 
+## UptimeRobot
+
+Run
+
+    pass otp insert -i UptimeRobot -a archlinux UptimeRobot/archlinux-master-token -s
+
+When asked for a secret, provide the `2FA token seed` from `misc/vaults/additional-credentials.vault`.
+You can then run
+
+    pass otp code UptimeRobot/archlinux-master-token
+
+to generate a token to log in.
+
+## Rsync.net
+
+Run
+
+    pass otp insert -i rsync.net -a archlinux Rsync.net/archlinux-master-token -s
+
+When asked for a secret, provide the `2FA token seed` from `host_vars/localhost/vault_rsync.net.yml`.
+You can then run
+
+    pass otp code Rsync.net/archlinux-master-token
+
+to generate a token to log in.
+
+## Vagrant Cloud
+
+Run
+
+    pass otp insert -i VagrantCloud -a archlinux VagrantCloud/archlinux-master-token -s
+
+When asked for a secret, provide the `vagrant_cloud_seed` from `misc/vaults/vault_vagrant_cloud.yml`.
+You can then run
+
+    pass otp code VagrantCloud/archlinux-master-token
+
+to generate a token to log in.
 
 ### Adding your own account