.ansible-lint

 exclude_paths:
   - misc
+  # FIXME: parser-error: couldn't resolve module/action 'hosts'. This often indicates a misspelling, missing collection, or incorrect module path.
+  - playbooks/tasks
+  - roles/prometheus/files/node.rules.yml
 skip_list:
-  - '204'
+  # yaml: line too long (x > 160 characters) (yaml[line-length])
+  - yaml[line-length]
+  # yaml: too many spaces inside braces (yaml[braces])
+  - yaml[braces]
+  # Do not recommend running tasks as handlers
+  - no-handler
+  # Do not force galaxy info in meta/main.yml
+  - meta-no-info
+  # Allow package versions to be specified as 'latest'
+  - package-latest
+  # Don't require fully-qualified collection names
+  - fqcn
+  # Allow free-form module calling syntax
+  - no-free-form
+  # Allow role includes with unprefixed role variables
+  - var-naming[no-role-prefix]

.gitlab-ci.yml

 image: "archlinux:latest"
 
-before_script:
-  - pacman -Syu --needed --noconfirm ansible-lint terraform terraform-provider-keycloak
-
 ansible-lint:
+  before_script:
+    - pacman -Syu --needed --noconfirm ansible-lint ansible python-jmespath
   script:
     # Fix weird ansible bug: https://github.com/trailofbits/algo/issues/1637
     # This probably happens due to gitlab-runner mounting the git repo into the container
     - chmod o-w .
-    - ansible-lint
+    # Fix syntax-check rule (https://github.com/ansible-community/ansible-lint/issues/1350#issuecomment-778764110)
+    - sed -i "/^vault_identity_list/d" ansible.cfg
+    - sed -i -e "/vars_files:/d" -e "/misc\/vaults\/vault_/d" playbooks/*.yml
+    # Fix load-failure: Failed to load or parse file
+    - ansible-lint $(printf -- "--exclude %s " */*/vault_*)
 
 terraform-validate:
+  before_script:
+    - pacman -Syu --needed --noconfirm terraform diffutils
   script:
     - cd tf-stage1
     - terraform init -backend=false
     - terraform validate
+    - terraform fmt --check
     - cd ../tf-stage2
     - terraform init -backend=false
     - terraform validate
+    - terraform fmt --check --diff

.gitlab/issue_templates/Decommission Server.md

+<!--
+This template should be used for decommissioning Arch Linux servers.
+-->
+
+# Decommissioning an Arch Linux server
+
+## Details
+
+- **Server name**: <!-- Put server hostname here -->
+
+## Steps
+
+- [ ] Notified arch-devops about decommissioning plan (at least a week prior)
+- [ ] (Optional) Notified arch-staff about decommissioning plan (at least two weeks prior)
+- [ ] Make sure data is backed up and migrated
+- [ ] Wait until decommissioning date
+- [ ] Boot server in rescue system
+- [ ] Securely wipe disks
+- [ ] Schedule server to be cancelled in provider interface
+- [ ] Report back to arch-devops
+- [ ] Report back to arch-staff

.gitlab/issue_templates/New Archive Mirror.md

+<!--
+This template should be used for adding a new Arch Linux Archive Mirror.
+-->
+
+# Add a new Archive Mirror Server
+
+## Details
+
+- **Server name**: <!-- Put server hostname here -->
+
+## Steps
+
+- [ ] Verify if the new mirror has enough diskspace for hosting the archive
+- [ ] Add a new domain in terraform
+- [ ] Add the new server to `archive_mirrors` in `hosts`
+- [ ] Run the dbscripts role to allow the new archive mirror to sync
+- [ ] Run the `archive-mirrors.yml` playbook
+- [ ] Run `certbot certonly --email webmaster@archlinux.org --agree-tos --rsa-key-size 4096 --renew-by-default --webroot -w /var/lib/letsencrypt/ -d <domain-name>`

.gitlab/issue_templates/New GitLab Pages Project.md

+<!--
+This template should be used by DevOps members when adding a GitLab Pages project to GitLab.
+
+In order to use GitLab Pages with Arch Linux, you'll have to specifically request a custom subdomain
+below `pkgbuild.com` or `archlinux.org` to be assigned. We don't allow random projects to use Pages
+because of legal and safety reasons (we don't want people to be able to trick others into thinking
+something hosted below one of our domains is official).
+-->
+
+# Procedure for adding a GitLab Pages project to GitLab
+
+## Details
+- **Project name**: hello
+- **Desired subdomain**: hello.archlinux.org
+
+## New Pages site checklist
+
+1. [ ] Have a pipeline that outputs some static HTML to `public/` during the build.
+1. [ ] Specify this path (`public/`) as an artifact path.
+1. [ ] GitLab should now recognize that you're trying to use Pages and will show some relevant
+       information at https://gitlab.archlinux.org/your-namespace/your-project/pages
+1. [ ] At this page, you'll also need to add your custom domain. Add the custom domain you requested earlier.
+       GitLab will then show domain verification information which you'll need in the next step.
+1. [ ] At this point, we'll need to add some stuff to the `archlinux_org_gitlab_pages` variable in `archlinux.tf`. It should look something like this.
+       Make sure to substitute the `your-domain` and `your-code-shown-by-gitlab` strings accordingly:
+
+        "your-domain" = "your-code-shown-by-gitlab"
+
+1. [ ] Run `terraform apply` and go back to GitLab. Hit `Verify` and it should pick up the new domain
+       verification code. It should then also automatically begin fetching a certificate via Let's
+       Encrypt. That should take roughly 10min.

.gitlab/issue_templates/New Mirror.md

+<!--
+This template should be used for adding a new Arch Linux Mirror on our own infrastructure.
+-->
+
+## Details
+
+- **Server name**: <!-- Put server hostname here -->
+
+## Steps
+
+1. [ ] Verify if the new mirror has enough diskspace for hosting a mirror.
+1. [ ] Add the new mirror in [archweb](https://archlinux.org/admin/mirrors/mirror/add) with these values:
+   - `Name`: <domain-name>
+   - `Tier`: `Tier 1`
+   - `Upstream`: `rsync.archlinux.org`
+   - `Public`: `checked`
+   - `Active`: `checked`
+   - `ISOs`: `checked`
+   - `Mirrors URLs`:
+      * `URL`: `https://<domain-name>/`
+      * `Country`: `Mirror country`
+      * `Bandwidth`: `Mirror bandwidth`
+   - `Mirrors Rsync IPs`: `Add the IPv4 and IPv6 address`
+1. [ ] Add a new domain in terraform
+1. [ ] Add the new server to `mirrors` in `hosts`
+1. [ ] Add a new `mirror_domain` key with <domain-name> as value
+1. [ ] Run the `gen_rsyncd` service on `rsync.archlinux.org` or wait till the systemd timer is triggered.
+1. [ ] Run the `mirrors.yml` playbook
+1. [ ] Run `certbot certonly --email webmaster@archlinux.org --agree-tos --rsa-key-size 4096 --renew-by-default --webroot -w /var/lib/letsencrypt/ -d <domain-name>`

.gitlab/issue_templates/New Official Project.md

+<!--
+This template should be used by DevOps members when adding a repository to GitLab.
+It can be used for migrations as well as new projects.
+-->
+
+# Procedure for adding an official project to GitLab
+
+## Details
+- **Project name**: my-example
+- **Type**: MIGRATION or NEW PROJECT <!-- delete one of these -->
+- **Current location**: git.archlinux.org/my-example.git <!-- delete this line if it's a new project and not a migration -->
+
+## New repo checklist
+
+If you want to add a new official project, here are some guidelines to follow:
+
+1. [ ] Evaluate whether the project can sit in the official [GitLab Arch Linux group](https://gitlab.archlinux.org/archlinux)
+       or whether it needs its own group. It only needs its own group if the primary
+       development group is somehow detached from Arch Linux and only losely related (for instance: [pacman](https://gitlab.archlinux.org/pacman))
+1. [ ] After project creation (use the GitLab import function if you migrate a repo), add the responsible people to the project
+       in the *Members* page (https://gitlab.archlinux.org/archlinux/my-example/-/project_members)
+       and give them the `Developer` role. The idea is to let these people mostly manage their own project while not giving them
+       enough permissions to be able to misconfigure the project.
+1. [ ] If mirroring to github.com is desired, work through the **GitHub.com mirroring checklist**
+       below and then return to this one.
+1. [ ] If the project needs a secure runner to build trusted artifacts, coordinate with
+       the rest of the DevOps team and if found to be reasonable, assign a secure runner
+       to a protected branch of the project.
+1. [ ] If a secure runner is used, create an MR to make sure the project's `.gitlab-ci.yml` specifies
+       `tags: secure`.
+1. [ ] Make sure that the *Push Rules* in https://gitlab.archlinux.org/archlinux/my-example/-/settings/repository
+       reflect these values:
+   - `Reject unverified users`: `on`
+   - `Reject unsigned commits`: `on`
+   - `Do not allow users to remove tags with git push`: `on`
+   - `Check whether author is a GitLab user`: `on`
+   - `Prevent pushing secret files`: `on`
+   - All of these should be activated by default as per group rules but it's good to check.
+1. [ ] The *Protected Branches* in https://gitlab.archlinux.org/archlinux/my-example/-/settings/repository should specify
+       `Allowed to merge` and `Allowed to push` as `Developers + Maintainers.`
+1. [ ] Disable unneeded project features under *Visibility, project features, permissions* (https://gitlab.archlinux.org/archlinux/my-example/edit)
+   Always:
+   - `Users can request access`: `off`
+   Often, but not always:
+   - Repository -> Container registry
+   - Repository -> Git Large File Storage (LFS)
+   - Repository -> Packages
+   - Analytics
+   - Requirements
+   - Security & Compliance
+   - Wiki
+   - Operations
+
+## GitHub.com mirroring checklist
+
+### GitLab side
+
+1. [ ] If you want to mirror your repository "my-example" from gitlab.archlinux.org to the github.com/archlinux organization,
+       you should create an empty project for your project at github.com/archlinux/my-example or
+       if that's an existing repository, make sure that the current histories of the source and
+       target repository are exactly the same.
+1. [ ] Go to https://gitlab.archlinux.org/archlinux/my-example/-/settings/repository and open
+       *Mirroring repositories*. Make sure it has these settings:
+   - `Git repository URL`: `ssh://git@github.com/archlinux/my-example.git`
+   - `Mirror direction`: `Push`
+   - `Authentication method`: `SSH public key`
+   - `Only mirror protected branches` : `off`
+1. [ ] Click `Mirror repository`.
+1. [ ] A new entry will pop up which has a button titled `Copy SSH public key`. Click that to copy the public key to your clipboard.
+
+### GitHub side
+
+1. [ ] Log in with your primary GitHub account.
+1. [ ] Go to https://github.com/archlinux/my-example/settings/access and assign the `Admin` role to the GitHub account
+       `archlinux-github`.
+1. [ ] Log in as the `archlinux-github` technical user. This is important as otherwise pushes won't be associated correctly.
+1. [ ] Go to https://github.com/archlinux/my-example/settings/keys and add a new deploy key.
+1. [ ] Name it "gitlab.archlinux.org" so we know where it's from.
+1. [ ] Paste the public key you copied from GitLab earlier.
+1. [ ] Check `Allow write access`.
+1. [ ] Click `Add key`.
+1. [ ] Verify the push mirror works by clicking the `Update now` button.
+1. [ ] In the repository settings on GitHub's side you should disable a few things to clean up the project page:
+   - `GitHub Actions`
+   - `Wiki`
+   - `Issues`
+   - `Projects`
+1. [ ] Go to https://github.com/archlinux/my-example/settings/hooks and add a new webhook
+   - `Payload URL`: `$(misc/get_key.py misc/vaults/vault_github.yml github_pull_closer_webhook_url)`
+   - `Content type`: `application/json`
+   - `Which events would you like to trigger this webhook?`
+     - `Let me select individual events.`: `Pull requests`
+1. [ ] In the GitHub description of the mirrored project, append " (read-only mirror)" so that people know it's a mirror.
+1. [ ] Disable `Packages` and `Environments` from being shown on the main page.
+1. [ ] In the website field put the full url to the repository on our GitLab.
+1. [ ] Go to https://github.com/archlinux/my-example/settings/access and remove the GitHub account `archlinux-github`
+1. [ ] Go to https://github.com/orgs/archlinux/teams/read-only-mirrors/repositories and add the repository with `write` permission

.gitlab/issue_templates/Offboarding.md

+<!--
+This template should be used for offboarding Arch Linux team members.
+-->
+
+# Offboarding an Arch Linux team member
+
+## Details
+
+- **Team member username**:
+- **Currently held roles**: <!-- Add known roles here like Package Maintainer, DevOps, etc -->
+- **Removal request**: <!-- Add link to relevant mailing list mail -->
+  - **Voting result**: <!-- Add link to relevant mailing list mail -->
+
+## All roles checklist
+
+- [ ] Remove user email by reverting instructions from [`docs/email.md`](docs/email.md).
+  - [ ] Remove entry in [`group_vars/all/archusers.yml`](group_vars/all/archusers.yml).
+  - [ ] Remove SSH pubkey from `pubkeys/<username>.pub`.
+  - [ ] Run `ansible-playbook -t archusers  $(git grep -l archusers playbooks/ | grep -v phrik)`.
+  - [ ] Setup forwarding if requested (please add the current date as a comment above the mail address in Postfix's `users` file).
+    - [ ] Inform the user of the conditions for forwarding.
+      - In most cases we only offer forwarding for 6 months.
+      - We will inform the user prior to disabling the forwarding.
+      - The forwarding can be extended if there are good reasons for doing so.
+- [ ] Set user to inactive in archweb: https://www.archlinux.org/admin/auth/user/
+- [ ] Remove member from [staff mailing list](https://lists.archlinux.org/mailman3/lists/staff.lists.archlinux.org/members/member/).
+- [ ] Moderate email address on [arch-dev-public](https://lists.archlinux.org/mailman3/lists/arch-dev-public.lists.archlinux.org/members/member/) (find member and moderate).
+- [ ] Ask the user to leave `#archlinux-staff` on Libera Chat and forget the password.
+- [ ] Remove staff cloak on Libera Chat ([Group contacts](https://wiki.archlinux.org/title/Arch_IRC_channels#Libera_Chat_group_contacts)). cc @archlinux/teams/irc/group-contacts
+- [ ] Remove the user from relevant staff groups on Keycloak.
+- [ ] Move the user from the public list of their usergroup on archweb ([support staff](https://archlinux.org/people/support-staff/) / [TUs](https://archlinux.org/people/trusted-users/) / [devs](https://archlinux.org/people/developers/)) to the respective fellow site ([fellow support staff](https://archlinux.org/people/support-staff-fellows/) / [fellow TUs](https://archlinux.org/people/trusted-user-fellows/) / [fellow devs](https://archlinux.org/people/developer-fellows/))
+- [ ] Remove the user from the Arch Linux GitHub organisation
+
+## Main key offboarding checklist
+
+- [ ] Remove user email for the `master-key.archlinux.org` subdomain by reverting instructions from [`docs/email.md`](docs/email.md).
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"Remove Main Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=Remove%20Main%20Key) template.
+
+## Package Maintainer/Developer offboarding checklist
+
+- [ ] Remove member from [arch-tu](https://lists.archlinux.org/mailman3/lists/arch-tu.lists.archlinux.org/members/member/) and/or [arch-dev](https://lists.archlinux.org/mailman3/lists/arch-dev.lists.archlinux.org/members/member/) mailing lists.
+- [ ] Ask the user to leave `#archlinux-tu` and/or `#archlinux-dev` aswell as `#archlinux-packaging` on Libera Chat and forget the password(s).
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"Remove Packager Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=Remove%20Packager%20Key) template.
+- [ ] Remove [stale package relations](https://archlinux.org/packages/stale_relations/) for the now inactive user.
+- [ ] Remove their extended permissions on AURweb
+
+## DevOps offboarding checklist
+
+- [ ] Remove entries in [`group_vars/all/root_access.yml`](group_vars/all/root_access.yml).
+- [ ] Run `ansible-playbook -t root_ssh playbooks/all-hosts-basic.yml`.
+- [ ] Run `ansible-playbook playbooks/hetzner_storagebox.yml playbooks/rsync.net.yml`.
+- [ ] Remove member from [arch-devops-private mailing lists](https://lists.archlinux.org/mailman3/lists/arch-devops-private.lists.archlinux.org/members/member/).
+- [ ] Remove pubkey from [Hetzner's key management](https://robot.your-server.de/key/index).
+
+## Wiki Administrator checklist
+
+- [ ] Remove member from [arch-wiki-admins mailing list](https://lists.archlinux.org/mailman3/lists/arch-wiki-admins.lists.archlinux.org/members/member/).

.gitlab/issue_templates/Onboarding.md

+<!--
+This template should be used for onboarding new Arch Linux team members.
+It can also be used as a reference for adding new roles to an existing team member.
+-->
+/confidential
+<!--
+NOTE: Do not remove the above short actions.
+They ensure that the ticket is created confidential and that personal
+information is not publicly visible.
+-->
+
+# Onboarding an Arch Linux team member
+
+## Details
+
+- **Team member username**: <!-- Used for SSO account and @archlinux.org e-mail address -->
+- **Application**: <!-- Add link to relevant mailing list mail -->
+- **Voting result**: <!-- Add link to relevant mailing list mail -->
+- **SSH public key**: <!-- Add this when a user's access to machines is added or updated -->
+- **Full Name**: <!-- Relevant for all new users -->
+- **Personal e-mail address**: <!-- Relevant for users who will get a new archweb and/or SSO account -->
+- **PGP key ID used with personal e-mail address**: <!-- Relevant for users who will get a new archweb account -->
+- **Communication e-mail address**: [arch, personal] <!-- Relevant for users who will be signed up to mailing lists. Either choose "arch" or "personal". -->
+
+<!--
+NOTE: When creating this ticket as the sponsor for a new trusted user or
+support staff member, attach the above information as a clearsigned document to
+this ticket.
+https://www.gnupg.org/gph/en/manual/x135.html
+-->
+
+## All roles checklist
+
+- [ ] Add user mail if TU or developer, or support staff and **communication e-mail address** is arch.
+  - [ ] Add new user email as per [`docs/email.md`](docs/email.md).
+  - [ ] Add entry in [`group_vars/all/archusers.yml`](group_vars/all/archusers.yml).
+    - If support staff `hosts` should be set to `mail.archlinux.org`.
+    - `homedir.archlinux.org` is also allowed for support staff, but it is opt-in.
+  - [ ] Add SSH pubkey to `pubkeys/<username>.pub`.
+  - [ ] Run `ansible-playbook -t archusers $(git grep -l archusers playbooks/ | grep -v phrik)`.
+- [ ] Create a new user in [archweb](https://www.archlinux.org/devel/newuser/). Select the appropriate group membership and allowed repos (if applicable).
+- [ ] Subscribe **communication e-mail address** to internal [staff mailing list](https://lists.archlinux.org/mailman3/lists/staff.lists.archlinux.org/mass_subscribe/).
+- [ ] Allow sending from **communication e-mail address** on [arch-dev-public](https://lists.archlinux.org/mailman3/lists/arch-dev-public.lists.archlinux.org/members/member/) (subscribe and/or find address and remove moderation).
+- [ ] Give the user access to `#archlinux-staff` on Libera Chat.
+- [ ] Give the user a link to our [staff services page](https://wiki.archlinux.org/title/DeveloperWiki:Staff_Services).
+- [ ] Replace the **Team member username** with the @-prefixed username on Gitlab.
+- [ ] Remove personal information (such as **Full Name** and **Personal e-mail
+  address**, as well as the clearsigned representation of this data), remove
+  the description history and make the issue non-confidential.
+- [ ] Request staff cloak on Libera Chat ([Group contacts](https://wiki.archlinux.org/title/Arch_IRC_channels#Libera_Chat_group_contacts)) cc @archlinux/teams/irc/group-contacts
+
+## Main key onboarding checklist
+
+- [ ] Add new user email for the `master-key.archlinux.org` subdomain as per [`docs/email.md`](docs/email.md).
+  <!-- The ticket should be created by the developer becoming a new main key holder -->
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"New Main Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=New%20Main%20Key) template.
+
+## Package Maintainer/Developer onboarding checklist
+
+<!-- The ticket should be created by a sponsor of the new packager -->
+- [ ] Create an issue in [archlinux-keyring](https://gitlab.archlinux.org/archlinux/archlinux-keyring) using the [*"New Packager Key"*](https://gitlab.archlinux.org/archlinux/archlinux-keyring/-/issues/new?issuable_template=New%20Packager%20Key) template.
+- [ ] Assign the user to the correct group in the `Arch Linux Staff/Package Maintainer Team/` group on Keycloak.
+- [ ] Assign the user to the `Package Maintainers` or `Developers` group on [archlinux.org](https://archlinux.org/admin/auth/user/).
+- [ ] Subscribe **communication e-mail address** to internal [arch-tu](https://lists.archlinux.org/mailman3/lists/arch-tu.lists.archlinux.org/mass_subscribe/) or [arch-dev](https://lists.archlinux.org/mailman3/lists/arch-dev.lists.archlinux.org/mass_subscribe/) mailing list.
+- [ ] Give the user access to `#archlinux-tu` or `#archlinux-dev` aswell as `#archlinux-packaging` on Libera Chat.
+
+## Support staff checklist
+
+- [ ] Assign the user to the proper support staff group on Keycloak.
+
+## DevOps onboarding checklist
+
+- [ ] Add entries in [`group_vars/all/root_access.yml`](group_vars/all/root_access.yml).
+- [ ] Run `ansible-playbook -t root_ssh playbooks/all-hosts-basic.yml`.
+- [ ] Run `ansible-playbook playbooks/hetzner_storagebox.yml playbooks/rsync.net.yml`.
+- [ ] Subscribe **communication e-mail address** to internal [arch-devops-private](https://lists.archlinux.org/mailman3/lists/arch-devops-private.lists.archlinux.org/mass_subscribe/) mailing list.
+- [ ] Add pubkey to [Hetzner's key management](https://robot.your-server.de/key/index) for Dedicated server rescue system.
+
+## Wiki Administrator checklist
+
+- [ ] Subscribe **communication e-mail address** to the [arch-wiki-admins](https://lists.archlinux.org/mailman3/lists/arch-wiki-admins.lists.archlinux.org/mass_subscribe/) mailing list.

.gitmodules

-[submodule "roles/common/files/contrib"]
-	path = roles/common/files/contrib
-	url = https://github.com/archlinux/contrib

LICENSE

+The MIT License (MIT)
+
+Copyright 2021 Arch Linux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -2,23 +2,25 @@
 
 This repository contains the complete collection of ansible playbooks and roles for the Arch Linux infrastructure.
 
-It also contains git submodules so you have to run `git submodule update --init
---recursive` after cloning or some tasks will fail to run.
+## Table of contents
+[[_TOC_]]
 
 ## Requirements
 
 Install these packages:
   - terraform
-  - terraform-provider-keycloak
+  - python-click
+  - python-jmespath
+  - moreutils (for playbooks/tasks/reencrypt-vault-key.yml)
 
 ### Instructions
 
 All systems are set up the same way. For the first time setup in the Hetzner rescue system,
-run the provisioning script: `ansible-playbook playbooks/tasks/install-arch.yml -l $host`.
+run the provisioning script: `ansible-playbook playbooks/tasks/install_arch.yml -l $host`.
 The provisioning script configures a sane basic systemd with sshd. By design, it is NOT idempotent.
 After the provisioning script has run, it is safe to reboot.
 
-Once in the new system, run the regular playbook: `HCLOUD_TOKEN=$(misc/get_key.py misc/vault_hetzner.yml hetzner_cloud_api_key) ansible-playbook playbooks/$hostname.yml`.
+Once in the new system, run the regular playbook: `HCLOUD_TOKEN=$(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_api_key) ansible-playbook playbooks/$hostname.yml`.
 This playbook is the one regularity used for administrating the server and is entirely idempotent.
 
 When adding a new machine you should also deploy our SSH known_hosts file and update the SSH hostkeys file in this git repo.
@@ -27,55 +29,49 @@ It will also deploy any new SSH host keys to all our machines.
 
 #### Note about GPG keys
 
-The root_access.yml file contains the root_gpgkeys variable that determine the users that have access to the vault, as well as the borg backup keys.
-All the keys should be on the local user gpg keyring and at **minimum** be locally signed with --lsign-key. This is necessary for running either the reencrypt-vault-key
-or the fetch-borg-keys tasks.
+The `root_access.yml` file contains the `vault_default_pgpkeys` variable which
+determines the users that have access to the `default` vault, as well as the
+borg backup keys. A separate `super` vault exists for storing highly sensitive
+secrets like Hetzner credentials; access to the `super` vault is controlled by
+the `vault_super_pgpkeys` variable.
 
-#### Note about Ansible dynamic inventories
-
-We use a dynamic inventory script in order to automatically get information for
-all servers directly from hcloud. You don't really have to do anything to make
-this work but you should keep in mind to NOT add hcloud servers to `hosts`!
-They'll be available automatically.
-
-#### Note about first time certificates
-
-The first time a certificate is issued, you'll have to do this manually by yourself. First, configure the DNS to
-point to the new server and then run a playbook onto the server which includes the nginx role. Then on the server,
-it is necessary to run the following once:
-
-    certbot certonly --email webmaster@archlinux.org --agree-tos --rsa-key-size 4096 --renew-by-default --webroot -w /var/lib/letsencrypt/ -d <domain-name>
-
-Note that some roles already run this automatically.
+All the keys should be on the local user gpg keyring and at **minimum** be
+locally signed with `--lsign-key` (or if you use TOFU, have `--tofu-policy
+good`). This is necessary for running any of the `reencrypt-vault-default-key`,
+`reencrypt-vault-super-key `or `fetch-borg-keys` tasks.
 
 #### Note about packer
 
 We use packer to build snapshots on hcloud to use as server base images.
 In order to use this, you need to install packer and then run
 
-    packer build -var $(misc/get_key.py misc/vault_hetzner.yml hetzner_cloud_api_key env) packer/archlinux.json
+    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_api_key --format env) packer/archlinux.pkr.hcl
 
 This will take some time after which a new snapshot will have been created on the primary hcloud archlinux project.
 
+For the sandbox project please run
+
+    packer build -var $(misc/get_key.py misc/vaults/vault_hetzner.yml hetzner_cloud_sandbox_infrastructure_api_key --format env | sed 's/_sandbox_infrastructure//') -var install_ec2_public_keys_service=true packer/archlinux.pkr.hcl
+
 #### Note about terraform
 
 We use terraform in two ways:
 
-    1) To provision a part of the infrastructure on hcloud (and possibly other service providers in the future)
-    2) To declaratively configure applications
+1. To provision a part of the infrastructure on hcloud (and possibly other service providers in the future)
+2. To declaratively configure applications
 
 For both of these, we have set up a separate terraform script. The reason for that is that sadly terraform can't have
 providers depend on other providers so we can't declaratively state that we want to configure software on a server which
-itself needs to be provisioned first. Therefore, we use a two-stage process. Generally speaking, scenario 1) is configured in
-`tf-stage1` and 2) is in `tf-stage2`. Maybe in the future, we can just have a single terraform script for everything
+itself needs to be provisioned first. Therefore, we use a two-stage process. Generally speaking, scenario 1. is configured in
+`tf-stage1` and 2. is in `tf-stage2`. Maybe in the future, we can just have a single terraform script for everything
 but for the time being, this is what we're stuck with.
 
 The very first time you run terraform on your system, you'll have to init it:
 
     cd tf-stage1  # and also tf-stage2
-    terraform init -backend-config="conn_str=postgres://terraform:$(../misc/get_key.py group_vars/all/vault_terraform.yml vault_terraform_db_password)@state.archlinux.org"
+    terraform init -backend-config="conn_str=postgres://terraform:$(../misc/get_key.py ../group_vars/all/vault_terraform.yml vault_terraform_db_password)@state.archlinux.org?sslmode=verify-full"
 
-After making changes to the infrastructure in `tf-stage1/archlinux.fg`, run
+After making changes to the infrastructure in `tf-stage1/archlinux.tf`, run
 
     terraform plan
 
@@ -97,13 +93,7 @@ set up.
 
 #### SMTP Configuration
 
-All hosts should be relaying email through our primary mx host (currently 'orion'). See docs/email.txt for full details.
-
-#### Note about opendkim
-
-The opendkim DNS data has to be added to DNS manually. The roles verifies that the DNS is correct before starting opendkim.
-
-The file that has to be added to the zone is `/etc/opendkim/private/$selector.txt`.
+All hosts should be relaying email through our primary mx host (currently 'mail.archlinux.org'). See [docs/email.md](./docs/email.md) for full details.
 
 ### Putting a service in maintenance mode
 
@@ -119,13 +109,13 @@ As long as you pass the maintenance variable to the playbook run, the web servic
 passing it on the command line and run the playbook again, the regular nginx configuration should resume and the service should accept
 requests by the end of the run.
 
-Passing maintenance=false, will also prevent the regular nginx configuration from resuming, but will not put the service into maintence
+Passing maintenance=false, will also prevent the regular nginx configuration from resuming, but will not put the service into maintenance
 mode.
 
 Keep in mind that passing the maintenance variable to the whole playbook, without any tag, will make all the web services that have the
 maintenance mode in them, to be put in maintenance mode. Use tags to affect only the services you want.
 
-Documentation on how to add the maintenance mode to a web service is inside docs/maintenance.txt
+Documentation on how to add the maintenance mode to a web service is inside [docs/maintenance.md](./docs/maintenance.md).
 
 ### Finding servers requiring security updates
 
@@ -133,172 +123,57 @@ Arch-audit can be used to find servers in need of updates for security issues.
 
     ansible all -a "arch-audit -u"
 
-#### Updating servers
-
-The following steps should be used to update our managed servers:
-
-  * pacman -Syu
-  * manually update the kernel, since it is in IgnorePkg by default
-  * sync
-  * checkservices
-  * reboot
-
-## Servers
-
-### vostok
-
-#### Services
-  - backups
-
-### orion
-
-#### Services
-  - repos/sync (repos.archlinux.org)
-  - sources (sources.archlinux.org)
-  - archive (archive.archlinux.org)
-  - torrent tracker hefurd (tracker.archlinux.org)
-
-### luna
-
-#### Services
-
-  - aur (aur.archlinux.org)
-  - mailman
-  - projects (projects.archlinux.org)
-
-### apollo
-
-#### Services
-  - wiki (wiki.archlinux.org)
-  - bugs (bugs.archlinux.org)
-  - archweb
-  - patchwork
-
-## bugs.archlinux.org
-
-#### Services
-  - flyspray
-
-## bbs.archlinux.org
-
-#### Services
-  - bbs
-
-### phrik.archlinux.org
+### Semi-automated server upgrades
 
-#### Services
-   - phrik (irc bot) users in the phrik group defined in
-     the hosts vars and re-used the archusers role. Users
-     in the phrik group are allowed to restar the irc bot.
+For updating all servers in a mostly unattented manner, the following playbook
+can be used:
 
-### dragon
+    ansible-playbook playbooks/tasks/upgrade-servers.yml [-l SUBSET]
 
-#### Services
-  - build server
-  - sogrep
-  - arch-boxes (packer)
+It runs `pacman -Syu` on the targeted hosts in batches and then reboots them.
+If any server fails to reboot successfully, the rolling update stops and
+further batches are cancelled. To display the packages updated on each host,
+you can pass the `--diff` option to ansible-playbook.
 
-### state.archlinux.org
+Using this update method, `.pacnew` files are left unmerged which is OK for
+most configuration files that are managed by Ansible. However, care must be
+taken with updates that require manual intervention (e.g. major PostgreSQL
+releases).
 
-#### Services
-  - postgres server for terraform state
-
-### quassel.archlinux.org
-
-#### Services
-  - quassel core
-
-### homedir.archlinux.org
-
-#### Services
-  - ~/user/ webhost
-
-### accounts.archlinux.org
-
-This server is /special/. It runs keycloak and is central to our unified Arch Linux account management world.
-It has an Ansible playbook for the keycloak service but that only installs the package and starts it but it's configured via a secondary Terraform file only for keycloak `keycloak.tf`.
-The reason for doing it this way is that Terraform support for Keycloak is much superior and it's declarative too which is great for making sure that no old config remains in the case of config changes.
-
-So to set up this server from scratch, run:
-
-  - `cd tf-stage1`
-  - `terraform apply`
-  - `cd ../tf-stage2`
-  - `terraform import keycloak_realm.master master`
-  - `terraform apply`
-
-#### Services
-  - keycloak
-
-### mirror.pkgbuild.com
-
-#### Services
-  - Load balancer for PIA mirrors across the world. Uses Maxmind's GeoIP City
-    database for routing users to their nearest mirror. Account information is
-    stored in the ansible vault.
-
-### reproducible.archlinux.org
-
-#### Services
-  - Runs a master rebuilderd instance with two PIA workers (repro1.pkgbuild.com,
-    repro2.pkgbuild.com and repro3.pkgbuild.com).
-    repro3.pkgbuild.com is packet.net machine which runs Ubuntu.
-
-### runner1.archlinux.org
-
-Slow-ish PIA box with spinning disks.
-
-#### Services
-  - GitLab runner
-
-### runner2.archlinux.org
-
-Medium-fast-ish packet.net box with Debian on it. Is currently maintained manually.
+## Servers
 
-#### Services
-  - GitLab runner
+This section has been moved to [docs/servers.md](docs/servers.md).
 
 ## Ansible repo workflows
 
-### Replace vault password and change vaulted passwords
-
-  - Generate a new key and save it as ./new-vault-pw: `pwgen -s 64 1 > new-vault-pw`
-  - `for i in $(ag ANSIBLE_VAULT -l); do ansible-vault rekey --new-vault-password-file new-vault-pw $i; done`
-  - Change the key in misc/vault-password.gpg
-  - `rm new-vault-pw`
-
-### Re-encrypting the vault after adding or removing a new GPG key
-
-  - Make sure you have all the GPG keys **at least** locally signed
-  - Run the playbooks/tasks/reencrypt-vault-key.yml playbook and make sure it does not have **any** failed task
-  - Test that the vault is working by running ansible-vault view on any encrypted vault file
-  - Commit and push your changes
-
 ### Fetching the borg keys for local storage
 
   - Make sure you have all the GPG keys **at least** locally signed
-  - Run the playbooks/tasks/fetch-borg-keys.yml playbook
+  - Run the `playbooks/tasks/fetch-borg-keys.yml` playbook
   - Make sure the playbook runs successfully and check the keys under the borg-keys directory
 
-## Backup documentation
+### Re-encrypting the vaults after adding a new PGP key
+
+Follow the instructions in [group_vars/all/root_access.yml](group_vars/all/root_access.yml).
 
-Backups should be checked now and then. Some common tasks:
+### Changing the vault password on encrypted files
 
-### Listing current backups per server
+See [docs/vault-rekeying.md](docs/vault-rekeying.md).
 
-    borg list borg@vostok.archlinux.org:/backup/<hostname>
+## Backup documentation
 
-Example
+We use BorgBackup for all of our backup needs. We have a primary backup storage as well as an
+additional offsite backup.
 
-    borg list borg@vostok.archlinux.org:/backup/homedir.archlinux.org
+See [docs/backups.md](./docs/backups.md) for detailed backup information.
 
-### Listing files in a backup
+## Updating Gitlab
 
-    borg list borg@vostok.archlinux.org:/backup/<hostname>::<archive name>
+Our Gitlab installation uses [Omnibus](https://docs.gitlab.com/omnibus/) to run Gitlab on Docker. Updating Gitlab is as simple as running the ansible gitlab playbook:
 
-Example
+    ansible-playbook playbooks/gitlab.archlinux.org.yml --diff -t gitlab
 
-    borg list borg@vostok.archlinux.org:/backup/homedir.archlinux.org::20191127-084357
+To view the current Gitlab version visit [this url](https://gitlab.archlinux.org/help/)
 
 ## One-shots
 

ansible.cfg

 [defaults]
-inventory = hosts,hcloud_inventory.py
+inventory = hosts
 library = library
 remote_tmp = $HOME/.ansible/tmp
 remote_user = root
 nocows = 1
 roles_path = roles
-vault_password_file = misc/get-vault-pass.sh
+vault_id_match = True
+vault_identity_list = default@misc/vault-keyring-client.sh,super@misc/vault-keyring-client.sh
 retry_files_enabled = False
 callback_plugins = plugins/callback
-callback_whitelist = profile_tasks
+callbacks_enabled = profile_tasks
+max_diff_size = 250000
+stdout_callback = debug
+interpreter_python = /usr/bin/python
 
 [ssh_connection]
 pipelining = True

docs/backups.md

+# Backups
+
+Backups should be checked now and then. Some common tasks are listed below.
+You'll have to get the correct username from the vault.
+
+## Accessing backup hosts
+
+We use two different borg backup hosts: A primary one and an offsite one.
+The URL format for the primary one is
+
+    ssh://u236610@u236610.your-storagebox.de:23/~/backup/<hostname>/repo
+
+while for the offsite one it's
+
+    ssh://zh1905@zh1905.rsync.net:22/~/backup/<hostname>
+
+In the examples below, we'll just abbreviate the full address as `<backup_address>`.
+If you want to use one of the examples below, you'll have to fill in the
+placeholder with your desired full address to the backup repository. For instance,
+
+    misc/borg.sh list <backup_address>
+
+becomes
+
+    misc/borg.sh list ssh://u236610@u236610.your-storagebox.de:23/~/backup/homedir.archlinux.org/repo
+
+A convenience wrapper script is available at `misc/borg.sh` which makes sure you
+use the correct keyfile for the given server.
+
+## Listing backups in repository
+
+This allows you to check which backups are currently available for the given server:
+
+    misc/borg.sh list <backup_address>
+
+## Listing files in a specific backup
+
+Once you figured out which backup you want to use, you can list the files inside via:
+
+    misc/borg.sh list <backup_address>::<archive_name>
+
+## Getting info for a repository
+
+Check how large all backups for a server are:
+
+    misc/borg.sh info <backup_address>
+
+## Getting info for a specific backup
+
+Check how large a single backup is and how long it took to perform:
+
+    misc/borg.sh info <backup_address>::<archive_name>
+
+## Mounting a backup
+
+One convenient way to access the files inside an archive is to mount it:
+
+    mkdir mnt
+    misc/borg.sh mount <backup_address>::<archive_name> mnt
+
+You might want to mount it with `-o ignore_permissions` depending on which user
+you're using to access the backup.
+
+## Extracing files from a backup
+
+Alternatively, if you don't want to mount it and instead want to extract files directly, you can
+do so. Either extract the whole backup:
+
+    misc/borg.sh extract <backup_address>::<archive_name>
+
+or just a sub-directory:
+
+    misc/borg.sh extract <backup_address>::<archive_name> backup/srv/gitlab
+
+## Special backups
+
+### Mariadb
+
+For Mariadb backups are made using mariabackup to `backup_mysql_dir`. Backups
+are made and can be restored using the `mariadb-backup` tool. See also
+[official MariaDB docs](https://mariadb.com/kb/en/full-backup-and-restore-with-mariabackup/).
+
+### PostgreSQL
+
+For PostgreSQL backups are made using pg_dump to `backup_postgres_dir`.
+
+Restoring backups can be done with `pg_restore`. See also [official PostgreSQL docs](https://www.postgresql.org/docs/current/app-pgrestore.html).
+
+## Adding a new server
+
+Adding a new server to be backed up goes as follows:
+
+* Make sure the new servers host key is synced to `docs/ssh-known_hosts.txt` if not run:
+
+      ansible-playbook playbooks/tasks/sync-ssh-hostkeys.yml
+
+* Add the server to [borg-clients] in hosts
+
+* Run the borg role on u236610.your-storagebox.de to allow the new machine to create backups
+
+      ansible-playbook playbooks/hetzner_storagebox.yml
+
+* Run the borg role for rsync.net to allow the new machine to create backups
+
+      ansible-playbook playbooks/rsync.net.yml
+
+* Run the borg role on the new machine to initialize the repository
+
+      ansible-playbook playbooks/$machine.yml -t borg

docs/banning.md

+# Banning IP Addresses for abuse
+
+For banning with an expiry `fail2ban` can be used, the expiry time depends on the configured fail2ban jail:
+
+```
+fail2ban-client set sshd banip 1.1.1.1
+```
+
+To permanently ban an IP address `firewall-cmd` can be used as shown below:
+
+```
+firewall-cmd --add-rich-rule="rule family='ipv4' source address='1.1.1.1' reject" --zone=public
+```
+
+```
+firewall-cmd --add-rich-rule="rule family='ipv6' source address='1:2:3:4:6::' reject" --zone=public
+```
+
+Note that on Gitlab, you must block the ip address for the docker zone:
+
+```
+firewall-cmd --add-rich-rule="rule family='ipv4' source address='1.1.1.1' reject" --zone=docker
+```
+
+To see the bans/rules:
+
+```
+firewall-cmd --list-all
+```
+
+To remove a banned IP Address:
+
+```
+firewall-cmd --remove-rich-rule='rule family="ipv6" source address="1:2:3:4:6::" reject' --zone=public
+```
+
+## Applying your changes
+
+After adding new rules you need to reload the firewall:
+```
+firewall-cmd --reload
+```

docs/becoming-devops.md

+# Becoming Arch Linux DevOps
+
+In Arch Linux, DevOps are expected to be reliable, trusthworthy, and self-directed.
+DevOps should be known and trusted by the community beforehand or be recommended by previous
+members.
+
+## Junior DevOps program
+
+In order to become a full DevOps, the applicant must first join the Junior DevOps program. This
+program requires applicants to
+
+0) have contributed to Arch multiple times in some meaningful ways,
+1) find two sponsors, and
+2) write an application to the arch-devops mailing list.
+
+The idea of Junior DevOps is that they don't get full access to all secrets and machines as opposed
+to full DevOps but access within the limited scope on which they have been assigned rights to work
+on. As trust grows the scope on which the Junior DevOps operates may be extended, while their
+sponsors are expected to help them learn and should feel responsible to review any meaningful
+changes.
+
+However, Junior DevOps can already help with many tasks and are expected to take charge of a given
+topic.
+
+After a lot of trust is built up, Junior DevOps may graduate to become full DevOps. This usually
+takes 3-9 months.

docs/email.md

 # Configuration for users
 
 SMTP/IMAP server: mail.archlinux.org
-SMTP port: 587 STARTTLS
-IMAP port: 143 (STARTTLS), 993 (TLS)
+
+SMTP port: 465 (TLS)
+
+IMAP port: 993 (TLS)
 
 username: the system account name
-password: set by each user themselves with `passwd` on orion
+
+password: set by each user themselves with `passwd` on mail.archlinux.org
 
 # Adding new archlinux.org email addresses
 
-Login to orion and edit `/etc/postfix/users`, add the new email address in the
+Login to mail.archlinux.org and edit `/etc/postfix/users`, add the new email address in the
 appropriate category and run `postmap /etc/postfix/users`.
 
 If the user wants to forward email, either enter the destination directly in
 the /etc/postfix/users file or enter a username and then put the destination
 into `~username/.forward` so that they can edit it themselves.
 
+If the user is a new onboarded user the password has to be made empty, so the
+user can login and set a password:
+
+```
+passwd -d $username
+```
+
 # SMTP Architecture
 
 All hosts should be relaying outbound SMTP traffic via our primary MX server
-(currently 'orion'). Each hosts authenticates using SASL over a TLS connection
+(currently 'mail.archlinux.org'). Each hosts authenticates using SASL over a TLS connection
 to the server. This gives us several benefits:
 
 1. DKIM signing can be done centrally.
@@ -31,16 +41,53 @@ to the server. This gives us several benefits:
 
 When a new host is provisioned:
 
-- The *postfix* role has a task delegated to 'orion' to create a local user
-  on 'orion' that is used for the new server to authenticate against. The user
+- The *postfix_null* role has a task delegated to 'mail.archlinux.org' to create a local user
+  on 'mail.archlinux.org' that is used for the new server to authenticate against. The user
   name is the shortname of the new servers hostname (ie, "foobar.archlinux.org"
   will authenticate with the username "foobar")
-- You will need to run the *postfwd* role against orion to update the
+- You will need to run the *postfwd* role against mail.archlinux.org to update the
   rate-limiting it performs (servers are given higher rate-limits than normal
   users - see `/etc/postfwd/postfwd.cf` for exact limits). This *should*
-  happen automatically as the *postfwd* role is a dependency of the *postfix*
-  role (using `delegate_to` to run it against 'orion' regardless of the target
+  happen automatically as the *postfwd* role is a dependency of the *postfix_null*
+  role (using `delegate_to` to run it against 'mail.archlinux.org' regardless of the target
   host that the postfix role is being run on)
-- Any services on the new host that need to relay mail should relay using SMTP
-  to `localhost` on port 10027 which bypasses any filtering/restrictions that
-  are applied by postfix to port 25 traffic.
+
+# Create new DKIM keys
+
+The rspamd role expects the key to exist in the vault. To generate new keys, run
+```
+rspamadm dkim_keygen -s dkim-ed25519 -b 0 -d archlinux.org -t ed25519 -k archlinux.org.dkim-ed25519.key
+rspamadm dkim_keygen -s dkim-rsa -b 4096 -d archlinux.org -t rsa -k archlinux.org.dkim-rsa.key
+```
+the ouput gives you the DNS entries to add to the terraform files.
+The keys generated need to go to the vault:
+```
+roles/rspamd/files/archlinux.org.dkim-rsa.key.vault
+roles/rspamd/files/archlinux.org.dkim-ed25519.key.vault
+```
+
+# GitLab Service Desk
+
+GitLab has a [Service Desk
+feature](https://docs.gitlab.com/ee/user/project/service_desk.html) which
+creates issues for incoming emails and allows multiple people to reply via
+GitLab on those issues and assign issues. GitLab generates a default email
+address with the following logic:
+
+```
+gitlab+<group>-<project>-<project-id>-issue-@archlinux.org
+```
+
+As we prefer to use user friendly addresses such as `privacy@archlinux.org` for communication a postfix alias is configured in `/etc/postix/aliases`.
+
+For a new GitLab Service Desk project, add a new alias to `/etc/postfix/aliases` as:
+
+```
+foobar: gitlab+<group>-<project>-<project-id>-issue-@archlinux.org
+```
+
+Then run `postalias`:
+
+```
+postalias /etc/postfix/aliases
+```

docs/fail2ban.md

+# fail2ban
+
+Fail2Ban is an intrusion prevention software framework that protects computer servers from brute-force attacks.
+
+By default all our servers playbook should include the `fail2ban` role which enables the `sshd` jail by default through `groups_vars`. The default `/etc/fail2ban/jail.local` configuration whitelists all the servers in `hosts`, take note that when adding a new postfix relayhost the `fail2ban` role has to be run on the postfix server to update the whitelist.
+
+## Jails
+
+Fail2ban can provide multiple jails for different services, to check the status of for example the `sshd` jail:
+
+```
+fail2ban-client status sshd
+```
+
+To unblock an IP Address:
+
+```
+fail2ban-client set sshd unbanip 8.8.8.8
+```
+
+### sshd
+
+The sshd jail should be enabled for every host we have, to block brute force ssh attacks.
+
+### postfix
+
+The postfix jail not enabled on any server. Adding it to a host:
+
+Add `fail2ban_jails` dict with `postfix: true` to the host's `host_vars`.
+
+### dovecot
+
+The dovecot jail is enabled for our mail server, blocking failed logins. Adding it to a host:
+
+Add `fail2ban_jails` dict with `dovecot: true` to the host's `host_vars`.
+
+### nginx_limit_req
+
+The nginx_limit_req jail is not enabled on any server. This jail bans IPs based repeated errors on nginx error log. Default blocking is 1 hour(s). Adding to a host:
+
+Add `fail2ban_jails` dict with `nginx_limit_req: true` to the host's `host_vars`.
+
+The `rsslimit` zone is whitelisted from being banned with `ignoreregex`, as we
+choose to not ban RSS abusers.

docs/geomirrors.md

+# Geo mirrors
+
+DevOps team maintain a geo mirror across the world. The Geo mirror is public facing on geo.mirror.pkgbuild.com domain and it will resolve the closest to the location of the requester mirror.
+
+## Locations
+
+| Mirror                                    | Location                    |
+| ----------------------------------------- | --------------------------- |
+| https://america.mirror.pkgbuild.com/      | Miami (United States)       |
+| https://asia.mirror.pkgbuild.com/         | Hong Kong                   |
+| https://berlin.mirror.pkgbuild.com/       | Berlin (Germany)            |
+| https://europe.mirror.pkgbuild.com/       | Prague (Czechia)            |
+| https://johannesburg.mirror.pkgbuild.com/ | Johannesburg (South Africa) |
+| https://london.mirror.pkgbuild.com/       | London (United Kingdom)     |
+| https://losangeles.mirror.pkgbuild.com/   | Los Angeles (United States) |
+| https://singapore.mirror.pkgbuild.com/    | Singapore                   |
+| https://sydney.mirror.pkgbuild.com/       | Sydney (Australia)          |
+| https://taipei.mirror.pkgbuild.com/       | Taipei (Taiwan)             |
+
+### Logical split
+The continent mirrors america, asia and europe contain the archive mirrors as well as repository mirrors. The city mirrors have just the repositories hosted.
+
+## Requirements
+- Host with Arch Linux installed
+- root access provided
+- Enough storage to host repos / debugrepos (at least)
+- Bandwidth (depends on location)
+
+## Adding a new mirror box
+- Add new entries in `hosts` file under `mirrors` and `geo_mirrors` sections
+- Adjust terraform `tf-stage1/archlinux.tf` to include the IPv4 and IPv6 entries of the new server
+- Adjust terraform `tf-stage1/templates.tf` to include the IPv4 and IPv6 entries of the new server as a `NS` record for `geo.mirror.pkgbuild.com`
+- Add a new files in `host_vars`
+    - `host_vars/<fqdn>/misc`
+        Containing all the information for the mirror itself
+    - `host_vars/<fqdn>/vault_wireguard.yml`
+        Containing the wireguard private key in encrypted vault
+
+## Ansible Playbooks execution
+
+| Playbook | Roles | Reason | Hosts (limits) |Comments |
+| ----------- | ----------- | ----------- | ----------- |  ----------- |
+| install_arch | All | Install Arch | | Optional if you can |
+| mirrors.yml | All | Setup mirror | `<fqdn>` | |
+| redirect.archlinux.org.yml | dyn_dns | Make TXT records | | |
+| repos.archlinux.org.yml | dbscripts | Allow debug repo syncing | | |
+| mirrors.yml | geo_dns | Add new domain to DNS | All other mirrors from geo.mirror | |
+| monitoring.archlinux.org.yml | wireguard,prometheus | Allow loki and prometheus to fetch data | | |
+| archlinux.org.yml | postgres,wireguard | Allow wireguard IP to connect for Mirror check | | Optional see Check Location below |
+
+### Add mirror in geo.mirror.pkgbuild.com
+
+Add mirror IP and FQDN in archweb admin https://archlinux.org/admin/mirrors/mirror/ under the `geo.mirror.pkgbuild.com` entry.
+
+### Check Location (optional)
+
+If you want the server to check for ping and stats create an entry in:
+ https://archlinux.org/admin/mirrors/checklocation/

docs/github-mirror.md

-# How to add a mirror to GitHub
-
-If you want to mirror your repository "myproject" from gitlab.archlinux.org to the github.com/archlinux organization,
-you should create an empty project for your project at github.com/archlinux/myproject or
-if that's an existing repository, make sure that the current histories of the source and
-target repository are exactly the same.
-
-Then, go to https://gitlab.archlinux.org/archlinux/myproject/-/settings/repository and open
-"Mirroring Repositories".
-Make sure it has these settings:
-
-* Git repository URL: `ssh://git@github.com/archlinux/myproject.git`
-* Mirror direction: Push
-* Authentication method: "SSH public key"
-* Only mirror protected branches: Off
-
-and then click Mirror repository.
-
-A new entry will pop up which has a button titled "Copy SSH public key". Click that.
-
-Then go to https://github.com/archlinux/myproject/settings/keys and add a new deploy key.
-Give it the title "gitlab.archlinux.org" so we know where it's from and then paste the
-public key you copied from GitLab just now.
-
-Your push mirror should now work.