.gitlab/issue_templates/Offboarding.md

@@ -15,6 +15,8 @@ This template should be used for offboarding Arch Linux team members.
 
 - [ ] Remove user email by reverting instructions from `docs/email.md`.
 - [ ] Set user to inactive in archweb: https://www.archlinux.org/admin/auth/user/
+- [ ] Remove member from [staff mailing lists](https://lists.archlinux.org/admin/staff/members)
+- [ ] Ask the user to leave `#archlinux-staff` on Freenode and forget the password
 
 ## TU/Developer offboarding checklist
 
@@ -24,7 +26,7 @@ This template should be used for offboarding Arch Linux team members.
 - [ ] Remove the user from the `Trusted Users`/`Developers` groups on Keycloak.
 - [ ] Moderate email address on [arch-dev-public](https://lists.archlinux.org/admin/arch-dev-public/members) (find member and moderate)
 - [ ] Remove member from [arch-tu mailing lists](https://lists.archlinux.org/admin/arch-tu/members)
-- [ ] Remove member from [staff mailing lists](https://lists.archlinux.org/admin/staff/members)
+- [ ] Create keyring revocation ticket
 
 ## DevOps offboarding checklist
 

.gitlab/issue_templates/Onboarding.md

@@ -17,6 +17,7 @@ It can also be used as a reference for adding new roles to an existing team memb
 - [ ] Create a new user in archweb: https://www.archlinux.org/devel/newuser/
       This is also linked in the django admin backend at the top
 - [ ] Subscribe user to internal [staff mailing list](https://lists.archlinux.org/admin/staff/members/add)
+- [ ] Give the user access to `#archlinux-staff` on Freenode
 
 ## Developer onboarding checklist
 

group_vars/all/archusers.yml

@@ -240,8 +240,13 @@ arch_users:
   hashworks:
     name: "Justin Kromlinger"
     ssh_key: hashworks.pub
+    shell: /bin/zsh
     groups:
       - tu
+    additional_ssh_keys:
+      - name: hashworks_yubikey_5_nfc.pub
+        hosts:
+          - all
   heftig:
     name: "Jan Steffens"
     ssh_key: heftig.pub

hosts

@@ -141,6 +141,7 @@ europe.mirror.pkgbuild.com
 repro2.pkgbuild.com
 runner1.archlinux.org
 md.archlinux.org
+man.archlinux.org
 
 [kape_servers]
 asia.mirror.pkgbuild.com

pubkeys/hashworks.pub

-sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBO5oIvnZWTBpP/Kzi8H3QTkhQgPP8uYQUJrSHwhsUWNp1AygiwmeGtB1rjysqwUN0kH7A24HUCHAizq/mFHfvGMAAAAEc3NoOg== hashworks@yubikey@2020-02-17
+sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBDGzbP1z+hTV1wOOFjVfQNCLTHmouswv4N8aBb1Jw9TOAmbNs/3LSvwy/Zo6jNL7+OS9tkPtr+nAdL03reDqYJEAAAAEc3NoOg== hashworks@yubikey-5c@2021-03-21

pubkeys/hashworks_yubikey_5_nfc.pub

+sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBO5oIvnZWTBpP/Kzi8H3QTkhQgPP8uYQUJrSHwhsUWNp1AygiwmeGtB1rjysqwUN0kH7A24HUCHAizq/mFHfvGMAAAAEc3NoOg== hashworks@yubikey-5nfc@2020-02-17

pubkeys/klausenbusk.pub

+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDwrC+miAaggiArz1d1QFMpEk8DTKymDIK4I8VagaZ8f+DUse5+ScxWt2xA+mgk5kDkjvnByCxa5mbuUuWJjAGu1e/TomRO6sKtbJyOwtWxlAzE4SuzE/V5g7obmhTcR5J8m81+mDsMWb+4/ac72/NRyhy9Dnqt/JJybskuEjYTE9GkBq5KV6xveAVVoTjwIE/L8MlCGzvUD4QsalcizkG1xfsI7aeRLonXvwv7Jw/uLTI6Z/Oa4alrip+RWpXQSPBoA6wpAASR41qR8QFiD5kN6aGRv3xqJNPdSfNc1mpkNDVKAeRJnFj5tzOEdaHHSkmB45SbjQk/ZtVlmBIv+3HAPIML01DHc3dger1Po/c2vHg3hxI0ve3rxio5ni4IN/54NUgNqBRBz3gZy6PSyt8HtUfuJzNwZRLdempG+KF1Rk9OHiW7hrokPHYDJWwQmG3x7WTgZxxZy5ou6SCLeE71xspeAHUT4h0Lujpdy5gC6E0TPsH6Z78Qt0O5cVIlZIBqxMwDs6SVtULnKR0K1tDEcX2TlZNGTwkWToq2KZMcj2c5ltTxgmscvhB3RHYIyRLthB6JocHChyD06kVfNDQhckNXwpi+9HCm3fz55O88kn/Mcnirr3etM6mb3Zp8DhX/2IZKLQWe5zr505XkNPVpL7CIBM/iG7fBC8qShj+dGw== klausenbusk

roles/archusers/templates/authorized_keys.j2

@@ -2,7 +2,7 @@
 {{ lookup('file', '../pubkeys/' + item.value.ssh_key) }}
 {% if item.value.additional_ssh_keys is defined %}
 	{% for key in item.value.additional_ssh_keys %}
-		{% if inventory_hostname in key.hosts %}
+		{% if inventory_hostname in key.hosts or 'all' in key.hosts %}
 {{ lookup('file', '../pubkeys/' + key.name) }}
 		{% endif %}
 	{% endfor %}

roles/archweb/defaults/main.yml

@@ -10,7 +10,6 @@ archweb_domains_redirects:
         'planet.archlinux.org': '/planet$request_uri'
 archweb_domains_templates:
         'ipxe.archlinux.org': 'ipxe.archlinux.org.j2'
-archweb_network_check_domain: 'www.archlinux.org'
 archweb_allowed_hosts: ["{{ archweb_domain }}", 'ipxe.archlinux.org']
 archweb_nginx_conf: '/etc/nginx/nginx.d/archweb.conf'
 archweb_repository: 'https://github.com/archlinux/archweb.git'

roles/archweb/templates/maintenance-nginx.d.conf.j2

@@ -15,13 +15,6 @@ server {
 
     include snippets/letsencrypt.conf;
 
-{% if domain == archweb_network_check_domain %}
-    location /check_network_status.txt {
-        access_log off;
-        add_header Cache-Control "max-age=0, must-revalidate";
-        return 200 'NetworkManager is online';
-    }
-{% endif %}
     location / {
         access_log off;
         return 301 https://$server_name$request_uri;
@@ -61,12 +54,6 @@ server {
 
     include snippets/letsencrypt.conf;
 
-    location /check_network_status.txt {
-        access_log off;
-        add_header Cache-Control "max-age=0, must-revalidate";
-        return 200 'NetworkManager is online';
-    }
-
     location / {
         access_log off;
         return 301 https://$server_name$request_uri;

roles/archweb/templates/nginx.d.conf.j2

@@ -28,14 +28,6 @@ server {
 
     include snippets/letsencrypt.conf;
 
-{% if domain['domain'] == archweb_network_check_domain %}
-    location /check_network_status.txt {
-        access_log off;
-        add_header Cache-Control "max-age=0, must-revalidate";
-        return 200 'NetworkManager is online';
-    }
-
-{% endif %}
     location /.well-known {
         add_header Access-Control-Allow-Origin *;
         return 301 https://$server_name$request_uri;
@@ -85,12 +77,6 @@ server {
 
     include snippets/letsencrypt.conf;
 
-    location /check_network_status.txt {
-        access_log off;
-        add_header Cache-Control "max-age=0, must-revalidate";
-        return 200 'NetworkManager is online';
-    }
-
     location / {
         access_log off;
         return 301 https://$server_name$request_uri;

roles/borg_client/templates/backup-mysql.sh.j2

@@ -4,5 +4,5 @@ mysql_opts="--defaults-file={{mysql_backup_defaults}}"
 backupdir="{{mysql_backup_dir}}"
 
 [[ ! -d "$backupdir" ]] && mkdir -p "$backupdir"
-rm -rf "$backupdir"/*
+rm -rf "${backupdir:?}"/*
 mariabackup $mysql_opts --backup --target-dir="$backupdir"

roles/dbscripts/tasks/main.yml

@@ -28,7 +28,7 @@
   include_role:
     name: certificate
   vars:
-    domains: ["{{ repos_domain }}"]
+    domains: ["{{ repos_domain }}", "{{ repos_rsync_domain }}"]
 
 - name: make nginx log dir
   file: path=/var/log/nginx/{{ repos_domain }} state=directory owner=root group=root mode=0755

roles/matrix/tasks/main.yml

@@ -78,7 +78,7 @@
 - name: install synapse
   pip:
     name:
-      - 'matrix-synapse[postgres,systemd,url_preview,redis]==1.29.0'
+      - 'matrix-synapse[postgres,systemd,url_preview,redis]==1.30.1'
     state: latest
     extra_args: '--upgrade-strategy=eager'
     virtualenv: /var/lib/synapse/venv
@@ -92,7 +92,7 @@
 - name: install pantalaimon
   pip:
     name:
-      - 'pantalaimon==0.9.1'
+      - 'pantalaimon==0.9.2'
     state: latest
     extra_args: '--upgrade-strategy=eager'
     virtualenv: /var/lib/synapse/venv-pantalaimon
@@ -148,7 +148,7 @@
   git:
     repo: https://github.com/matrix-org/matrix-appservice-irc
     dest: /var/lib/synapse/matrix-appservice-irc
-    version: 0.24.0
+    version: 0.25.0
   become: true
   become_user: synapse
   become_method: sudo
@@ -157,8 +157,9 @@
     - restart matrix-appservice-irc
 
 - name: install matrix-appservice-irc
-  npm:
+  community.general.npm:
     path: /var/lib/synapse/matrix-appservice-irc
+    ci: true
   become: true
   become_user: synapse
   become_method: sudo

roles/matrix/templates/homeserver.yaml.j2

@@ -79,8 +79,7 @@ public_baseurl: https://{{ matrix_domain }}/
 # Whether to require authentication to retrieve profile data (avatars,
 # display names) of other users through the client API. Defaults to
 # 'false'. Note that profile data is also available via the federation
-# API, so this setting is of limited value if federation is enabled on
-# the server.
+# API, unless allow_profile_lookup_over_federation is set to false.
 #
 #require_auth_for_profile_requests: true
 
@@ -1785,7 +1784,26 @@ saml2_config:
 #
 #   client_id: Required. oauth2 client id to use.
 #
-#   client_secret: Required. oauth2 client secret to use.
+#   client_secret: oauth2 client secret to use. May be omitted if
+#        client_secret_jwt_key is given, or if client_auth_method is 'none'.
+#
+#   client_secret_jwt_key: Alternative to client_secret: details of a key used
+#      to create a JSON Web Token to be used as an OAuth2 client secret. If
+#      given, must be a dictionary with the following properties:
+#
+#          key: a pem-encoded signing key. Must be a suitable key for the
+#              algorithm specified. Required unless 'key_file' is given.
+#
+#          key_file: the path to file containing a pem-encoded signing key file.
+#              Required unless 'key' is given.
+#
+#          jwt_header: a dictionary giving properties to include in the JWT
+#              header. Must include the key 'alg', giving the algorithm used to
+#              sign the JWT, such as "ES256", using the JWA identifiers in
+#              RFC7518.
+#
+#          jwt_payload: an optional dictionary giving properties to include in
+#              the JWT payload. Normally this should include an 'iss' key.
 #
 #   client_auth_method: auth method to use when exchanging the token. Valid
 #       values are 'client_secret_basic' (default), 'client_secret_post' and
@@ -1906,7 +1924,7 @@ oidc_providers:
   #
   #- idp_id: github
   #  idp_name: Github
-  #  idp_brand: org.matrix.github
+  #  idp_brand: github
   #  discover: false
   #  issuer: "https://github.com/"
   #  client_id: "your-client-id" # TO BE FILLED
@@ -2663,19 +2681,20 @@ user_directory:
 
 
 
-# Local statistics collection. Used in populating the room directory.
-#
-# 'bucket_size' controls how large each statistics timeslice is. It can
-# be defined in a human readable short form -- e.g. "1d", "1y".
+# Settings for local room and user statistics collection. See
+# docs/room_and_user_statistics.md.
 #
-# 'retention' controls how long historical statistics will be kept for.
-# It can be defined in a human readable short form -- e.g. "1d", "1y".
-#
-#
-#stats:
-#   enabled: true
-#   bucket_size: 1d
-#   retention: 1y
+stats:
+  # Uncomment the following to disable room and user statistics. Note that doing
+  # so may cause certain features (such as the room directory) not to work
+  # correctly.
+  #
+  #enabled: false
+
+  # The size of each timeslice in the room_stats_historical and
+  # user_stats_historical tables, as a time period. Defaults to "1d".
+  #
+  #bucket_size: 1h
 
 
 # Server Notices room configuration

roles/matrix/templates/irc-bridge.yaml.j2

 # Configuration specific to AS registration. Unless other marked, all fields
 # are *REQUIRED*.
+# Unless otherwise specified, these keys CANNOT be hot-reloaded.
 homeserver:
   # The URL to the home server for client-server API calls, also used to form the
   # media URLs as displayed in bridged IRC channels:
@@ -8,7 +9,7 @@ homeserver:
   # The URL of the homeserver hosting media files. This is only used to transform
   # mxc URIs to http URIs when bridging m.room.[file|image] events. Optional. By
   # default, this is the homeserver URL, specified above.
-  #
+  # This key CAN be hot-reloaded.
   # media_url: "http://media.repo:8008"
 
   # Drop Matrix messages which are older than this number of seconds, according to
@@ -21,6 +22,7 @@ homeserver:
   # clock times and hence produce different origin_server_ts values, which may be old
   # enough to cause *all* events from the homeserver to be dropped.
   # Default: 0 (don't ever drop)
+  # This key CAN be hot-reloaded.
   # dropMatrixMessagesAfterSecs: 300 # 5 minutes
 
   # The 'domain' part for user IDs on this home server. Usually (but not always)
@@ -42,6 +44,8 @@ homeserver:
 
 # Configuration specific to the IRC service
 ircService:
+  # All server keys can be hot-reloaded, however existing IRC connections
+  # will not have changes applied to them.
   servers:
 {% for network in vault_matrix_secrets.irc_networks %}
     # The address of the server to connect to.
@@ -62,9 +66,9 @@ ircService:
       # An ID for uniquely identifying this server amongst other servers being bridged.
       # networkId: "example"
 
-      # URL to an icon used as the network icon whenever this network appear in
+      # MXC URL to an icon used as the network icon whenever this network appear in
       # a network list. (Like in the riot room directory, for instance.)
-      # icon: https://example.com/images/hash.png
+      icon: mxc://matrix.org/LpsSLrbANVrEIEOgEaVteItf
 
       # The port to connect to. Optional.
       port: {{ network.port }}
@@ -383,6 +387,7 @@ ircService:
   # Configuration for an ident server. If you are running a public bridge it is
   # advised you setup an ident server so IRC mods can ban specific matrix users
   # rather than the application service itself.
+  # This key CANNOT be hot-reloaded
   ident:
     # True to listen for Ident requests and respond with the
     # matrix user's user_id (converted to ASCII, respecting RFC 1413).
@@ -405,6 +410,7 @@ ircService:
 
   # Configuration for logging. Optional. Default: console debug level logging
   # only.
+  # This key CANNOT be hot-reloaded
   logging:
     # Level to log on console/logfile. One of error|warn|info|debug
     level: "info"
@@ -420,6 +426,7 @@ ircService:
     maxFiles: 5
 
   # Metrics will then be available via GET /metrics on the bridge listening port (-p).
+  # This key CANNOT be hot-reloaded
   metrics:
     # Whether to actually enable the metric endpoint. Default: false
     enabled: true
@@ -446,6 +453,7 @@ ircService:
   #   POST /irc/$domain/user/$user_id => Issue a raw IRC command down this connection.
   #                                      Format: new line delimited commands as per IRC protocol.
   #
+  # This key CANNOT be hot-reloaded
   debugApi:
     # True to enable the HTTP API endpoint. Default: false.
     enabled: false
@@ -458,6 +466,7 @@ ircService:
   # GET /_matrix/provision/unlink
   # GET /_matrix/provision/listlinks
   #
+  # This key CANNOT be hot-reloaded
   provisioning:
     # True to enable the provisioning HTTP endpoint. Default: false.
     enabled: false
@@ -485,9 +494,11 @@ ircService:
   # for storage in the database. Passwords are stored by using the admin room command
   # `!storepass server.name passw0rd. When a connection is made to IRC on behalf of
   # the Matrix user, this password will be sent as the server password (PASS command).
+  # This key CANNOT be hot-reloaded
   passwordEncryptionKeyPath: "/etc/synapse/{{ matrix_server_name }}.ircpass.key"
 
   # Config for Matrix -> IRC bridging
+  # This key CANNOT be hot-reloaded
   matrixHandler:
     # Cache this many matrix events in memory to be used for m.relates_to messages (usually replies).
     eventCacheSize: 4096
@@ -523,11 +534,14 @@ advanced:
   # however for large bridges it is important to rate limit the bridge to avoid
   # accidentally overloading the homeserver. Defaults to 1000, which should be
   # enough for the vast majority of use cases.
+  # This key CAN be hot-reloaded
   maxHttpSockets: 1000
   # Max size of an appservice transaction payload, in bytes. Defaults to 10Mb
+  # This key CANNOT be hot-reloaded.
   maxTxnSize: 10000000
 
 # Capture information to a sentry.io instance
+# This key CANNOT be hot-reloaded.
 sentry:
   enabled: false
   dsn: "https://<key>@sentry.io/<project>"
@@ -537,6 +551,7 @@ sentry:
   # serverName: ""
 
 # Use an external database to store bridge state.
+# This key CANNOT be hot-reloaded.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
   engine: "postgres"

roles/prometheus/files/node.rules.yml

@@ -3,13 +3,13 @@ groups:
     interval: 60s
     rules:
       - alert: HostHighCpuLoad
-        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle",instance!~"build.archlinux.org",instance!~"repro1.pkgbuild.com",instance!~"repro2.pkgbuild.com"}[5m])) * 100) > 80
+        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle",instance!~"build.archlinux.org",instance!~"repro1.pkgbuild.com",instance!~"repro2.pkgbuild.com",instance!~"runner2.archlinux.org"}[5m])) * 100) > 90
         for: 5m
         labels:
           severity: warning
         annotations:
           summary: "Host high CPU load (instance {{ $labels.instance }})"
-          description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+          description: "CPU load is > 90%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
 
       - alert: HostSwapIsFillingUp
         expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80
@@ -65,14 +65,14 @@ groups:
           summary: "Host out of disk space (instance {{ $labels.instance }})"
           description: "Disk is almost full (< 20% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
 
-      - alert: HostDiskWillFillIn4Hours
-        expr: predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs",mountpoint!~"/backup"}[1h], 4 * 3600) < 0
-        for: 5m
+      - alert: HostDiskWillFillIn24Hours
+        expr: (node_filesystem_avail_bytes{mountpoint!~"/backup"} * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs",mountpoint!~"/backup"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+        for: 2m
         labels:
           severity: warning
         annotations:
-          summary: "Host disk will fill in 4 hours (instance {{ $labels.instance }})"
-          description: "Disk will fill in 4 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+          summary: "Host disk will fill in 24 hours (instance {{ $labels.instance }})"
+          description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
 
       - alert: HostOutOfInodes
         expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10
@@ -211,6 +211,14 @@ groups:
         annotations:
           description: 'host {{ $labels.instance }} has out of date packages'
           summary: '{{ $labels.instance }} has {{ $value }} > 50 out of date packages'
+      - alert: pacman_security_updates_pending
+        expr: pacman_security_updates_pending > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          description: 'host {{ $labels.instance }} has vulnerable date packages'
+          summary: '{{ $labels.instance }} has {{ $value }} vulnerable packages'
 
   - name: btrfs
     interval: 2m

roles/syncrepo/files/syncrepo

@@ -59,9 +59,9 @@ bwlimit=0
 source_url='rsync://rsync.archlinux.org/ftp_tier1'
 
 # An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
-# If you are a tier 1 mirror use: http://rsync.archlinux.org/lastupdate
+# If you are a tier 1 mirror use: https://rsync.archlinux.org/lastupdate
 # Otherwise use the HTTP(S) URL from your chosen mirror.
-lastupdate_url='http://rsync.archlinux.org/lastupdate'
+lastupdate_url='https://rsync.archlinux.org/lastupdate'
 
 #### END CONFIG
 

tf-stage1/archlinux.tf

@@ -154,6 +154,7 @@ locals {
     "conf"           = "60a06a1c02e42b36c3b4919f4d6de6bf"
     "whatcanwedofor" = "b5f8011047c1610ace52e754b568c834"
     "openpgpkey"     = "7533dfbf3947a5730d9cbcc1e5e63102"
+    "bugs-old"       = "1f3308c8d5763eecb4f9013291aeeac4"
   }
 
   # This creates archlinux.org TXT DNS entries
@@ -291,6 +292,7 @@ locals {
     rsync         = { value = "gemini" }
     sources       = { value = "gemini" }
     "static.conf" = { value = "redirect" }
+    logging       = { value = "monitoring" }
     status        = { value = "stats.uptimerobot.com." }
     svn           = { value = "gemini" }