Merge branch 'morten/debuginfod' into 'master'

debuginfod: Implement role

See merge request !168

docs/ssh-hostkeys.txt

@@ -108,6 +108,17 @@
 256 MD5:5b:6b:10:c6:78:b3:ad:cf:0b:3f:84:e4:24:7b:92:5a root@archlinux-packer (ED25519)
 3072 MD5:2c:88:5f:24:07:2a:63:ef:86:27:1b:f1:18:2d:fe:dd root@archlinux-packer (RSA)
 
+# debuginfod.archlinux.org
+1024 SHA256:Pr4dHixKB9iUWfnsGrBJttz2WRP1xmVkdDETCF1U5FM root@archlinux-packer (DSA)
+256 SHA256:64Tuq5ZDPuHQYVlvpY/RqNN4EZCgOw0SLWwU8esFF10 root@archlinux-packer (ECDSA)
+256 SHA256:h3PoOLj4fBkElmcwBa146uHFsggXl8hSgDUYQrpjJ9o root@archlinux-packer (ED25519)
+3072 SHA256:9j0GGtHsWWlbDH0COPinY02QeS+ykl40LCSKnuGDVRc root@archlinux-packer (RSA)
+
+1024 MD5:ba:d3:54:53:2d:0d:c3:da:61:d9:f7:af:dc:f2:f8:c4 root@archlinux-packer (DSA)
+256 MD5:ce:c1:de:8d:6e:fd:13:de:0e:82:08:1f:29:76:41:6d root@archlinux-packer (ECDSA)
+256 MD5:32:66:ec:d5:e2:75:66:c7:0d:a7:8a:8c:17:ba:dc:4b root@archlinux-packer (ED25519)
+3072 MD5:b2:79:fd:7c:e7:4f:2f:62:2f:17:71:21:d2:94:2d:2a root@archlinux-packer (RSA)
+
 # europe.mirror.pkgbuild.com
 1024 SHA256:Oq3eikchfo8Wt6AUzWAiU1mDR24rXudJR/zqKBFnrMo root@europe.mirror.pkgbuild.com (DSA)
 256 SHA256:3S0HuO72jHUUrPM8BjfcjsB0FNXkubxovc7Sm5jZBjc root@europe.mirror.pkgbuild.com (ECDSA)

docs/ssh-known_hosts.txt

@@ -48,6 +48,11 @@ dashboards.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAA
 dashboards.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINBlMtNCc3M9ZlCFCXVzdRscvJfB6DJpCEeOoraVD4/b
 dashboards.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCfE077N+TtbUCXn4vau12+OhnrSM7GY3vyEDF5+GkVKAQA3M2Cvme8K0o64sAnEegpGDAysVVCg81vlHonqERFxCp0Reu0qvbIqMc4nsesGUyM8Mqps7BJtZOiYUv6cWZAEeTvRvU1717M+43Os/q/3YFOPSvC5y7Sud4JSmOKp1+zTbDZF+9kL0EjlS452FaCcAA5PG74LsZzqmOwbw+TyFwE73Dt1D7CifFoTwejDFhY5mQoGHuopwO3BA+cqfM2v3LNq0j2AuJebq5PajK9YzdNPEMqEoxw6U3VWpQAU9QNkxLrxsIT8Vu18YAFg84520ie3GQ49F8Lj6dw9qBhUzSMHoLdsMFKwUfHJrYeXNsRpQNiPOlqXX7QCw62joF211Er5cGf19rFGC7y1F2iG8MWrSEW6x9rgrgisBT05ecGtyEx0LXH/vINAhA5/zvWjDuefn/wCSy7IR0I9n7nSTXMXm0/qr4brtsFA0V+O7mmAA9ErclGE6lIBHpWfd0=
 
+# debuginfod.archlinux.org
+debuginfod.archlinux.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGQoDiPiCKhyb6T8Wsh/jQe4VWkZIGDaMH67OYnRN/fXN1Urq4uVuRx7iyMBtk9H8I4jtfVVTALJjjKH3xapHd4=
+debuginfod.archlinux.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICDH8NaxqnAme6QXO8S/4NrqrffUNEMSi5xnyrQt6RSf
+debuginfod.archlinux.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiOZRxYQxY9A9K8gk11bURFgWxtWk3KwHs1Y4PTxlfAl3Cf8fGxxouBc9bwCO1qivnaPeIYt0gedJRtQOmt4zIRaK32tb3tV6j8nucufQAgQ8jgfIyvLeyv+Jt/DuUtTkGjT4pu/LB14fU/U4CGy9NTAQBNuFfnFgmQqAHPmfwx2QLovVUBpdqFn+GBw6DJAhcQgrTKZHGg1hJDCEhK222TAAKTc95fyZLX8YG41p24hccCUhSGTkfH+TWQO7MPHVNK0fyDJpnC1pD+jYqKiHS7eEsSzndRV3WrwRdodbecWdIIoLEgw1i07Z4obdnFj2CPt1G8F0ohNiC9135z0Hc09dQ5uyX7LY1k21NUJ0hHGpdhomYpFwQhhWRdKaqq+S00K0Mzc0oiPaFClIXFw48fpvvRRj54bnVYi2GPO+y8kvfl5x/4vdsJoyQcPaZ3ztftECS8mOIn806rY0tKaxIm3OMVP2DeC8okgXzo9R0/sQkYNmyH72FgJHwcWsOLZk=
+
 # europe.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBASkHNbJZvfME9OAFLZpxoVMt7JfKhN8/VpH6JPRD8eRXfXc2Wt0YOZQGzJsrUNoFchEUUGeNxs7vmj8nwtfqGI=
 europe.mirror.pkgbuild.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF6sVYSrTIVj+xwyC5uJdFVU+X50tAIDEndsnGta75C7

host_vars/debuginfod.archlinux.org/misc

+---
+filesystem: btrfs
+ipv4_address: 168.119.240.111
+ipv6_address: 2a01:4f8:c010:74d4::1
+wireguard_address: 10.0.0.35
+wireguard_public_key: Wp9ruR2+pCj0TsATuJZiUxk9x6BwcUhXs/yZlmGYjRE=

host_vars/debuginfod.archlinux.org/vault_wireguard.yml

+$ANSIBLE_VAULT;1.1;AES256
+38656365643863646566323864326165613033393365383837653166306463643562643763663965
+6265656632396263643636326366643338386238336335330a643164343961386162656363373738
+31336461626537656366666232376461343431346361353239623530336637316536333565373733
+3062623432663531610a386339633161656331343965613833383538306533663730356534393364
+37396232653737666234393432636135656562626635353539353732376133303135306539373333
+30303535383933363234303135383065313163613762393533343536643033623439383439616235
+65386263323931323464396164373333613130303530656433306165633531383230313533643961
+64613738656332306266

host_vars/mirror.pkgbuild.com/misc

 ---
 mirror_domain: mirror.pkgbuild.com
+mirror_debug_packages: false
 archweb_mirrorcheck_locations: [20, 21]
 filesystem: btrfs
 

hosts

@@ -141,6 +141,7 @@ man.archlinux.org
 dashboards.archlinux.org
 lists.archlinux.org
 gluebuddy.archlinux.org
+debuginfod.archlinux.org
 
 [wireguard]
 archlinux.org
@@ -178,6 +179,7 @@ man.archlinux.org
 dashboards.archlinux.org
 lists.archlinux.org
 gluebuddy.archlinux.org
+debuginfod.archlinux.org
 
 [kape_servers]
 asia.mirror.pkgbuild.com

playbooks/debuginfod.archlinux.org.yml

+---
+- name: setup debuginfod.archlinux.org
+  hosts: debuginfod.archlinux.org
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: firewalld }
+    - { role: wireguard }
+    - { role: hardening }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: certbot }
+    - { role: nginx }
+    - { role: debuginfod }
+    - { role: syncdebug }
+    - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/mirrors.yml

@@ -9,6 +9,7 @@
     - { role: certbot }
     - { role: nginx }
     - { role: syncrepo, tags: ['nginx'] }
+    - { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
     - { role: archweb, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
     - { role: prometheus_exporters }
     - { role: promtail }

roles/dbscripts/templates/rsyncd.conf.proto.j2

@@ -91,6 +91,15 @@ hosts deny = *
 	secrets file = /etc/rsyncd.secrets
 	max connections = 0
 
+# Debug repositories
+[debug_packages]
+	path = /srv/ftp
+	comment =  debug packages
+	exclude = *
+	include = /*-debug/*** /pool /pool/*-debug/***
+	hosts allow = {{ groups['mirrors'] | map('extract', hostvars, ['ipv4_address']) | join(' ') }} {{ groups['mirrors'] | map('extract', hostvars, ['ipv6_address']) | join(' ') }} {{ hostvars['debuginfod.archlinux.org']['ipv4_address'] }} {{ hostvars['debuginfod.archlinux.org']['ipv6_address'] }}
+	max connections = 0
+
 # Individual repositories
 [core]
 	path = /srv/ftp/core

roles/debuginfod/defaults/main.yml

+---
+debuginfod_httpd: false
+debuginfod_domain: debuginfod.archlinux.org
+debuginfod_port: 8002
+debuginfod_database: /var/cache/debuginfod/debuginfod.sqlite
+debuginfod_package_paths:
+  - /srv/ftp/pool/packages-debug
+  - /srv/ftp/pool/community-debug

roles/debuginfod/files/index.html

+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Debuginfod service - Arch Linux</title>
+  </head>
+  <body>
+    <h1>Debuginfod service - Arch Linux</h1>
+    <img src="archlinux.png">
+
+    <p>This is the <a href="https://sourceware.org/elfutils/Debuginfod.html">debuginfod</a> service for Arch Linux. For more information about it, please refer to <a href="https://wiki.archlinux.org/title/Debuginfod">https://wiki.archlinux.org/title/Debuginfod</a>.</p>
+
+    <p>The following environment variable must be set in your shell to use it:</p>
+    <pre>DEBUGINFOD_URLS="https://debuginfod.archlinux.org"</pre>
+
+    <p>A dashboard can be found on <a href="https://dashboards.archlinux.org/d/U0xac1x7z/debuginfod?orgId=1">https://dashboards.archlinux.org/</a>.</p>
+    <p>List of currently available packages can be found on <a href="/packages">here</a>.</p>
+  </body>
+</html>

roles/debuginfod/files/packagelist.service

+[Unit]
+Description=Synchronize package list
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/bash -c "find /srv/ftp/pool/*-debug/ -type f -not -name *.sig > /srv/http/debuginfod/packages"

roles/debuginfod/files/packagelist.timer

+[Unit]
+Description=Sync package lists every minute
+
+[Timer]
+OnCalendar=minutely
+AccuracySec=1m
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/debuginfod/handlers/main.yml

+---
+
+- name: reload debuginfod
+  service: name=debuginfod state=reloaded

roles/debuginfod/tasks/main.yml

+---
+- name: install debuginfod
+  pacman: name=debuginfod state=present
+
+- name: create ssl cert
+  include_role:
+    name: certificate
+  vars:
+    domains: ["{{ debuginfod_domain }}"]
+  when: debuginfod_domain
+
+- name: configure debuginfod systemd service
+  template: src=debuginfod.service.j2 dest=/usr/lib/systemd/system/debuginfod.service owner=root group=root mode=0644
+  vars:
+    debuginfod_package_path: "{{ debuginfod_package_paths | join(' ') }}"
+  notify:
+    - reload debuginfod
+
+- name: create http directory for debuginfod website files
+  file: path=/srv/http/debuginfod state=directory owner=root group=root mode=0755
+
+- name: install website files
+  copy: src={{ item }} dest=/srv/http/debuginfod/{{ item }} owner=root group=root mode=0644
+  loop:
+    - archlinux.png
+    - index.html
+
+- name: install packagelist units
+  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
+  loop:
+    - packagelist.timer
+    - packagelist.service
+
+- name: start and enable packagelist.timer
+  service: name=packagelist.timer enabled=yes state=started
+
+- name: make nginx log dir
+  file: path=/var/log/nginx/{{ debuginfod_domain }} state=directory owner=root group=root mode=0755
+
+- name: set up nginx
+  template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/debuginfod.conf owner=root group=root mode=0644
+  notify:
+    - reload nginx
+  when: debuginfod_domain
+  tags: ['nginx']
+
+- name: open debuginfod ipv4 port for monitoring.archlinux.org
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=8002 accept"
+  tags:
+    - firewall
+
+- name: start and enable debuginfod
+  service: name=debuginfod enabled=yes state=started

roles/debuginfod/templates/debuginfod.service.j2

+[Unit]
+Description=debuginfod server
+After=network.target network-online.target nss-lookup.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/debuginfod -d {{debuginfod_database}} -p {{debuginfod_port}} -F -Z .tar.zst=zstdcat {{debuginfod_package_path}}
+Restart=on-failure
+DynamicUser=yes
+PrivateDevices=true
+PrivateUsers=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+MemoryDenyWriteExecute=true
+RestrictRealtime=true
+ReadOnlyPaths={{debuginfod_package_path}}
+CacheDirectory=debuginfod
+
+[Install]
+WantedBy=multi-user.target

roles/debuginfod/templates/nginx.d.conf.j2

+server {
+    listen       80;
+    listen       [::]:80;
+    server_name  {{ debuginfod_domain }};
+
+    access_log   /var/log/nginx/{{ debuginfod_domain }}/access.log reduced;
+    access_log   /var/log/nginx/{{ debuginfod_domain }}/access.log.json json_reduced;
+    error_log    /var/log/nginx/{{ debuginfod_domain }}/error.log;
+
+    include snippets/letsencrypt.conf;
+
+    location / {
+        access_log off;
+        return 301 https://$server_name$request_uri;
+    }
+}
+
+server {
+    listen       443 ssl http2;
+    listen       [::]:443 ssl http2;
+    server_name  {{ debuginfod_domain }};
+
+    access_log   /var/log/nginx/{{ debuginfod_domain }}/access.log reduced;
+    access_log   /var/log/nginx/{{ debuginfod_domain }}/access.log.json json_reduced;
+    error_log    /var/log/nginx/{{ debuginfod_domain }}/error.log;
+
+    ssl_certificate      /etc/letsencrypt/live/{{ debuginfod_domain }}/fullchain.pem;
+    ssl_certificate_key  /etc/letsencrypt/live/{{ debuginfod_domain }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ debuginfod_domain }}/chain.pem;
+
+    root  /srv/http/debuginfod;
+    index index.html;
+
+    location / {
+        try_files $uri $uri/ @debuginfod;
+    }
+
+    location = /packages {
+        default_type "text/plain";
+    }
+
+    location @debuginfod {
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_pass http://127.0.0.1:8002;
+    }
+}

roles/prometheus/defaults/main.yml

@@ -3,6 +3,11 @@ gitlab_runner_exporter_port: '9252'
 prometheus_domain: "{{ hostvars['dashboards.archlinux.org']['wireguard_address'] }}"
 prometheus_mysqld_exporter_port: '9104'
 prometheus_receive_only: false
+prometheus_remote_write_relabel_configs:
+  - label: job
+    regex: debuginfod
+  - label: __name__
+    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
 
 # for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/  - https:\/\//"
 blackbox_targets:
@@ -57,6 +62,7 @@ blackbox_targets:
     - mail.archlinux.org:465
     - mail.archlinux.org:993
     - coc.archlinux.org:443
+    - debuginfod.archlinux.org:443
   smtp_starttls:
     - mail.archlinux.org:25
     - lists.archlinux.org:25

roles/prometheus/templates/prometheus.yml.j2

@@ -14,14 +14,16 @@ alerting:
        - localhost:9093
 
 remote_write:
+{% for relabel_config in prometheus_remote_write_relabel_configs %}
 - url: http://{{ prometheus_domain }}:9090/api/v1/write
   write_relabel_configs:
-  - source_labels: [__name__]
-    regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
+  - source_labels: [{{ relabel_config.label }}]
+    regex: "{{ relabel_config.regex }}"
     action: keep
   basic_auth:
     username: {{ vault_prometheus_user }}
     password: {{ vault_prometheus_passwd }}
+{% endfor %}
 
 scrape_configs:
   - job_name: prometheus
@@ -67,6 +69,12 @@ scrape_configs:
 
     {% endfor %}
 
+  - job_name: 'debuginfod'
+    static_configs:
+    - targets: ['{{ hostvars['debuginfod.archlinux.org']['wireguard_address'] }}:8002']
+      labels:
+        instance: "debuginfod.archlinux.org"
+
   - job_name: 'gitlab_runner_exporter'
     static_configs:
     {% for host in groups['gitlab_runners'] %}