Implement centralized logging

Fix #263

docs/servers.md

@@ -111,7 +111,7 @@ Medium-fast-ish packet.net Arch Linux box.
 
 ## monitoring.archlinux.org
 
-  Prometheus and Grafana server which collects performance/metrics from our services and runs alertmanager.
+  Prometheus, Loki and Grafana server which collects performance/metrics and logs from our services and runs alertmanager.
 
 ### Services
   - Alertmanager

playbooks/accounts.archlinux.org.yml

@@ -20,3 +20,4 @@
     - { role: keycloak }
     - { role: borg_client, tags: ["borg"] }
     - { role: fail2ban }
+    - { role: promtail }

playbooks/archive-mirrors.yml

@@ -14,3 +14,4 @@
     - { role: syncarchive }
     - { role: archive_web }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/archlinux.org.yml

@@ -38,3 +38,4 @@
     - { role: archweb, archweb_planet: true }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/aur-dev.archlinux.org.yml

@@ -20,3 +20,4 @@
     - { role: fail2ban }
     - { role: aurweb, aurweb_domain: 'aur-dev.archlinux.org', aurweb_version: 'pu' }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/aur.archlinux.org.yml

@@ -9,6 +9,7 @@
     - { role: sshd, sshd_enable_includes: true }
     - { role: root_ssh }
     - { role: prometheus_exporters }
+    - { role: promtail }
     - { role: certbot }
     - { role: nginx }
     - { role: mariadb, mariadb_query_cache_type: '0', mariadb_innodb_file_per_table: true, mariadb_innodb_buffer_pool_size: '1G' }

playbooks/bbs.archlinux.org.yml

@@ -18,3 +18,4 @@
     - { role: postfix, postfix_relayhost: "mail.archlinux.org" }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/bugs.archlinux.org.yml

@@ -18,3 +18,4 @@
     - { role: postfix, postfix_relayhost: "mail.archlinux.org" }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/build.archlinux.org.yml

@@ -14,3 +14,4 @@
     - { role: archbuild }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/gemini.archlinux.org.yml

@@ -26,3 +26,4 @@
     - { role: postfix, postfix_relayhost: "mail.archlinux.org" }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/gitlab-runners.yml

@@ -10,4 +10,5 @@
     - { role: root_ssh }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }
     - { role: gitlab_runner }

playbooks/gitlab.archlinux.org.yml

@@ -16,4 +16,5 @@
         gitlab_pages_https_addresses: ['116.203.6.156:443', '[2a01:4f8:c2c:5d2d::2]:443']}
     - { role: borg_client, tags: ["borg"] }
     - { role: prometheus_exporters }
+    - { role: promtail }
     - { role: fail2ban }

playbooks/homedir.archlinux.org.yml

@@ -14,4 +14,5 @@
     - { role: public_html, public_domain: "pkgbuild.com", tags: ['nginx'] }
     - { role: borg_client, tags: ["borg"] }
     - { role: prometheus_exporters }
+    - { role: promtail }
     - { role: fail2ban }

playbooks/luna.yml

@@ -29,6 +29,7 @@
     - rspamd
     - { role: mariadb, mariadb_query_cache_type: '0', mariadb_innodb_file_per_table: true }
     - { role: prometheus_exporters }
+    - { role: promtail }
 # luna is hosting mailman lists; this postfix role does not cater to this yet
 # TODO: make postfix role handle mailman config?
 #    - { role: postfix, tags: ["postfix"], postfix_relayhost: "mail.archlinux.org" }

playbooks/mail.archlinux.org.yml

@@ -18,3 +18,4 @@
     - { role: archusers }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/man.archlinux.org.yml

@@ -14,6 +14,7 @@
     - { role: nginx }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }
     - { role: postgres }
     - { role: uwsgi }
     - { role: archmanweb, archmanweb_version: 'v1.1' }

playbooks/matrix.archlinux.org.yml

@@ -23,3 +23,4 @@
     - { role: matrix }
     - { role: fail2ban }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/md.archlinux.org.yml

@@ -19,3 +19,4 @@
       postgres_effective_cache_size: 1GB
     - { role: hedgedoc, hedgedoc_domain: "md.archlinux.org" }
     - { role: prometheus_exporters }
+    - { role: promtail }

playbooks/mirrors.yml

@@ -13,4 +13,5 @@
     - { role: archweb, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
     - { role: arch32_mirror, tags: ['nginx'] }
     - { role: prometheus_exporters }
+    - { role: promtail }
     - { role: fail2ban }

playbooks/monitoring.archlinux.org.yml

@@ -11,6 +11,8 @@
     - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
     - { role: prometheus }
     - { role: prometheus_exporters }
+    - { role: loki }
+    - { role: promtail }
     - { role: certbot }
     - { role: nginx }
     - { role: grafana, grafana_domain: 'monitoring.archlinux.org' }