From 7235e726d6abd6efb9531a4f65abd3c9663ac586 Mon Sep 17 00:00:00 2001
From: Kristian Klausen <kristian@klausen.dk>
Date: Fri, 26 Feb 2021 22:14:05 +0100
Subject: [PATCH] Implement centralized logging
Fix #263
---
docs/servers.md | 2 +-
playbooks/accounts.archlinux.org.yml | 1 +
playbooks/archive-mirrors.yml | 1 +
playbooks/archlinux.org.yml | 1 +
playbooks/aur-dev.archlinux.org.yml | 1 +
playbooks/aur.archlinux.org.yml | 1 +
playbooks/bbs.archlinux.org.yml | 1 +
playbooks/bugs.archlinux.org.yml | 1 +
playbooks/build.archlinux.org.yml | 1 +
playbooks/gemini.archlinux.org.yml | 1 +
playbooks/gitlab-runners.yml | 1 +
playbooks/gitlab.archlinux.org.yml | 1 +
playbooks/homedir.archlinux.org.yml | 1 +
playbooks/luna.yml | 1 +
playbooks/mail.archlinux.org.yml | 1 +
playbooks/man.archlinux.org.yml | 1 +
playbooks/matrix.archlinux.org.yml | 1 +
playbooks/md.archlinux.org.yml | 1 +
playbooks/mirrors.yml | 1 +
playbooks/monitoring.archlinux.org.yml | 2 +
playbooks/patchwork.archlinux.org.yml | 1 +
playbooks/phrik.yml | 1 +
playbooks/quassel.archlinux.org.yml | 1 +
playbooks/rebuilderd-workers.yml | 1 +
playbooks/redirect.archlinux.org.yml | 1 +
playbooks/reproducible.archlinux.org.yml | 1 +
playbooks/security.archlinux.org.yml | 1 +
playbooks/state.archlinux.org.yml | 1 +
playbooks/wiki.archlinux.org.yml | 1 +
roles/grafana/templates/datasources.yaml.j2 | 4 ++
roles/grafana/templates/nginx.d.conf.j2 | 7 +++
roles/loki/defaults/main.yml | 2 +
roles/loki/files/loki.yaml | 62 ++++++++++++++++++++
roles/loki/handlers/main.yml | 3 +
roles/loki/tasks/main.yml | 36 ++++++++++++
roles/loki/templates/nginx.d.conf.j2 | 45 ++++++++++++++
roles/prometheus/defaults/main.yml | 1 +
roles/prometheus/templates/prometheus.yml.j2 | 16 +++++
roles/promtail/defaults/main.yml | 1 +
roles/promtail/handlers/main.yml | 3 +
roles/promtail/tasks/main.yml | 16 +++++
roles/promtail/templates/promtail.yaml.j2 | 28 +++++++++
42 files changed, 254 insertions(+), 1 deletion(-)
create mode 100644 roles/loki/defaults/main.yml
create mode 100644 roles/loki/files/loki.yaml
create mode 100644 roles/loki/handlers/main.yml
create mode 100644 roles/loki/tasks/main.yml
create mode 100644 roles/loki/templates/nginx.d.conf.j2
create mode 100644 roles/promtail/defaults/main.yml
create mode 100644 roles/promtail/handlers/main.yml
create mode 100644 roles/promtail/tasks/main.yml
create mode 100644 roles/promtail/templates/promtail.yaml.j2
diff --git a/docs/servers.md b/docs/servers.md
index d4c16955c..23e082500 100644
--- a/docs/servers.md
+++ b/docs/servers.md
@@ -111,7 +111,7 @@ Medium-fast-ish packet.net Arch Linux box.
## monitoring.archlinux.org
- Prometheus and Grafana server which collects performance/metrics from our services and runs alertmanager.
+ Prometheus, Loki and Grafana server which collects performance/metrics and logs from our services and runs alertmanager.
### Services
- Alertmanager
diff --git a/playbooks/accounts.archlinux.org.yml b/playbooks/accounts.archlinux.org.yml
index 68848fd77..3eeb99245 100644
--- a/playbooks/accounts.archlinux.org.yml
+++ b/playbooks/accounts.archlinux.org.yml
@@ -20,3 +20,4 @@
- { role: keycloak }
- { role: borg_client, tags: ["borg"] }
- { role: fail2ban }
+ - { role: promtail }
diff --git a/playbooks/archive-mirrors.yml b/playbooks/archive-mirrors.yml
index a1ab954f8..c5df2995e 100644
--- a/playbooks/archive-mirrors.yml
+++ b/playbooks/archive-mirrors.yml
@@ -14,3 +14,4 @@
- { role: syncarchive }
- { role: archive_web }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/archlinux.org.yml b/playbooks/archlinux.org.yml
index d9b33e452..e51290b91 100644
--- a/playbooks/archlinux.org.yml
+++ b/playbooks/archlinux.org.yml
@@ -38,3 +38,4 @@
- { role: archweb, archweb_planet: true }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/aur-dev.archlinux.org.yml b/playbooks/aur-dev.archlinux.org.yml
index 558cda801..6c7cb1a05 100644
--- a/playbooks/aur-dev.archlinux.org.yml
+++ b/playbooks/aur-dev.archlinux.org.yml
@@ -20,3 +20,4 @@
- { role: fail2ban }
- { role: aurweb, aurweb_domain: 'aur-dev.archlinux.org', aurweb_version: 'pu' }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/aur.archlinux.org.yml b/playbooks/aur.archlinux.org.yml
index ea634ebea..d4f1e120d 100644
--- a/playbooks/aur.archlinux.org.yml
+++ b/playbooks/aur.archlinux.org.yml
@@ -9,6 +9,7 @@
- { role: sshd, sshd_enable_includes: true }
- { role: root_ssh }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: certbot }
- { role: nginx }
- { role: mariadb, mariadb_query_cache_type: '0', mariadb_innodb_file_per_table: true, mariadb_innodb_buffer_pool_size: '1G' }
diff --git a/playbooks/bbs.archlinux.org.yml b/playbooks/bbs.archlinux.org.yml
index fc4bc7125..a6e9cfec4 100644
--- a/playbooks/bbs.archlinux.org.yml
+++ b/playbooks/bbs.archlinux.org.yml
@@ -18,3 +18,4 @@
- { role: postfix, postfix_relayhost: "mail.archlinux.org" }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/bugs.archlinux.org.yml b/playbooks/bugs.archlinux.org.yml
index 999299eb0..ab90441db 100644
--- a/playbooks/bugs.archlinux.org.yml
+++ b/playbooks/bugs.archlinux.org.yml
@@ -18,3 +18,4 @@
- { role: postfix, postfix_relayhost: "mail.archlinux.org" }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/build.archlinux.org.yml b/playbooks/build.archlinux.org.yml
index 97cb6c6c0..01ff31445 100644
--- a/playbooks/build.archlinux.org.yml
+++ b/playbooks/build.archlinux.org.yml
@@ -14,3 +14,4 @@
- { role: archbuild }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/gemini.archlinux.org.yml b/playbooks/gemini.archlinux.org.yml
index f34939346..d64cd0637 100644
--- a/playbooks/gemini.archlinux.org.yml
+++ b/playbooks/gemini.archlinux.org.yml
@@ -26,3 +26,4 @@
- { role: postfix, postfix_relayhost: "mail.archlinux.org" }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/gitlab-runners.yml b/playbooks/gitlab-runners.yml
index 3980aed0b..65b088117 100644
--- a/playbooks/gitlab-runners.yml
+++ b/playbooks/gitlab-runners.yml
@@ -10,4 +10,5 @@
- { role: root_ssh }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: gitlab_runner }
diff --git a/playbooks/gitlab.archlinux.org.yml b/playbooks/gitlab.archlinux.org.yml
index 2ab4cc23d..83b5b07c1 100644
--- a/playbooks/gitlab.archlinux.org.yml
+++ b/playbooks/gitlab.archlinux.org.yml
@@ -16,4 +16,5 @@
gitlab_pages_https_addresses: ['116.203.6.156:443', '[2a01:4f8:c2c:5d2d::2]:443']}
- { role: borg_client, tags: ["borg"] }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: fail2ban }
diff --git a/playbooks/homedir.archlinux.org.yml b/playbooks/homedir.archlinux.org.yml
index aca254469..ec38949d8 100644
--- a/playbooks/homedir.archlinux.org.yml
+++ b/playbooks/homedir.archlinux.org.yml
@@ -14,4 +14,5 @@
- { role: public_html, public_domain: "pkgbuild.com", tags: ['nginx'] }
- { role: borg_client, tags: ["borg"] }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: fail2ban }
diff --git a/playbooks/luna.yml b/playbooks/luna.yml
index 0e00327b2..81b5f4084 100644
--- a/playbooks/luna.yml
+++ b/playbooks/luna.yml
@@ -29,6 +29,7 @@
- rspamd
- { role: mariadb, mariadb_query_cache_type: '0', mariadb_innodb_file_per_table: true }
- { role: prometheus_exporters }
+ - { role: promtail }
# luna is hosting mailman lists; this postfix role does not cater to this yet
# TODO: make postfix role handle mailman config?
# - { role: postfix, tags: ["postfix"], postfix_relayhost: "mail.archlinux.org" }
diff --git a/playbooks/mail.archlinux.org.yml b/playbooks/mail.archlinux.org.yml
index 70272765d..7cc301f0c 100644
--- a/playbooks/mail.archlinux.org.yml
+++ b/playbooks/mail.archlinux.org.yml
@@ -18,3 +18,4 @@
- { role: archusers }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/man.archlinux.org.yml b/playbooks/man.archlinux.org.yml
index 2f3ca9bfa..88cb01a43 100644
--- a/playbooks/man.archlinux.org.yml
+++ b/playbooks/man.archlinux.org.yml
@@ -14,6 +14,7 @@
- { role: nginx }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: postgres }
- { role: uwsgi }
- { role: archmanweb, archmanweb_version: 'v1.1' }
diff --git a/playbooks/matrix.archlinux.org.yml b/playbooks/matrix.archlinux.org.yml
index 9b2c8e890..5c5ce6dc5 100644
--- a/playbooks/matrix.archlinux.org.yml
+++ b/playbooks/matrix.archlinux.org.yml
@@ -23,3 +23,4 @@
- { role: matrix }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/md.archlinux.org.yml b/playbooks/md.archlinux.org.yml
index c9a8ee658..272853a9f 100644
--- a/playbooks/md.archlinux.org.yml
+++ b/playbooks/md.archlinux.org.yml
@@ -19,3 +19,4 @@
postgres_effective_cache_size: 1GB
- { role: hedgedoc, hedgedoc_domain: "md.archlinux.org" }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/mirrors.yml b/playbooks/mirrors.yml
index 15feca053..0f13391c1 100644
--- a/playbooks/mirrors.yml
+++ b/playbooks/mirrors.yml
@@ -13,4 +13,5 @@
- { role: archweb, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
- { role: arch32_mirror, tags: ['nginx'] }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: fail2ban }
diff --git a/playbooks/monitoring.archlinux.org.yml b/playbooks/monitoring.archlinux.org.yml
index 997296ce5..53cc030f2 100644
--- a/playbooks/monitoring.archlinux.org.yml
+++ b/playbooks/monitoring.archlinux.org.yml
@@ -11,6 +11,8 @@
- { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
- { role: prometheus }
- { role: prometheus_exporters }
+ - { role: loki }
+ - { role: promtail }
- { role: certbot }
- { role: nginx }
- { role: grafana, grafana_domain: 'monitoring.archlinux.org' }
diff --git a/playbooks/patchwork.archlinux.org.yml b/playbooks/patchwork.archlinux.org.yml
index d9bd76b9f..00fcd84d8 100644
--- a/playbooks/patchwork.archlinux.org.yml
+++ b/playbooks/patchwork.archlinux.org.yml
@@ -20,3 +20,4 @@
- { role: patchwork }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/phrik.yml b/playbooks/phrik.yml
index c0c3e81fb..88d02ea23 100644
--- a/playbooks/phrik.yml
+++ b/playbooks/phrik.yml
@@ -12,3 +12,4 @@
- { role: sshd }
- { role: root_ssh }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/quassel.archlinux.org.yml b/playbooks/quassel.archlinux.org.yml
index bd2b473f6..b423b1469 100644
--- a/playbooks/quassel.archlinux.org.yml
+++ b/playbooks/quassel.archlinux.org.yml
@@ -18,3 +18,4 @@
postgres_effective_cache_size: 1GB
- { role: quassel, quassel_domain: "quassel.archlinux.org" }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/rebuilderd-workers.yml b/playbooks/rebuilderd-workers.yml
index a14b19fce..ea5e6b46f 100644
--- a/playbooks/rebuilderd-workers.yml
+++ b/playbooks/rebuilderd-workers.yml
@@ -11,4 +11,5 @@
- { role: root_ssh }
- { role: rebuilderd_worker }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: fail2ban }
diff --git a/playbooks/redirect.archlinux.org.yml b/playbooks/redirect.archlinux.org.yml
index 5cf96d694..29a5ac4f0 100644
--- a/playbooks/redirect.archlinux.org.yml
+++ b/playbooks/redirect.archlinux.org.yml
@@ -11,5 +11,6 @@
- { role: nginx }
- { role: redirects }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: hardening }
- { role: ping }
diff --git a/playbooks/reproducible.archlinux.org.yml b/playbooks/reproducible.archlinux.org.yml
index 2d2674aaa..2405582a4 100644
--- a/playbooks/reproducible.archlinux.org.yml
+++ b/playbooks/reproducible.archlinux.org.yml
@@ -14,4 +14,5 @@
- { role: nginx }
- { role: rebuilderd }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: fail2ban }
diff --git a/playbooks/security.archlinux.org.yml b/playbooks/security.archlinux.org.yml
index 30a57e268..782265fcf 100644
--- a/playbooks/security.archlinux.org.yml
+++ b/playbooks/security.archlinux.org.yml
@@ -20,3 +20,4 @@
security_tracker_dir: "/srv/http/security-tracker"
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/playbooks/state.archlinux.org.yml b/playbooks/state.archlinux.org.yml
index 7d6e479c3..e2a722f9b 100644
--- a/playbooks/state.archlinux.org.yml
+++ b/playbooks/state.archlinux.org.yml
@@ -20,4 +20,5 @@
postgres_ssl_hosts6: ['::/0']
- { role: terraform_state }
- { role: prometheus_exporters }
+ - { role: promtail }
- { role: fail2ban }
diff --git a/playbooks/wiki.archlinux.org.yml b/playbooks/wiki.archlinux.org.yml
index 56bd59b28..e8eb3950a 100644
--- a/playbooks/wiki.archlinux.org.yml
+++ b/playbooks/wiki.archlinux.org.yml
@@ -19,3 +19,4 @@
- { role: archwiki }
- { role: fail2ban }
- { role: prometheus_exporters }
+ - { role: promtail }
diff --git a/roles/grafana/templates/datasources.yaml.j2 b/roles/grafana/templates/datasources.yaml.j2
index 4088d0a19..81e2ee4b2 100644
--- a/roles/grafana/templates/datasources.yaml.j2
+++ b/roles/grafana/templates/datasources.yaml.j2
@@ -5,4 +5,8 @@ datasources:
type: prometheus
access: proxy
url: http://localhost:9090
+- name: Loki
+ type: loki
+ access: proxy
+ url: http://localhost:3100
diff --git a/roles/grafana/templates/nginx.d.conf.j2 b/roles/grafana/templates/nginx.d.conf.j2
index db191a4d6..e3018cbf7 100644
--- a/roles/grafana/templates/nginx.d.conf.j2
+++ b/roles/grafana/templates/nginx.d.conf.j2
@@ -2,6 +2,11 @@ upstream grafana {
server localhost:3000;
}
+map $http_upgrade $connection_upgrade {
+ default upgrade;
+ '' close;
+}
+
server {
listen 80;
listen [::]:80;
@@ -36,5 +41,7 @@ server {
access_log /var/log/nginx/{{ grafana_domain }}/access.log main;
proxy_pass http://grafana;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection $connection_upgrade;
}
}
diff --git a/roles/loki/defaults/main.yml b/roles/loki/defaults/main.yml
new file mode 100644
index 000000000..e8b17f135
--- /dev/null
+++ b/roles/loki/defaults/main.yml
@@ -0,0 +1,2 @@
+logging_domain: logging.archlinux.org
+loki_nginx_htpasswd: /etc/nginx/auth/loki
diff --git a/roles/loki/files/loki.yaml b/roles/loki/files/loki.yaml
new file mode 100644
index 000000000..93546a828
--- /dev/null
+++ b/roles/loki/files/loki.yaml
@@ -0,0 +1,62 @@
+# Enables authentication through the X-Scope-OrgID header, which must be present
+# if true. If false, the OrgID will always be set to "fake".
+auth_enabled: false
+
+server:
+ http_listen_address: 127.0.0.1
+ http_listen_port: 3100
+ grpc_listen_address: 127.0.0.1
+ grpc_listen_port: 9095
+
+ingester:
+ wal:
+ enabled: true
+ dir: /var/lib/loki/wal
+ replay_memory_ceiling: 200MB
+ lifecycler:
+ address: 127.0.0.1
+ ring:
+ kvstore:
+ store: inmemory
+ replication_factor: 1
+ final_sleep: 0s
+ chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
+ max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
+ chunk_target_size: 1536000 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+ chunk_encoding: zstd
+ chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+ max_transfer_retries: 0 # Chunk transfers disabled
+
+schema_config:
+ configs:
+ - from: 2020-10-24
+ store: boltdb-shipper
+ object_store: filesystem
+ schema: v11
+ index:
+ prefix: index_
+ period: 24h
+
+storage_config:
+ boltdb_shipper:
+ active_index_directory: /var/lib/loki/boltdb-shipper-active
+ cache_location: /var/lib/loki/boltdb-shipper-cache
+ cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
+ shared_store: filesystem
+ filesystem:
+ directory: /var/lib/loki/chunks
+
+compactor:
+ working_directory: /var/lib/loki/boltdb-shipper-compactor
+ shared_store: filesystem
+
+limits_config:
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h
+
+chunk_store_config:
+ max_look_back_period: 672h
+
+table_manager:
+ retention_deletes_enabled: true
+ retention_period: 672h # 28 days
diff --git a/roles/loki/handlers/main.yml b/roles/loki/handlers/main.yml
new file mode 100644
index 000000000..d87b9ce65
--- /dev/null
+++ b/roles/loki/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: restart loki
+ service: name=loki state=restarted
diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml
new file mode 100644
index 000000000..e303a2b1d
--- /dev/null
+++ b/roles/loki/tasks/main.yml
@@ -0,0 +1,36 @@
+---
+- name: create ssl cert
+ include_role:
+ name: certificate
+ vars:
+ domains: ["{{ logging_domain }}"]
+
+- name: install loki and logcli
+ pacman: name=loki,logcli state=present
+
+- name: install loki configuration
+ copy: src=loki.yaml dest=/etc/loki/ owner=root group=root mode=0644
+ notify: restart loki
+
+- name: install python-passlib
+ pacman: name=python-passlib
+
+- name: create htpasswd for nginx loki endpoint
+ htpasswd:
+ path: "{{ loki_nginx_htpasswd }}"
+ name: "{{ vault_loki_nginx_user }}"
+ password: "{{ vault_loki_nginx_passwd }}"
+ owner: root
+ group: http
+ mode: 0640
+
+- name: make nginx log dir
+ file: path=/var/log/nginx/{{ logging_domain }} state=directory owner=root group=root mode=0755
+
+- name: set up nginx
+ template: src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/logging.conf" owner=root group=root mode=644
+ notify: reload nginx
+ tags: ['nginx']
+
+- name: start and enable loki
+ systemd: name=loki.service enabled=yes daemon_reload=yes state=started
diff --git a/roles/loki/templates/nginx.d.conf.j2 b/roles/loki/templates/nginx.d.conf.j2
new file mode 100644
index 000000000..4269815d7
--- /dev/null
+++ b/roles/loki/templates/nginx.d.conf.j2
@@ -0,0 +1,45 @@
+server {
+ listen 80;
+ listen [::]:80;
+ server_name {{ logging_domain }};
+
+ access_log /var/log/nginx/{{ logging_domain }}/access.log main;
+ error_log /var/log/nginx/{{ logging_domain }}/error.log;
+
+ include snippets/letsencrypt.conf;
+
+ location / {
+ access_log off;
+ return 301 https://$server_name$request_uri;
+ }
+}
+
+server {
+ listen 443 ssl http2;
+ listen [::]:443 ssl http2;
+ server_name {{ logging_domain }};
+
+ access_log /var/log/nginx/{{ logging_domain }}/access.log main;
+ error_log /var/log/nginx/{{ logging_domain }}/error.log;
+
+ ssl_certificate /etc/letsencrypt/live/{{ logging_domain }}/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/{{ logging_domain }}/privkey.pem;
+ ssl_trusted_certificate /etc/letsencrypt/live/{{ logging_domain }}/chain.pem;
+
+ location = /loki/api/v1/push {
+ auth_basic "Loki :)";
+ auth_basic_user_file {{ loki_nginx_htpasswd }};
+
+ proxy_pass http://127.0.0.1:3100$request_uri;
+ }
+
+ # We return a 200 so the monitoring is happy!
+ location = / {
+ default_type text/plain;
+ return 200 'Nothing to see here..\n';
+ }
+
+ location / {
+ return 404;
+ }
+}
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
index 38a997f4a..c4f187d5e 100644
--- a/roles/prometheus/defaults/main.yml
+++ b/roles/prometheus/defaults/main.yml
@@ -23,6 +23,7 @@ blackbox_targets:
- https://git.archlinux.org
- https://gitlab.archlinux.org
- https://ipxe.archlinux.org
+ - https://logging.archlinux.org
- https://lists.archlinux.org
- https://mailman.archlinux.org
- https://man.archlinux.org
diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2
index 585a5b7d5..03e36e7ba 100644
--- a/roles/prometheus/templates/prometheus.yml.j2
+++ b/roles/prometheus/templates/prometheus.yml.j2
@@ -13,6 +13,12 @@ alerting:
- localhost:9093
scrape_configs:
+ - job_name: loki
+ static_configs:
+ - targets: ['127.0.0.1:3100']
+ labels:
+ instance: "{{ ansible_fqdn }}"
+
- job_name: 'node_exporter'
static_configs:
{% for host in groups['node_exporters'] %}
@@ -23,6 +29,16 @@ scrape_configs:
{% endfor %}
+ - job_name: 'promtail'
+ static_configs:
+ {% for host in groups['node_exporters'] %}
+
+ - targets: ['{{ host }}:9080']
+ labels:
+ instance: "{{ host }}"
+
+ {% endfor %}
+
- job_name: 'gitlab_runner_exporter'
static_configs:
{% for host in groups['gitlab_runners'] %}
diff --git a/roles/promtail/defaults/main.yml b/roles/promtail/defaults/main.yml
new file mode 100644
index 000000000..57bb5d15f
--- /dev/null
+++ b/roles/promtail/defaults/main.yml
@@ -0,0 +1 @@
+logging_domain: logging.archlinux.org
diff --git a/roles/promtail/handlers/main.yml b/roles/promtail/handlers/main.yml
new file mode 100644
index 000000000..6ea6ab6f7
--- /dev/null
+++ b/roles/promtail/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: restart promtail
+ service: name=promtail state=restarted
diff --git a/roles/promtail/tasks/main.yml b/roles/promtail/tasks/main.yml
new file mode 100644
index 000000000..b9f73da79
--- /dev/null
+++ b/roles/promtail/tasks/main.yml
@@ -0,0 +1,16 @@
+---
+- name: install promtail
+ pacman: name=promtail state=present
+
+- name: install promtail configuration
+ template: src=promtail.yaml.j2 dest=/etc/loki/promtail.yaml owner=root group=promtail mode=0640
+ notify: restart promtail
+
+- name: open promtail ipv4 port for monitoring.archlinux.org
+ ansible.posix.firewalld: state=enabled permanent=true immediate=yes
+ rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port=9080 accept"
+ tags:
+ - firewall
+
+- name: start and enable promtail
+ systemd: name=promtail.service enabled=yes daemon_reload=yes state=started
diff --git a/roles/promtail/templates/promtail.yaml.j2 b/roles/promtail/templates/promtail.yaml.j2
new file mode 100644
index 000000000..d3ac8fd36
--- /dev/null
+++ b/roles/promtail/templates/promtail.yaml.j2
@@ -0,0 +1,28 @@
+server:
+ http_listen_address: 127.0.0.1
+ http_listen_port: 9080
+ grpc_listen_address: 127.0.0.1
+ grpc_listen_port: 0 # 0 means random
+
+positions:
+ filename: /var/lib/promtail/positions.yaml
+
+clients:
+ - url: https://{{ logging_domain }}/loki/api/v1/push
+ basic_auth:
+ username: '{{ vault_loki_nginx_user }}'
+ password: '{{ vault_loki_nginx_passwd }}'
+
+scrape_configs:
+ - job_name: journal
+ journal:
+ json: true
+ max_age: 72h
+ path: /var/log/journal
+ labels:
+ job: systemd-journal
+ relabel_configs:
+ - source_labels: ['__journal__systemd_unit']
+ target_label: unit
+ - source_labels: ["__journal__hostname"]
+ target_label: instance
--
GitLab