Send promtail logs and scrap its metrics over WireGuard

roles/loki/defaults/main.yml

-logging_domain: logging.archlinux.org
 loki_nginx_htpasswd: /etc/nginx/auth/loki

roles/loki/tasks/main.yml

 ---
-- name: create ssl cert
-  include_role:
-    name: certificate
-  vars:
-    domains: ["{{ logging_domain }}"]
-
 - name: install loki and logcli
   pacman: name=loki,logcli state=present
 
@@ -25,12 +19,15 @@
     mode: 0640
 
 - name: make nginx log dir
-  file: path=/var/log/nginx/{{ logging_domain }} state=directory owner=root group=root mode=0755
+  file: path=/var/log/nginx/loki state=directory owner=root group=root mode=0755
 
 - name: set up nginx
   template: src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/logging.conf" owner=root group=root mode=644
   notify: reload nginx
   tags: ['nginx']
 
+- name: open firewall hole
+  ansible.posix.firewalld: service=http zone=wireguard permanent=true state=enabled immediate=yes
+
 - name: start and enable loki
   systemd: name=loki.service enabled=yes daemon_reload=yes state=started

roles/loki/templates/nginx.d.conf.j2

-server {
-    listen       80;
-    listen       [::]:80;
-    server_name  {{ logging_domain }};
-
-    access_log   /var/log/nginx/{{ logging_domain }}/access.log main;
-    access_log   /var/log/nginx/{{ logging_domain }}/access.log.json json_main;
-    error_log    /var/log/nginx/{{ logging_domain }}/error.log;
-
-    include snippets/letsencrypt.conf;
-
-    location / {
-        access_log off;
-        return 301 https://$server_name$request_uri;
-    }
-}
-
 # We don't want to log (/loki/api/v1/push) request from yourself as it would cause a infinite loop
 map $remote_addr $loggable {
-    {{ ansible_default_ipv4.address }} 0;
-    {{ ansible_default_ipv6.address }} 0;
+    {{ wireguard_address }} 0;
     default 1;
 }
 
 server {
-    listen       443 ssl http2;
-    listen       [::]:443 ssl http2;
-    server_name  {{ logging_domain }};
-
-    access_log   /var/log/nginx/{{ logging_domain }}/access.log main;
-    access_log   /var/log/nginx/{{ logging_domain }}/access.log.json json_main if=$loggable;
-    error_log    /var/log/nginx/{{ logging_domain }}/error.log;
+    listen       {{ wireguard_address }}:80;
 
-    ssl_certificate      /etc/letsencrypt/live/{{ logging_domain }}/fullchain.pem;
-    ssl_certificate_key  /etc/letsencrypt/live/{{ logging_domain }}/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ logging_domain }}/chain.pem;
+    access_log   /var/log/nginx/loki/access.log main;
+    access_log   /var/log/nginx/loki/access.log.json json_main if=$loggable;
+    error_log    /var/log/nginx/loki/error.log;
 
     location = /loki/api/v1/push {
         auth_basic "Loki :)";

roles/prometheus/defaults/main.yml

@@ -7,6 +7,7 @@ prometheus_receive_only: false
 # for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/  - https:\/\//"
 blackbox_targets:
   http_prometheus:
+    - http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
     - https://accounts.archlinux.org
     - https://america.archive.pkgbuild.com
     - https://america.mirror.pkgbuild.com
@@ -25,7 +26,6 @@ blackbox_targets:
     - https://europe.mirror.pkgbuild.com
     - https://gitlab.archlinux.org
     - https://ipxe.archlinux.org
-    - https://logging.archlinux.org
     - https://lists.archlinux.org
     - https://mailman.archlinux.org
     - https://man.archlinux.org

roles/prometheus/templates/prometheus.yml.j2

@@ -61,7 +61,7 @@ scrape_configs:
     static_configs:
     {% for host in groups['node_exporters'] %}
 
-    - targets: ['{{ host }}:9080']
+    - targets: ['{{ hostvars[host]['wireguard_address'] }}:9080']
       labels:
         instance: "{{ host }}"
 

roles/promtail/defaults/main.yml

-logging_domain: logging.archlinux.org

roles/promtail/tasks/main.yml

@@ -7,8 +7,8 @@
   notify: restart promtail
 
 - name: open promtail ipv4 port for monitoring.archlinux.org
-  ansible.posix.firewalld: state=enabled permanent=true immediate=yes
-    rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port=9080 accept"
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=9080 accept"
   tags:
     - firewall
 

roles/promtail/templates/promtail.yaml.j2

 server:
-  http_listen_address: 0.0.0.0
+  http_listen_address: {{ wireguard_address }}
   http_listen_port: 9080
   grpc_listen_address: 127.0.0.1
   grpc_listen_port: 0 # 0 means random
@@ -8,7 +8,7 @@ positions:
   filename: /var/lib/promtail/positions.yaml
 
 clients:
-  - url: https://{{ logging_domain }}/loki/api/v1/push
+  - url: http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}/loki/api/v1/push
     basic_auth:
       username: '{{ vault_loki_nginx_user }}'
       password: '{{ vault_loki_nginx_passwd }}'

tf-stage1/archlinux.tf

@@ -285,7 +285,6 @@ locals {
     rsync         = { value = "gemini" }
     sources       = { value = "gemini" }
     "static.conf" = { value = "redirect" }
-    logging       = { value = "monitoring" }
     status        = { value = "stats.uptimerobot.com." }
     svn           = { value = "gemini" }
     coc           = { value = "redirect" }