Commit 03276398 authored by Kristian Klausen's avatar Kristian Klausen 🎉
Browse files

Send promtail logs and scrap its metrics over WireGuard

parent 1d5e02d3
logging_domain: logging.archlinux.org
loki_nginx_htpasswd: /etc/nginx/auth/loki
---
- name: create ssl cert
include_role:
name: certificate
vars:
domains: ["{{ logging_domain }}"]
- name: install loki and logcli
pacman: name=loki,logcli state=present
......@@ -25,12 +19,15 @@
mode: 0640
- name: make nginx log dir
file: path=/var/log/nginx/{{ logging_domain }} state=directory owner=root group=root mode=0755
file: path=/var/log/nginx/loki state=directory owner=root group=root mode=0755
- name: set up nginx
template: src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/logging.conf" owner=root group=root mode=644
notify: reload nginx
tags: ['nginx']
- name: open firewall hole
ansible.posix.firewalld: service=http zone=wireguard permanent=true state=enabled immediate=yes
- name: start and enable loki
systemd: name=loki.service enabled=yes daemon_reload=yes state=started
server {
listen 80;
listen [::]:80;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
access_log /var/log/nginx/{{ logging_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
include snippets/letsencrypt.conf;
location / {
access_log off;
return 301 https://$server_name$request_uri;
}
}
# We don't want to log (/loki/api/v1/push) request from yourself as it would cause a infinite loop
map $remote_addr $loggable {
{{ ansible_default_ipv4.address }} 0;
{{ ansible_default_ipv6.address }} 0;
{{ wireguard_address }} 0;
default 1;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
access_log /var/log/nginx/{{ logging_domain }}/access.log.json json_main if=$loggable;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
listen {{ wireguard_address }}:80;
ssl_certificate /etc/letsencrypt/live/{{ logging_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ logging_domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ logging_domain }}/chain.pem;
access_log /var/log/nginx/loki/access.log main;
access_log /var/log/nginx/loki/access.log.json json_main if=$loggable;
error_log /var/log/nginx/loki/error.log;
location = /loki/api/v1/push {
auth_basic "Loki :)";
......
......@@ -7,6 +7,7 @@ prometheus_receive_only: false
# for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/ - https:\/\//"
blackbox_targets:
http_prometheus:
- http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
- https://accounts.archlinux.org
- https://america.archive.pkgbuild.com
- https://america.mirror.pkgbuild.com
......@@ -25,7 +26,6 @@ blackbox_targets:
- https://europe.mirror.pkgbuild.com
- https://gitlab.archlinux.org
- https://ipxe.archlinux.org
- https://logging.archlinux.org
- https://lists.archlinux.org
- https://mailman.archlinux.org
- https://man.archlinux.org
......
......@@ -61,7 +61,7 @@ scrape_configs:
static_configs:
{% for host in groups['node_exporters'] %}
- targets: ['{{ host }}:9080']
- targets: ['{{ hostvars[host]['wireguard_address'] }}:9080']
labels:
instance: "{{ host }}"
......
logging_domain: logging.archlinux.org
......@@ -7,8 +7,8 @@
notify: restart promtail
- name: open promtail ipv4 port for monitoring.archlinux.org
ansible.posix.firewalld: state=enabled permanent=true immediate=yes
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port=9080 accept"
ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=9080 accept"
tags:
- firewall
......
server:
http_listen_address: 0.0.0.0
http_listen_address: {{ wireguard_address }}
http_listen_port: 9080
grpc_listen_address: 127.0.0.1
grpc_listen_port: 0 # 0 means random
......@@ -8,7 +8,7 @@ positions:
filename: /var/lib/promtail/positions.yaml
clients:
- url: https://{{ logging_domain }}/loki/api/v1/push
- url: http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}/loki/api/v1/push
basic_auth:
username: '{{ vault_loki_nginx_user }}'
password: '{{ vault_loki_nginx_passwd }}'
......
......@@ -285,7 +285,6 @@ locals {
rsync = { value = "gemini" }
sources = { value = "gemini" }
"static.conf" = { value = "redirect" }
logging = { value = "monitoring" }
status = { value = "stats.uptimerobot.com." }
svn = { value = "gemini" }
coc = { value = "redirect" }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment