Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Arch Linux
infrastructure
Commits
03276398
Commit
03276398
authored
Jul 07, 2021
by
Kristian Klausen
🎉
Browse files
Send promtail logs and scrap its metrics over WireGuard
parent
1d5e02d3
Changes
9
Hide whitespace changes
Inline
Side-by-side
roles/loki/defaults/main.yml
View file @
03276398
logging_domain
:
logging.archlinux.org
loki_nginx_htpasswd
:
/etc/nginx/auth/loki
roles/loki/tasks/main.yml
View file @
03276398
---
-
name
:
create ssl cert
include_role
:
name
:
certificate
vars
:
domains
:
[
"
{{
logging_domain
}}"
]
-
name
:
install loki and logcli
pacman
:
name=loki,logcli state=present
...
...
@@ -25,12 +19,15 @@
mode
:
0640
-
name
:
make nginx log dir
file
:
path=/var/log/nginx/
{{ logging_domain }}
state=directory owner=root group=root mode=0755
file
:
path=/var/log/nginx/
loki
state=directory owner=root group=root mode=0755
-
name
:
set up nginx
template
:
src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/logging.conf" owner=root group=root mode=644
notify
:
reload nginx
tags
:
[
'
nginx'
]
-
name
:
open firewall hole
ansible.posix.firewalld
:
service=http zone=wireguard permanent=true state=enabled immediate=yes
-
name
:
start and enable loki
systemd
:
name=loki.service enabled=yes daemon_reload=yes state=started
roles/loki/templates/nginx.d.conf.j2
View file @
03276398
server {
listen 80;
listen [::]:80;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
access_log /var/log/nginx/{{ logging_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
include snippets/letsencrypt.conf;
location / {
access_log off;
return 301 https://$server_name$request_uri;
}
}
# We don't want to log (/loki/api/v1/push) request from yourself as it would cause a infinite loop
map $remote_addr $loggable {
{{ ansible_default_ipv4.address }} 0;
{{ ansible_default_ipv6.address }} 0;
{{ wireguard_address }} 0;
default 1;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
access_log /var/log/nginx/{{ logging_domain }}/access.log.json json_main if=$loggable;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
listen {{ wireguard_address }}:80;
ssl_certificate /etc/letsencrypt/live/{{ logging_domain }}/fullchain.pem
;
ssl_certificate_key /etc/letsencrypt/live/{{ logging_domain }}/privkey.pem
;
ssl_trusted_certificate /etc/letsencrypt/live/{{
loggin
g_domain }}/chain.pem
;
access_log /var/log/nginx/loki/access.log main
;
access_log /var/log/nginx/loki/access.log.json json_main if=$loggable
;
error_log /var/
log
/n
gin
x/loki/error.log
;
location = /loki/api/v1/push {
auth_basic "Loki :)";
...
...
roles/prometheus/defaults/main.yml
View file @
03276398
...
...
@@ -7,6 +7,7 @@ prometheus_receive_only: false
# for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/ - https:\/\//"
blackbox_targets
:
http_prometheus
:
-
http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
-
https://accounts.archlinux.org
-
https://america.archive.pkgbuild.com
-
https://america.mirror.pkgbuild.com
...
...
@@ -25,7 +26,6 @@ blackbox_targets:
-
https://europe.mirror.pkgbuild.com
-
https://gitlab.archlinux.org
-
https://ipxe.archlinux.org
-
https://logging.archlinux.org
-
https://lists.archlinux.org
-
https://mailman.archlinux.org
-
https://man.archlinux.org
...
...
roles/prometheus/templates/prometheus.yml.j2
View file @
03276398
...
...
@@ -61,7 +61,7 @@ scrape_configs:
static_configs:
{% for host in groups['node_exporters'] %}
- targets: ['{{ host }}:9080']
- targets: ['{{ host
vars[host]['wireguard_address']
}}:9080']
labels:
instance: "{{ host }}"
...
...
roles/promtail/defaults/main.yml
deleted
100644 → 0
View file @
1d5e02d3
logging_domain
:
logging.archlinux.org
roles/promtail/tasks/main.yml
View file @
03276398
...
...
@@ -7,8 +7,8 @@
notify
:
restart promtail
-
name
:
open promtail ipv4 port for monitoring.archlinux.org
ansible.posix.firewalld
:
state=enabled permanent=true immediate=yes
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['
ipv4
_address'] }} port protocol=tcp port=9080 accept"
ansible.posix.firewalld
:
zone=wireguard
state=enabled permanent=true immediate=yes
rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['
wireguard
_address'] }} port protocol=tcp port=9080 accept"
tags
:
-
firewall
...
...
roles/promtail/templates/promtail.yaml.j2
View file @
03276398
server:
http_listen_address:
0.0.0.0
http_listen_address:
{{ wireguard_address }}
http_listen_port: 9080
grpc_listen_address: 127.0.0.1
grpc_listen_port: 0 # 0 means random
...
...
@@ -8,7 +8,7 @@ positions:
filename: /var/lib/promtail/positions.yaml
clients:
- url: http
s
://{{
logging_domain
}}/loki/api/v1/push
- url: http://{{
hostvars['monitoring.archlinux.org']['wireguard_address']
}}/loki/api/v1/push
basic_auth:
username: '{{ vault_loki_nginx_user }}'
password: '{{ vault_loki_nginx_passwd }}'
...
...
tf-stage1/archlinux.tf
View file @
03276398
...
...
@@ -285,7 +285,6 @@ locals {
rsync
=
{
value
=
"gemini"
}
sources
=
{
value
=
"gemini"
}
"static.conf"
=
{
value
=
"redirect"
}
logging
=
{
value
=
"monitoring"
}
status
=
{
value
=
"stats.uptimerobot.com."
}
svn
=
{
value
=
"gemini"
}
coc
=
{
value
=
"redirect"
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment