Verified Commit b70d04fa authored by Kristian Klausen's avatar Kristian Klausen 🎉 Committed by Jelle van der Waa
Browse files

Send the nginx logs to Loki

A extra access_log entry was added with the following commands:
$ cd roles
$ grep -lr access_log | xargs -P 1 -n 1 sed -i '/access_log/ s/\(.*\)\( \)\(\(reduced\|main\);$\)/\1 \3\n\1.json json_\3/'
parent 7235e726
......@@ -7,6 +7,7 @@ server {
root {{ arch32_mirror_dir }};
access_log /var/log/nginx/{{ arch32_mirror_domain }}/access.log reduced;
access_log /var/log/nginx/{{ arch32_mirror_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ arch32_mirror_domain }}/error.log;
include snippets/letsencrypt.conf;
......
......@@ -4,6 +4,7 @@ server {
server_name {{ archive_domain }};
access_log /var/log/nginx/{{ archive_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archive_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archive_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -20,6 +21,7 @@ server {
server_name {{ archive_domain }};
access_log /var/log/nginx/{{ archive_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archive_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archive_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ archive_domain }}/fullchain.pem;
......
......@@ -8,6 +8,7 @@ server {
server_name {{ archmanweb_domain }};
access_log /var/log/nginx/{{ archmanweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archmanweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archmanweb_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -24,6 +25,7 @@ server {
server_name {{ archmanweb_domain }};
access_log /var/log/nginx/{{ archmanweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archmanweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archmanweb_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ archmanweb_domain }}/fullchain.pem;
......@@ -44,6 +46,7 @@ server {
location / {
access_log /var/log/nginx/{{ archmanweb_domain }}/access.log main;
access_log /var/log/nginx/{{ archmanweb_domain }}/access.log.json json_main;
include uwsgi_params;
uwsgi_pass archmanweb;
}
......
......@@ -4,6 +4,7 @@ server {
server_name {{ domain['domain_name'] }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -20,6 +21,7 @@ server {
server_name {{ domain['domain_name'] }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
ssl_ciphers AES128-SHA:AES256-SHA:AES128-SHA256:AES256-SHA256;
......@@ -30,6 +32,7 @@ server {
location /releng/netboot {
access_log /var/log/nginx/{{ archweb_domain }}/access.log main;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_main;
include uwsgi_params;
uwsgi_pass archweb;
}
......
......@@ -11,6 +11,7 @@ server {
server_name {{ domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
include snippets/letsencrypt.conf;
......@@ -27,6 +28,7 @@ server {
server_name {{ domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
ssl_certificate /etc/letsencrypt/live/{{ service_domain }}/fullchain.pem;
......@@ -50,6 +52,7 @@ server {
server_name {{ service_domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
include snippets/letsencrypt.conf;
......@@ -66,6 +69,7 @@ server {
server_name {{ service_domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
ssl_certificate /etc/letsencrypt/live/{{ service_domain }}/fullchain.pem;
......@@ -150,6 +154,7 @@ server {
if ($maintenance_remote_machine = true) {
access_log /var/log/nginx/{{ archweb_domain }}/access.log main;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_main;
uwsgi_pass archweb;
break;
}
......
......@@ -24,6 +24,7 @@ server {
server_name {{ domain['domain'] }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -45,6 +46,7 @@ server {
server_name {{ domain['domain'] }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ archweb_domain }}/fullchain.pem;
......@@ -73,6 +75,7 @@ server {
server_name {{ archweb_domain }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -89,6 +92,7 @@ server {
server_name {{ archweb_domain }};
access_log /var/log/nginx/{{ archweb_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archweb_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ archweb_domain }}/fullchain.pem;
......@@ -176,6 +180,7 @@ server {
location / {
access_log /var/log/nginx/{{ archweb_domain }}/access.log main;
access_log /var/log/nginx/{{ archweb_domain }}/access.log.json json_main;
include uwsgi_params;
uwsgi_pass archweb;
......
......@@ -11,6 +11,7 @@ server {
server_name {{ archwiki_domain }};
access_log /var/log/nginx/{{ archwiki_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archwiki_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archwiki_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -27,6 +28,7 @@ server {
server_name {{ archwiki_domain }};
access_log /var/log/nginx/{{ archwiki_domain }}/access.log reduced;
access_log /var/log/nginx/{{ archwiki_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ archwiki_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ archwiki_domain }}/fullchain.pem;
......@@ -53,6 +55,7 @@ server {
# special case due to our '/index.php/Main_Page' type URLs
location ~ ^/(?:index|redirect)\.php(?:/.*)$ {
access_log /var/log/nginx/{{ archwiki_domain }}/access.log main;
access_log /var/log/nginx/{{ archwiki_domain }}/access.log.json json_main;
fastcgi_pass archwiki;
fastcgi_index index.php;
fastcgi_split_path_info ^(.+\.php)(.*)$;
......@@ -65,6 +68,7 @@ server {
# special case for '/load.php' type URLs to cache css/js in nginx to relieve php-fpm
location = /load.php {
access_log /var/log/nginx/{{ archwiki_domain }}/access.log main;
access_log /var/log/nginx/{{ archwiki_domain }}/access.log.json json_main;
fastcgi_pass archwiki;
fastcgi_index index.php;
fastcgi_split_path_info ^(.+\.php)(.*)$;
......@@ -82,6 +86,7 @@ server {
# normal PHP FastCGI handler
location ~ ^/[^/]+\.php$ {
access_log /var/log/nginx/{{ archwiki_domain }}/access.log main;
access_log /var/log/nginx/{{ archwiki_domain }}/access.log.json json_main;
fastcgi_pass archwiki;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
......
......@@ -16,6 +16,7 @@ server {
server_name {{ aurweb_domain }};
access_log /var/log/nginx/{{ aurweb_domain }}/access.log main;
access_log /var/log/nginx/{{ aurweb_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ aurweb_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -31,6 +32,7 @@ server {
server_name {{ aurweb_domain }};
access_log /var/log/nginx/{{ aurweb_domain }}/access.log main;
access_log /var/log/nginx/{{ aurweb_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ aurweb_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ aurweb_domain }}/fullchain.pem;
......
......@@ -15,6 +15,7 @@ server {
satisfy any;
access_log /var/log/nginx/{{ repos_domain }}/access.log reduced;
access_log /var/log/nginx/{{ repos_domain }}/access.log.json json_reduced;
location /lastupdate {
allow all;
......
......@@ -8,6 +8,7 @@ server {
server_name {{ flyspray_domain }};
access_log /var/log/nginx/{{ flyspray_domain }}/access.log reduced;
access_log /var/log/nginx/{{ flyspray_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ flyspray_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -24,6 +25,7 @@ server {
server_name {{ flyspray_domain }};
access_log /var/log/nginx/{{ flyspray_domain }}/access.log reduced;
access_log /var/log/nginx/{{ flyspray_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ flyspray_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ flyspray_domain }}/fullchain.pem;
......@@ -125,6 +127,7 @@ server {
location ~ \.php$ {
access_log /var/log/nginx/{{ flyspray_domain }}/access.log main;
access_log /var/log/nginx/{{ flyspray_domain }}/access.log.json json_main;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
fastcgi_pass flyspray;
......
......@@ -13,6 +13,7 @@ server {
server_name {{ grafana_domain }};
access_log /var/log/nginx/{{ grafana_domain }}/access.log reduced;
access_log /var/log/nginx/{{ grafana_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ grafana_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -29,6 +30,7 @@ server {
server_name {{ grafana_domain }};
access_log /var/log/nginx/{{ grafana_domain }}/access.log reduced;
access_log /var/log/nginx/{{ grafana_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ grafana_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ grafana_domain }}/fullchain.pem;
......@@ -39,6 +41,7 @@ server {
location / {
access_log /var/log/nginx/{{ grafana_domain }}/access.log main;
access_log /var/log/nginx/{{ grafana_domain }}/access.log.json json_main;
proxy_pass http://grafana;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
......
......@@ -13,6 +13,7 @@ server {
server_name {{ hedgedoc_domain }};
access_log /var/log/nginx/{{ hedgedoc_domain }}/access.log main;
access_log /var/log/nginx/{{ hedgedoc_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ hedgedoc_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -28,6 +29,7 @@ server {
server_name {{ hedgedoc_domain }};
access_log /var/log/nginx/{{ hedgedoc_domain }}/access.log main;
access_log /var/log/nginx/{{ hedgedoc_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ hedgedoc_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ hedgedoc_domain }}/fullchain.pem;
......
......@@ -4,6 +4,7 @@ server {
server_name {{ keycloak_domain }};
access_log /var/log/nginx/{{ keycloak_domain }}/access.log reduced;
access_log /var/log/nginx/{{ keycloak_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ keycloak_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -20,6 +21,7 @@ server {
server_name {{ keycloak_domain }};
access_log /var/log/nginx/{{ keycloak_domain }}/access.log reduced;
access_log /var/log/nginx/{{ keycloak_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ keycloak_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ keycloak_domain }}/fullchain.pem;
......@@ -38,6 +40,7 @@ server {
auth_basic_user_file {{ keycloak_nginx_htpasswd }};
access_log /var/log/nginx/{{ keycloak_domain }}/access.log main;
access_log /var/log/nginx/{{ keycloak_domain }}/access.log.json json_main;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
......@@ -48,6 +51,7 @@ server {
location / {
access_log /var/log/nginx/{{ keycloak_domain }}/access.log main;
access_log /var/log/nginx/{{ keycloak_domain }}/access.log.json json_main;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
......
......@@ -4,6 +4,7 @@ server {
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
access_log /var/log/nginx/{{ logging_domain }}/access.log.json json_main;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -14,12 +15,20 @@ server {
}
}
# We don't want to log (/loki/api/v1/push) request from yourself as it would cause a infinite loop
map $remote_addr $loggable {
{{ ansible_default_ipv4.address }} 0;
{{ ansible_default_ipv6.address }} 0;
default 1;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
access_log /var/log/nginx/{{ logging_domain }}/access.log.json json_main if=$loggable;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ logging_domain }}/fullchain.pem;
......
......@@ -4,6 +4,7 @@ server {
server_name mailman.archlinux.org;
access_log /var/log/nginx/{{ mailman_domain }}/access.log reduced;
access_log /var/log/nginx/{{ mailman_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ mailman_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -20,6 +21,7 @@ server {
server_name mailman.archlinux.org;
access_log /var/log/nginx/{{ mailman_domain }}/access.log reduced;
access_log /var/log/nginx/{{ mailman_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ mailman_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/mailman.archlinux.org/fullchain.pem;
......
......@@ -7,6 +7,7 @@ server {
server_name {{ domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
include snippets/letsencrypt.conf;
......@@ -23,6 +24,7 @@ server {
server_name {{ domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
ssl_certificate /etc/letsencrypt/live/{{ service_domain }}/fullchain.pem;
......@@ -46,6 +48,7 @@ server {
server_name {{ service_domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
include snippets/letsencrypt.conf;
......@@ -62,6 +65,7 @@ server {
server_name {{ service_domain }};
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log reduced;
access_log {{ maintenance_logs_dir }}/{{ service_domain }}-access.log.json json_reduced;
error_log {{ maintenance_logs_dir }}/{{ service_domain }}-error.log;
ssl_certificate /etc/letsencrypt/live/{{ service_domain }}/fullchain.pem;
......
......@@ -10,6 +10,7 @@ server {
server_name {{ matrix_domain }};
access_log /var/log/nginx/{{ matrix_domain }}/access.log reduced;
access_log /var/log/nginx/{{ matrix_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ matrix_domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -26,6 +27,7 @@ server {
server_name {{ matrix_domain }};
access_log /var/log/nginx/{{ matrix_domain }}/access.log reduced;
access_log /var/log/nginx/{{ matrix_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ matrix_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ matrix_domain }}/fullchain.pem;
......@@ -37,6 +39,7 @@ server {
{% for location in config.locations %}
location {{ location }} {
access_log /var/log/nginx/{{ matrix_domain }}/access.log main;
access_log /var/log/nginx/{{ matrix_domain }}/access.log.json json_main;
proxy_pass http://matrix_{{ config.name }};
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Proto $scheme;
......
......@@ -6,6 +6,7 @@ server {
server_name mta-sts.{{ config.domains | join(' mta-sts.') }};
access_log /var/log/nginx/{{ domain }}/access.log reduced;
access_log /var/log/nginx/{{ domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ domain }}/error.log;
include snippets/letsencrypt.conf;
......@@ -22,6 +23,7 @@ server {
server_name mta-sts.{{ config.domains | join(' mta-sts.') }};
access_log /var/log/nginx/{{ domain }}/access.log reduced;
access_log /var/log/nginx/{{ domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
......
/var/log/nginx/*/*log {
/var/log/nginx/*/*.log {
missingok
notifempty
create 640 http log
......@@ -11,3 +11,16 @@
endscript
}
# The json files are consumed by promtail so we don't need to keep them around for long
/var/log/nginx/*/*.json {
missingok
notifempty
create 640 http log
sharedscripts
compress
size 10M
rotate 5
postrotate
test ! -r /run/nginx.pid || kill -USR1 `cat /run/nginx.pid`
endscript
}
......@@ -33,13 +33,19 @@ http {
'"host":"$host",'
'"remote_user":"$remote_user",'
'"time_local":"$time_local",'
'"request":"$request",'
'"request_method":"$request_method",'
'"request_uri":"$request_uri",'
'"status": "$status",'
'"body_bytes_sent":"$body_bytes_sent",'
'"http_referrer":"$http_referer",'
'"http_user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for",'
'"request_time":"$request_time"'
'"request_time":"$request_time",'
# This was added to keep every log line unique as Loki drops
# log line with the same timestamp and log text:
# https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
'"connection":"$connection",'
'"connection_requests":"$connection_requests"'
'}';
log_format json_reduced escape=json
......@@ -48,13 +54,19 @@ http {
'"host":"$host",'
'"remote_user":"$remote_user",'
'"time_local":"$time_local",'
'"request":"$request",'
'"request_method":"$request_method",'
'"request_uri":"$request_uri",'
'"status": "$status",'
'"body_bytes_sent":"$body_bytes_sent",'
'"http_referrer":"$http_referer",'
'"http_user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for",'
'"request_time":"$request_time"'
'"request_time":"$request_time",'
# This was added to keep every log line unique as Loki drops
# log line with the same timestamp and log text:
# https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
'"connection":"$connection",'
'"connection_requests":"$connection_requests"'
'}';
sendfile on;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment