Verified Commit 7235e726 authored by Kristian Klausen's avatar Kristian Klausen 🎉 Committed by Jelle van der Waa
Browse files

Implement centralized logging

Fix #263
parent 99d76922
...@@ -20,3 +20,4 @@ ...@@ -20,3 +20,4 @@
- { role: patchwork } - { role: patchwork }
- { role: fail2ban } - { role: fail2ban }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
...@@ -12,3 +12,4 @@ ...@@ -12,3 +12,4 @@
- { role: sshd } - { role: sshd }
- { role: root_ssh } - { role: root_ssh }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
...@@ -18,3 +18,4 @@ ...@@ -18,3 +18,4 @@
postgres_effective_cache_size: 1GB postgres_effective_cache_size: 1GB
- { role: quassel, quassel_domain: "quassel.archlinux.org" } - { role: quassel, quassel_domain: "quassel.archlinux.org" }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
...@@ -11,4 +11,5 @@ ...@@ -11,4 +11,5 @@
- { role: root_ssh } - { role: root_ssh }
- { role: rebuilderd_worker } - { role: rebuilderd_worker }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
- { role: fail2ban } - { role: fail2ban }
...@@ -11,5 +11,6 @@ ...@@ -11,5 +11,6 @@
- { role: nginx } - { role: nginx }
- { role: redirects } - { role: redirects }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
- { role: hardening } - { role: hardening }
- { role: ping } - { role: ping }
...@@ -14,4 +14,5 @@ ...@@ -14,4 +14,5 @@
- { role: nginx } - { role: nginx }
- { role: rebuilderd } - { role: rebuilderd }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
- { role: fail2ban } - { role: fail2ban }
...@@ -20,3 +20,4 @@ ...@@ -20,3 +20,4 @@
security_tracker_dir: "/srv/http/security-tracker" security_tracker_dir: "/srv/http/security-tracker"
- { role: fail2ban } - { role: fail2ban }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
...@@ -20,4 +20,5 @@ ...@@ -20,4 +20,5 @@
postgres_ssl_hosts6: ['::/0'] postgres_ssl_hosts6: ['::/0']
- { role: terraform_state } - { role: terraform_state }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
- { role: fail2ban } - { role: fail2ban }
...@@ -19,3 +19,4 @@ ...@@ -19,3 +19,4 @@
- { role: archwiki } - { role: archwiki }
- { role: fail2ban } - { role: fail2ban }
- { role: prometheus_exporters } - { role: prometheus_exporters }
- { role: promtail }
...@@ -5,4 +5,8 @@ datasources: ...@@ -5,4 +5,8 @@ datasources:
type: prometheus type: prometheus
access: proxy access: proxy
url: http://localhost:9090 url: http://localhost:9090
- name: Loki
type: loki
access: proxy
url: http://localhost:3100
...@@ -2,6 +2,11 @@ upstream grafana { ...@@ -2,6 +2,11 @@ upstream grafana {
server localhost:3000; server localhost:3000;
} }
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server { server {
listen 80; listen 80;
listen [::]:80; listen [::]:80;
...@@ -36,5 +41,7 @@ server { ...@@ -36,5 +41,7 @@ server {
access_log /var/log/nginx/{{ grafana_domain }}/access.log main; access_log /var/log/nginx/{{ grafana_domain }}/access.log main;
proxy_pass http://grafana; proxy_pass http://grafana;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
} }
} }
logging_domain: logging.archlinux.org
loki_nginx_htpasswd: /etc/nginx/auth/loki
# Enables authentication through the X-Scope-OrgID header, which must be present
# if true. If false, the OrgID will always be set to "fake".
auth_enabled: false
server:
http_listen_address: 127.0.0.1
http_listen_port: 3100
grpc_listen_address: 127.0.0.1
grpc_listen_port: 9095
ingester:
wal:
enabled: true
dir: /var/lib/loki/wal
replay_memory_ceiling: 200MB
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 1536000 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_encoding: zstd
chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /var/lib/loki/boltdb-shipper-active
cache_location: /var/lib/loki/boltdb-shipper-cache
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: filesystem
filesystem:
directory: /var/lib/loki/chunks
compactor:
working_directory: /var/lib/loki/boltdb-shipper-compactor
shared_store: filesystem
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 672h
table_manager:
retention_deletes_enabled: true
retention_period: 672h # 28 days
---
- name: restart loki
service: name=loki state=restarted
---
- name: create ssl cert
include_role:
name: certificate
vars:
domains: ["{{ logging_domain }}"]
- name: install loki and logcli
pacman: name=loki,logcli state=present
- name: install loki configuration
copy: src=loki.yaml dest=/etc/loki/ owner=root group=root mode=0644
notify: restart loki
- name: install python-passlib
pacman: name=python-passlib
- name: create htpasswd for nginx loki endpoint
htpasswd:
path: "{{ loki_nginx_htpasswd }}"
name: "{{ vault_loki_nginx_user }}"
password: "{{ vault_loki_nginx_passwd }}"
owner: root
group: http
mode: 0640
- name: make nginx log dir
file: path=/var/log/nginx/{{ logging_domain }} state=directory owner=root group=root mode=0755
- name: set up nginx
template: src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/logging.conf" owner=root group=root mode=644
notify: reload nginx
tags: ['nginx']
- name: start and enable loki
systemd: name=loki.service enabled=yes daemon_reload=yes state=started
server {
listen 80;
listen [::]:80;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
include snippets/letsencrypt.conf;
location / {
access_log off;
return 301 https://$server_name$request_uri;
}
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ logging_domain }};
access_log /var/log/nginx/{{ logging_domain }}/access.log main;
error_log /var/log/nginx/{{ logging_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ logging_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ logging_domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ logging_domain }}/chain.pem;
location = /loki/api/v1/push {
auth_basic "Loki :)";
auth_basic_user_file {{ loki_nginx_htpasswd }};
proxy_pass http://127.0.0.1:3100$request_uri;
}
# We return a 200 so the monitoring is happy!
location = / {
default_type text/plain;
return 200 'Nothing to see here..\n';
}
location / {
return 404;
}
}
...@@ -23,6 +23,7 @@ blackbox_targets: ...@@ -23,6 +23,7 @@ blackbox_targets:
- https://git.archlinux.org - https://git.archlinux.org
- https://gitlab.archlinux.org - https://gitlab.archlinux.org
- https://ipxe.archlinux.org - https://ipxe.archlinux.org
- https://logging.archlinux.org
- https://lists.archlinux.org - https://lists.archlinux.org
- https://mailman.archlinux.org - https://mailman.archlinux.org
- https://man.archlinux.org - https://man.archlinux.org
......
...@@ -13,6 +13,12 @@ alerting: ...@@ -13,6 +13,12 @@ alerting:
- localhost:9093 - localhost:9093
scrape_configs: scrape_configs:
- job_name: loki
static_configs:
- targets: ['127.0.0.1:3100']
labels:
instance: "{{ ansible_fqdn }}"
- job_name: 'node_exporter' - job_name: 'node_exporter'
static_configs: static_configs:
{% for host in groups['node_exporters'] %} {% for host in groups['node_exporters'] %}
...@@ -23,6 +29,16 @@ scrape_configs: ...@@ -23,6 +29,16 @@ scrape_configs:
{% endfor %} {% endfor %}
- job_name: 'promtail'
static_configs:
{% for host in groups['node_exporters'] %}
- targets: ['{{ host }}:9080']
labels:
instance: "{{ host }}"
{% endfor %}
- job_name: 'gitlab_runner_exporter' - job_name: 'gitlab_runner_exporter'
static_configs: static_configs:
{% for host in groups['gitlab_runners'] %} {% for host in groups['gitlab_runners'] %}
......
logging_domain: logging.archlinux.org
---
- name: restart promtail
service: name=promtail state=restarted
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment