Skip to content
Snippets Groups Projects
Verified Commit fe742be3 authored by Kristian Klausen's avatar Kristian Klausen :tada:
Browse files

Merge branch 'loki-recording-rules' into 'master' 

Add Loki recording rules and Grafana dashboard for tracking cache hit ratio, HTTP version and TLS version.

See merge request !865
parents 12fbdc54 99977f2f
No related branches found
No related tags found
1 merge request!865Add Loki recording rules and Grafana dashboard for tracking cache hit ratio, HTTP version and TLS version.
Pipeline #115222 passed
Stage: test
Hide whitespace changes
Inline Side-by-side
roles/grafana/files/dashboards/nginx-stats.json 0 → 100644
+ 388
− 0
View file @ fe742be3
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "https://gitlab.archlinux.org/archlinux/infrastructure/-/merge_requests/865",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 32,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Only cacheable requests are included (e.g. {fastcgi,proxy}_cache is configured)",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [
"mean"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "instance_cache_status:requests:rate1m{instance=\"$instance\",cache_status!=\"\"} / ignoring(cache_status) group_left sum by (instance) (instance_cache_status:requests:rate1m{instance=\"$instance\",cache_status!=\"\"})",
"instant": false,
"legendFormat": "{{cache_status}}",
"range": true,
"refId": "A"
}
],
"title": "Cache hit ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by (instance, http_version) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",http_version!=\"\"}) / ignoring(http_version) group_left sum by (instance) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",http_version!=\"\"})",
"instant": false,
"legendFormat": "{{http_version}}",
"range": true,
"refId": "A"
}
],
"title": "HTTP version",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 100,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"id": 3,
"options": {
"legend": {
"calcs": [
"mean"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by (instance, tls_version) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",tls_version!=\"\"}) / ignoring(tls_version) group_left sum by (instance) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",tls_version!=\"\"})",
"instant": false,
"legendFormat": "{{tls_version}}",
"range": true,
"refId": "A"
}
],
"title": "TLS version",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [],
"templating": {
"list": [
{
"current": {
"text": "wiki.archlinux.org",
"value": "wiki.archlinux.org"
},
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"definition": "label_values(instance_cache_status:requests:rate1m,instance)",
"includeAll": false,
"label": "Instance",
"name": "instance",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(instance_cache_status:requests:rate1m,instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Nginx stats",
"uid": "cdvt95bi4660wc",
"version": 16,
"weekStart": ""
}
\ No newline at end of file
roles/loki/files/loki.yaml
+ 11
− 1
View file @ fe742be3
......@@ -12,7 +12,7 @@ common:
storage:
filesystem:
chunks_directory: /var/lib/loki/chunks
rules_directory: /var/lib/loki/rules
rules_directory: /etc/loki/rules
replication_factor: 1
instance_addr: 127.0.0.1
ring:
......@@ -25,6 +25,16 @@ ingester:
replay_memory_ceiling: 200MB
chunk_encoding: zstd
ruler:
rule_path: /var/lib/loki/rules-tmp
wal:
dir: /var/lib/loki/ruler-wal
remote_write:
enabled: true
clients:
local:
url: http://127.0.0.1:9090/api/v1/write
schema_config:
configs:
- from: 2022-06-07
......
roles/loki/files/rules.yaml 0 → 100644
+ 8
− 0
View file @ fe742be3
groups:
- name: NginxRules
interval: 1m
rules:
- record: instance_http_version_tls_version_tls_cipher:requests:rate1m
expr: 'sum by (instance, http_version, tls_version, tls_cipher) (rate({job="nginx"}[1m] | json http_version="server_protocol", tls_version="ssl_protocol", tls_cipher="ssl_cipher"))'
- record: instance_cache_status:requests:rate1m
expr: 'sum by (instance, cache_status) (rate({job="nginx"}[1m] | json cache_status="upstream_cache_status"))'
roles/loki/tasks/main.yml
+ 10
− 0
View file @ fe742be3
......@@ -5,6 +5,16 @@
copy: src=loki.yaml dest=/etc/loki/ owner=root group=root mode=0644
notify: Restart loki
- name: Create directories for loki recording rules
file: path={{ item }} state=directory owner=root group=root mode=0755
loop:
- /etc/loki/rules
- /etc/loki/rules/fake
- name: Install loki recording rules
copy: src=rules.yaml dest=/etc/loki/rules/fake/ owner=root group=root mode=0644
notify: Restart loki
- name: Make nginx log dir
file: path=/var/log/nginx/loki state=directory owner=root group=root mode=0755
......
roles/nginx/templates/nginx.conf.j2
+ 10
− 2
View file @ fe742be3
......@@ -28,12 +28,14 @@ http {
'$remote_addr $host $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" $request_time '
'$server_protocol';
'$server_protocol $ssl_protocol $ssl_cipher '
'$upstream_cache_status';
log_format reduced
'$host [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $server_protocol';
'"$http_user_agent" $server_protocol $ssl_protocol '
'$ssl_cipher $upstream_cache_status';
log_format json_main escape=json
'{'
......@@ -50,6 +52,9 @@ http {
'"http_x_forwarded_for":"$http_x_forwarded_for",'
'"request_time":"$request_time",'
'"server_protocol":"$server_protocol",'
'"ssl_protocol":"$ssl_protocol",'
'"ssl_cipher":"$ssl_cipher",'
'"upstream_cache_status":"$upstream_cache_status",'
# This was added to keep every log line unique as Loki drops
# log line with the same timestamp and log text:
# https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
......@@ -68,6 +73,9 @@ http {
'"http_referrer":"$http_referer",'
'"http_user_agent":"$http_user_agent",'
'"server_protocol":"$server_protocol",'
'"ssl_protocol":"$ssl_protocol",'
'"ssl_cipher":"$ssl_cipher",'
'"upstream_cache_status":"$upstream_cache_status",'
# This was added to keep every log line unique as Loki drops
# log line with the same timestamp and log text:
# https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
......
roles/prometheus/templates/prometheus.conf.j2
+ 1
− 1
View file @ fe742be3
{% if prometheus_receive_only %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --web.enable-remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml --web.listen-address={{ wireguard_address }}:9090"
{% else %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --web.enable-remote-write-receiver"
{% endif %}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment