diff --git a/roles/grafana/files/dashboards/nginx-stats.json b/roles/grafana/files/dashboards/nginx-stats.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc535c358ac8a2031bd8dac018a245ab7664ef44
--- /dev/null
+++ b/roles/grafana/files/dashboards/nginx-stats.json
@@ -0,0 +1,388 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "https://gitlab.archlinux.org/archlinux/infrastructure/-/merge_requests/865",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 32,
+ "links": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "description": "Only cacheable requests are included (e.g. {fastcgi,proxy}_cache is configured)",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic",
+ "seriesBy": "last"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "gradientMode": "opacity",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 1,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "instance_cache_status:requests:rate1m{instance=\"$instance\",cache_status!=\"\"} / ignoring(cache_status) group_left sum by (instance) (instance_cache_status:requests:rate1m{instance=\"$instance\",cache_status!=\"\"})",
+ "instant": false,
+ "legendFormat": "{{cache_status}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cache hit ratio",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic",
+ "seriesBy": "last"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "gradientMode": "opacity",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum by (instance, http_version) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",http_version!=\"\"}) / ignoring(http_version) group_left sum by (instance) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",http_version!=\"\"})",
+ "instant": false,
+ "legendFormat": "{{http_version}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "HTTP version",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic",
+ "seriesBy": "last"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "gradientMode": "opacity",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "id": 3,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.3.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum by (instance, tls_version) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",tls_version!=\"\"}) / ignoring(tls_version) group_left sum by (instance) (instance_http_version_tls_version_tls_cipher:requests:rate1m{instance=\"$instance\",tls_version!=\"\"})",
+ "instant": false,
+ "legendFormat": "{{tls_version}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "TLS version",
+ "type": "timeseries"
+ }
+ ],
+ "preload": false,
+ "schemaVersion": 40,
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "text": "wiki.archlinux.org",
+ "value": "wiki.archlinux.org"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "definition": "label_values(instance_cache_status:requests:rate1m,instance)",
+ "includeAll": false,
+ "label": "Instance",
+ "name": "instance",
+ "options": [],
+ "query": {
+ "qryType": 1,
+ "query": "label_values(instance_cache_status:requests:rate1m,instance)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "regex": "",
+ "type": "query"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "browser",
+ "title": "Nginx stats",
+ "uid": "cdvt95bi4660wc",
+ "version": 16,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/roles/loki/files/loki.yaml b/roles/loki/files/loki.yaml
index a4349046a19cb5108496da36d073e8f4bf49ff02..d9e604a8cf2b35eada214ab4b229c89c511f064c 100644
--- a/roles/loki/files/loki.yaml
+++ b/roles/loki/files/loki.yaml
@@ -12,7 +12,7 @@ common:
storage:
filesystem:
chunks_directory: /var/lib/loki/chunks
- rules_directory: /var/lib/loki/rules
+ rules_directory: /etc/loki/rules
replication_factor: 1
instance_addr: 127.0.0.1
ring:
@@ -25,6 +25,16 @@ ingester:
replay_memory_ceiling: 200MB
chunk_encoding: zstd
+ruler:
+ rule_path: /var/lib/loki/rules-tmp
+ wal:
+ dir: /var/lib/loki/ruler-wal
+ remote_write:
+ enabled: true
+ clients:
+ local:
+ url: http://127.0.0.1:9090/api/v1/write
+
schema_config:
configs:
- from: 2022-06-07
diff --git a/roles/loki/files/rules.yaml b/roles/loki/files/rules.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c29ea5f8d7e033e9b3bd167b3bebbcb5c0961b52
--- /dev/null
+++ b/roles/loki/files/rules.yaml
@@ -0,0 +1,8 @@
+groups:
+ - name: NginxRules
+ interval: 1m
+ rules:
+ - record: instance_http_version_tls_version_tls_cipher:requests:rate1m
+ expr: 'sum by (instance, http_version, tls_version, tls_cipher) (rate({job="nginx"}[1m] | json http_version="server_protocol", tls_version="ssl_protocol", tls_cipher="ssl_cipher"))'
+ - record: instance_cache_status:requests:rate1m
+ expr: 'sum by (instance, cache_status) (rate({job="nginx"}[1m] | json cache_status="upstream_cache_status"))'
diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml
index 0eaa79c39795435de0a213968313e158a2acf661..7bbbfca934dffe5446ffd527c117d2517f9e99f6 100644
--- a/roles/loki/tasks/main.yml
+++ b/roles/loki/tasks/main.yml
@@ -5,6 +5,16 @@
copy: src=loki.yaml dest=/etc/loki/ owner=root group=root mode=0644
notify: Restart loki
+- name: Create directories for loki recording rules
+ file: path={{ item }} state=directory owner=root group=root mode=0755
+ loop:
+ - /etc/loki/rules
+ - /etc/loki/rules/fake
+
+- name: Install loki recording rules
+ copy: src=rules.yaml dest=/etc/loki/rules/fake/ owner=root group=root mode=0644
+ notify: Restart loki
+
- name: Make nginx log dir
file: path=/var/log/nginx/loki state=directory owner=root group=root mode=0755
diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2
index f2fb8734db0d76659cd4412c849b08c60f024760..d6da832b7546fefe64e2f5ccc5dedc276ccec58d 100644
--- a/roles/nginx/templates/nginx.conf.j2
+++ b/roles/nginx/templates/nginx.conf.j2
@@ -28,12 +28,14 @@ http {
'$remote_addr $host $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" $request_time '
- '$server_protocol';
+ '$server_protocol $ssl_protocol $ssl_cipher '
+ '$upstream_cache_status';
log_format reduced
'$host [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
- '"$http_user_agent" $server_protocol';
+ '"$http_user_agent" $server_protocol $ssl_protocol '
+ '$ssl_cipher $upstream_cache_status';
log_format json_main escape=json
'{'
@@ -50,6 +52,9 @@ http {
'"http_x_forwarded_for":"$http_x_forwarded_for",'
'"request_time":"$request_time",'
'"server_protocol":"$server_protocol",'
+ '"ssl_protocol":"$ssl_protocol",'
+ '"ssl_cipher":"$ssl_cipher",'
+ '"upstream_cache_status":"$upstream_cache_status",'
# This was added to keep every log line unique as Loki drops
# log line with the same timestamp and log text:
# https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
@@ -68,6 +73,9 @@ http {
'"http_referrer":"$http_referer",'
'"http_user_agent":"$http_user_agent",'
'"server_protocol":"$server_protocol",'
+ '"ssl_protocol":"$ssl_protocol",'
+ '"ssl_cipher":"$ssl_cipher",'
+ '"upstream_cache_status":"$upstream_cache_status",'
# This was added to keep every log line unique as Loki drops
# log line with the same timestamp and log text:
# https://grafana.com/docs/loki/latest/overview/#timestamp-ordering
diff --git a/roles/prometheus/templates/prometheus.conf.j2 b/roles/prometheus/templates/prometheus.conf.j2
index 22b419160eef91b46d449365797d992a54219884..0bc042bb2219f1d2733a8f8f923169a9d8cc73a9 100644
--- a/roles/prometheus/templates/prometheus.conf.j2
+++ b/roles/prometheus/templates/prometheus.conf.j2
@@ -1,5 +1,5 @@
{% if prometheus_receive_only %}
PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --web.enable-remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml --web.listen-address={{ wireguard_address }}:9090"
{% else %}
-PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
+PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --web.enable-remote-write-receiver"
{% endif %}