loki/prometheus: Add plumbing for using loki recording rules[1]

The plan is to use this for creating metrics from the nginx log lines
(e.g. requests per second).

[1] https://grafana.com/docs/loki/latest/alert/#recording-rules

roles/loki/files/loki.yaml

@@ -12,7 +12,7 @@ common:
   storage:
     filesystem:
       chunks_directory: /var/lib/loki/chunks
-      rules_directory: /var/lib/loki/rules
+      rules_directory: /etc/loki/rules
   replication_factor: 1
   instance_addr: 127.0.0.1
   ring:
@@ -25,6 +25,16 @@ ingester:
     replay_memory_ceiling: 200MB
   chunk_encoding: zstd
 
+ruler:
+  rule_path: /var/lib/loki/rules-tmp
+  wal:
+    dir: /var/lib/loki/ruler-wal
+  remote_write:
+    enabled: true
+    clients:
+      local:
+        url: http://127.0.0.1:9090/api/v1/write
+
 schema_config:
   configs:
     - from: 2022-06-07

roles/loki/files/rules.yaml

+groups: []

roles/loki/tasks/main.yml

@@ -5,6 +5,16 @@
   copy: src=loki.yaml dest=/etc/loki/ owner=root group=root mode=0644
   notify: Restart loki
 
+- name: Create directories for loki recording rules
+  file: path={{ item }} state=directory owner=root group=root mode=0755
+  loop:
+    - /etc/loki/rules
+    - /etc/loki/rules/fake
+
+- name: Install loki recording rules
+  copy: src=rules.yaml dest=/etc/loki/rules/fake/ owner=root group=root mode=0644
+  notify: Restart loki
+
 - name: Make nginx log dir
   file: path=/var/log/nginx/loki state=directory owner=root group=root mode=0755
 

roles/prometheus/templates/prometheus.conf.j2

 {% if prometheus_receive_only %}
 PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --web.enable-remote-write-receiver --web.config.file=/etc/prometheus/web-config.yml --web.listen-address={{ wireguard_address }}:9090"
 {% else %}
-PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d"
+PROMETHEUS_ARGS="--storage.tsdb.retention.time=365d --web.enable-remote-write-receiver"
 {% endif %}