Deploy tempo

This will be our backend for a Grafana-based APM.
It is sorely required for gaining insights into why application such as aurweb are slow.

We currently only enable the OTLP receiver as it seems to be the most
modern and best supported one.

We connect directly to the prometheus at localhost for the generated
metrics.

We're also using just storing traces locally in files instead of
something like S3.

playbooks/monitoring.archlinux.org.yml

@@ -14,6 +14,7 @@
     - { role: prometheus_exporters }
     - { role: loki }
     - { role: promtail }
+    - { role: tempo }
     - { role: certbot }
     - { role: nginx }
     - { role: grafana, grafana_domain: 'monitoring.archlinux.org' }

roles/tempo/handlers/main.yml

+- name: Restart tempo
+  service: name=tempo state=restarted

roles/tempo/tasks/main.yml

+- name: Install tempo
+  pacman: name=tempo state=present
+
+- name: Open promtail ipv4 port for monitoring.archlinux.org
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=4318 accept"
+  tags:
+    - firewall
+
+- name: Configure tempo
+  template: src=config.yml.j2 dest=/etc/tempo/config.yml owner=tempo group=tempo mode=0644
+  notify:
+    - Restart tempo
+
+- name: Enable tempo server service
+  systemd: name=tempo enabled=yes daemon_reload=yes state=started

roles/tempo/templates/config.yml.j2

+stream_over_http_enabled: true
+server:
+  http_listen_address: 127.0.0.1
+  http_listen_port: 3200
+  grpc_listen_address: 127.0.0.1
+  grpc_listen_port: 3201
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: {{ wireguard_address }}:4318
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/lib/tempo/generator/wal
+    remote_write:
+      - url: http://localhost:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/lib/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /var/lib/tempo/wal             # where to store the wal locally
+    local:
+      path: /var/lib/tempo/blocks
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator