From 0c97667925bd349a21706901fef3b5c160b2bc54 Mon Sep 17 00:00:00 2001
From: Sven-Hendrik Haase <svenstaro@gmail.com>
Date: Sat, 3 Aug 2024 04:15:22 +0200
Subject: [PATCH] Deploy tempo
This will be our backend for a Grafana-based APM.
It is sorely required for gaining insights into why application such as aurweb are slow.
We currently only enable the OTLP receiver as it seems to be the most
modern and best supported one.
We connect directly to the prometheus at localhost for the generated
metrics.
We're also using just storing traces locally in files instead of
something like S3.
---
playbooks/monitoring.archlinux.org.yml | 1 +
roles/handlers/main.yml | 0
roles/tempo/handlers/main.yml | 2 +
roles/tempo/tasks/main.yml | 16 ++++++++
roles/tempo/templates/config.yml.j2 | 51 ++++++++++++++++++++++++++
5 files changed, 70 insertions(+)
create mode 100644 roles/handlers/main.yml
create mode 100644 roles/tempo/handlers/main.yml
create mode 100644 roles/tempo/tasks/main.yml
create mode 100644 roles/tempo/templates/config.yml.j2
diff --git a/playbooks/monitoring.archlinux.org.yml b/playbooks/monitoring.archlinux.org.yml
index 2bd20f858..14e93f5be 100644
--- a/playbooks/monitoring.archlinux.org.yml
+++ b/playbooks/monitoring.archlinux.org.yml
@@ -14,6 +14,7 @@
- { role: prometheus_exporters }
- { role: loki }
- { role: promtail }
+ - { role: tempo }
- { role: certbot }
- { role: nginx }
- { role: grafana, grafana_domain: 'monitoring.archlinux.org' }
diff --git a/roles/handlers/main.yml b/roles/handlers/main.yml
new file mode 100644
index 000000000..e69de29bb
diff --git a/roles/tempo/handlers/main.yml b/roles/tempo/handlers/main.yml
new file mode 100644
index 000000000..cece918ab
--- /dev/null
+++ b/roles/tempo/handlers/main.yml
@@ -0,0 +1,2 @@
+- name: Restart tempo
+ service: name=tempo state=restarted
diff --git a/roles/tempo/tasks/main.yml b/roles/tempo/tasks/main.yml
new file mode 100644
index 000000000..bb7a23ee0
--- /dev/null
+++ b/roles/tempo/tasks/main.yml
@@ -0,0 +1,16 @@
+- name: Install tempo
+ pacman: name=tempo state=present
+
+- name: Open promtail ipv4 port for monitoring.archlinux.org
+ ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+ rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['wireguard_address'] }} port protocol=tcp port=4318 accept"
+ tags:
+ - firewall
+
+- name: Configure tempo
+ template: src=config.yml.j2 dest=/etc/tempo/config.yml owner=tempo group=tempo mode=0644
+ notify:
+ - Restart tempo
+
+- name: Enable tempo server service
+ systemd: name=tempo enabled=yes daemon_reload=yes state=started
diff --git a/roles/tempo/templates/config.yml.j2 b/roles/tempo/templates/config.yml.j2
new file mode 100644
index 000000000..0b1b4f9bd
--- /dev/null
+++ b/roles/tempo/templates/config.yml.j2
@@ -0,0 +1,51 @@
+stream_over_http_enabled: true
+server:
+ http_listen_address: 127.0.0.1
+ http_listen_port: 3200
+ grpc_listen_address: 127.0.0.1
+ grpc_listen_port: 3201
+ log_level: info
+
+query_frontend:
+ search:
+ duration_slo: 5s
+ throughput_bytes_slo: 1.073741824e+09
+ trace_by_id:
+ duration_slo: 5s
+
+distributor:
+ receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: {{ wireguard_address }}:4318
+
+ingester:
+ max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+ compaction:
+ block_retention: 1h # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+ registry:
+ external_labels:
+ source: tempo
+ storage:
+ path: /var/lib/tempo/generator/wal
+ remote_write:
+ - url: http://localhost:9090/api/v1/write
+ send_exemplars: true
+ traces_storage:
+ path: /var/lib/tempo/generator/traces
+
+storage:
+ trace:
+ backend: local # backend configuration to use
+ wal:
+ path: /var/lib/tempo/wal # where to store the wal locally
+ local:
+ path: /var/lib/tempo/blocks
+
+overrides:
+ metrics_generator_processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
--
GitLab