From 434182cb22aef3bba8af09dc0c0c4f4ffb8cbe26 Mon Sep 17 00:00:00 2001
From: Kristian Klausen <kristian@klausen.dk>
Date: Fri, 15 Mar 2024 19:42:05 +0100
Subject: [PATCH] fastly billing alert
---
.../monitoring.archlinux.org/vault_fastly.yml | 10 +++++
roles/prometheus/files/node.rules.yml | 12 ++++++
roles/prometheus_exporters/tasks/main.yml | 15 ++++++++
.../templates/fastly-textcollector.sh.j2 | 32 ++++++++++++++++
...prometheus-fastly-textcollector.service.j2 | 37 +++++++++++++++++++
.../prometheus-fastly-textcollector.timer.j2 | 10 +++++
6 files changed, 116 insertions(+)
create mode 100644 host_vars/monitoring.archlinux.org/vault_fastly.yml
create mode 100644 roles/prometheus_exporters/templates/fastly-textcollector.sh.j2
create mode 100644 roles/prometheus_exporters/templates/prometheus-fastly-textcollector.service.j2
create mode 100644 roles/prometheus_exporters/templates/prometheus-fastly-textcollector.timer.j2
diff --git a/host_vars/monitoring.archlinux.org/vault_fastly.yml b/host_vars/monitoring.archlinux.org/vault_fastly.yml
new file mode 100644
index 000000000..be57d2320
--- /dev/null
+++ b/host_vars/monitoring.archlinux.org/vault_fastly.yml
@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+32326533316233336234643230396235366233656366666434373939376530323066383263656430
+3262616564623638653562383861353837646336313662660a633364333463363230313437303134
+37303165383030383765383331646162646238633131636639376333313932633333666266373361
+3436666263363766390a613362333865353161353333336134663066326463633237346632613265
+36623930316361653838336161323063363633656333313233353234613335383635626261386233
+66363565656136336437623734613737616463373861663263316131303434613734653833646566
+35623064343134386533386237393536313536646539656265646266346232336565393564333232
+33623930393361643935333661646462316463636264653931373032643137616564636438616364
+39316564316331613135623565393766363563653935376336643832366132383764
diff --git a/roles/prometheus/files/node.rules.yml b/roles/prometheus/files/node.rules.yml
index 136b34103..09610ffdc 100644
--- a/roles/prometheus/files/node.rules.yml
+++ b/roles/prometheus/files/node.rules.yml
@@ -500,3 +500,15 @@ groups:
annotations:
summary: "Rebuilderd workers offline {{ $labels.instance }})"
description: "Not all rebuilder-workers are online, currently {{ $value }} workers are online"
+
+ - name: fastly
+ interval: 60m
+ rules:
+ - alert: FastlyCostNotZero
+ expr: max_over_time(fastly_mtd_estimate_dollars_total[2h]) > 0
+ labels:
+ severity: warning
+ service: fastly
+ annotations:
+ summary: "Fastly is consuming real money"
+ description: "Fastly's month-to-date billing estimate is now: {{ $value }}"
diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml
index 1c42ee139..f8251dde3 100644
--- a/roles/prometheus_exporters/tasks/main.yml
+++ b/roles/prometheus_exporters/tasks/main.yml
@@ -180,6 +180,21 @@
systemd: name=prometheus-repository-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "inventory_hostname == 'gemini.archlinux.org'"
+- name: Install fastly textcollector script
+ template: src=fastly-textcollector.sh.j2 dest=/usr/local/bin/fastly-textcollector.sh owner=root group=node_exporter mode=0750
+ when: "inventory_hostname == 'monitoring.archlinux.org'"
+
+- name: Install fastly textcollector service and timer
+ template: src={{ item }} dest=/etc/systemd/system/{{ item | regex_replace('\\.j2$', '') }} owner=root group=root mode=644
+ loop:
+ - prometheus-fastly-textcollector.service.j2
+ - prometheus-fastly-textcollector.timer.j2
+ when: "inventory_hostname == 'monitoring.archlinux.org'"
+
+- name: Enable and start prometheus fastly textcollector timer
+ systemd: name=prometheus-fastly-textcollector.timer enabled=yes daemon_reload=yes state=started
+ when: "inventory_hostname == 'monitoring.archlinux.org'"
+
- name: Install sudoers for btrfs
copy: src=sudoers dest=/etc/sudoers.d/node_exporter owner=root group=root mode=0440
when: filesystem == "btrfs"
diff --git a/roles/prometheus_exporters/templates/fastly-textcollector.sh.j2 b/roles/prometheus_exporters/templates/fastly-textcollector.sh.j2
new file mode 100644
index 000000000..ff2b7b063
--- /dev/null
+++ b/roles/prometheus_exporters/templates/fastly-textcollector.sh.j2
@@ -0,0 +1,32 @@
+#!/bin/bash
+set -o errexit -o nounset -o pipefail
+fastly_token="{{ vault_fastly_billing_token }}"
+fastly_customer_id="{{ vault_fastly_customer_id }}"
+
+if (( $# != 1 )); then
+ echo "Missing textcollector directory argument"
+ exit 1
+fi
+
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/fastly.prom
+
+
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+MTD="$(curl --silent \
+ --show-error \
+ --fail \
+ --header @<(echo Fastly-Key: ${fastly_token}) \
+ --header "Accept: application/json" \
+ "https://api.fastly.com/billing/v2/account_customers/${fastly_customer_id}/mtd_invoice" | jq .total.cost)"
+
+echo "# HELP fastly_mtd_estimate_dollars_total month-to-date billing estimate" >> $TMP_FILE
+echo "# TYPE fastly_mtd_estimate_dollars_total gauge" >> $TMP_FILE
+echo "fastly_mtd_estimate_dollars_total $MTD" >> $TMP_FILE
+
+mv -f $TMP_FILE $PROM_FILE
diff --git a/roles/prometheus_exporters/templates/prometheus-fastly-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-fastly-textcollector.service.j2
new file mode 100644
index 000000000..79fc587a6
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-fastly-textcollector.service.j2
@@ -0,0 +1,37 @@
+[Unit]
+Description=Prometheus Fastly Exporter
+After=network.target
+
+[Service]
+Type=oneshot
+User=node_exporter
+ExecStart=/usr/local/bin/fastly-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+CapabilityBoundingSet=
+UMask=077
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native
diff --git a/roles/prometheus_exporters/templates/prometheus-fastly-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-fastly-textcollector.timer.j2
new file mode 100644
index 000000000..f2df92859
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-fastly-textcollector.timer.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Prometheus Fastly Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=1h
+OnBootSec=15min
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target
--
GitLab