Add alert for Fastly cost

If the cost exceeds $0, it indicates that we have run out of credit
and/or are doing something wrong, in either case we want to be alerted.

host_vars/monitoring.archlinux.org/vault_fastly.yml

+$ANSIBLE_VAULT;1.1;AES256
+32326533316233336234643230396235366233656366666434373939376530323066383263656430
+3262616564623638653562383861353837646336313662660a633364333463363230313437303134
+37303165383030383765383331646162646238633131636639376333313932633333666266373361
+3436666263363766390a613362333865353161353333336134663066326463633237346632613265
+36623930316361653838336161323063363633656333313233353234613335383635626261386233
+66363565656136336437623734613737616463373861663263316131303434613734653833646566
+35623064343134386533386237393536313536646539656265646266346232336565393564333232
+33623930393361643935333661646462316463636264653931373032643137616564636438616364
+39316564316331613135623565393766363563653935376336643832366132383764

roles/prometheus/files/node.rules.yml

@@ -500,3 +500,15 @@ groups:
         annotations:
           summary: "Rebuilderd workers offline {{ $labels.instance }})"
           description: "Not all rebuilder-workers are online, currently {{ $value }} workers are online"
+
+  - name: fastly
+    interval: 60m
+    rules:
+      - alert: FastlyCostNotZero
+        expr: max_over_time(fastly_mtd_estimate_dollars_total[2h]) > 0
+        labels:
+          severity: warning
+          service: fastly
+        annotations:
+          summary: "Fastly is consuming real money"
+          description: "Fastly month-to-date billing estimate is now: {{ $value }}"

roles/prometheus_exporters/tasks/main.yml

@@ -180,6 +180,21 @@
   systemd: name=prometheus-repository-textcollector.timer enabled=yes daemon_reload=yes state=started
   when: "inventory_hostname == 'gemini.archlinux.org'"
 
+- name: Install fastly textcollector script
+  template: src=fastly-textcollector.sh.j2 dest=/usr/local/bin/fastly-textcollector.sh owner=root group=node_exporter mode=0750
+  when: "inventory_hostname == 'monitoring.archlinux.org'"
+
+- name: Install fastly textcollector service and timer
+  template: src={{ item }} dest=/etc/systemd/system/{{ item | regex_replace('\\.j2$', '') }} owner=root group=root mode=644
+  loop:
+    - prometheus-fastly-textcollector.service.j2
+    - prometheus-fastly-textcollector.timer.j2
+  when: "inventory_hostname == 'monitoring.archlinux.org'"
+
+- name: Enable and start prometheus fastly textcollector timer
+  systemd: name=prometheus-fastly-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "inventory_hostname == 'monitoring.archlinux.org'"
+
 - name: Install sudoers for btrfs
   copy: src=sudoers dest=/etc/sudoers.d/node_exporter owner=root group=root mode=0440 validate='visudo -cf %s'
   when: filesystem == "btrfs"

roles/prometheus_exporters/templates/fastly-textcollector.sh.j2

+#!/bin/bash
+set -o errexit -o nounset -o pipefail
+fastly_token="{{ vault_fastly_billing_token }}"
+fastly_customer_id="{{ vault_fastly_customer_id }}"
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/fastly.prom
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+MTD="$(curl --silent \
+            --show-error \
+            --fail \
+            --header @<(echo Fastly-Key: ${fastly_token}) \
+            --header "Accept: application/json" \
+            "https://api.fastly.com/billing/v2/account_customers/${fastly_customer_id}/mtd_invoice" | jq .total.cost)"
+
+echo "# HELP fastly_mtd_estimate_dollars_total month-to-date billing estimate" >> $TMP_FILE
+echo "# TYPE fastly_mtd_estimate_dollars_total gauge" >> $TMP_FILE
+echo "fastly_mtd_estimate_dollars_total $MTD" >> $TMP_FILE
+
+mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/templates/prometheus-fastly-textcollector.service.j2

+[Unit]
+Description=Prometheus Fastly Exporter
+After=network.target
+
+[Service]
+Type=oneshot
+User=node_exporter
+ExecStart=/usr/local/bin/fastly-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+CapabilityBoundingSet=
+UMask=077
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native

roles/prometheus_exporters/templates/prometheus-fastly-textcollector.timer.j2

+[Unit]
+Description=Prometheus Fastly Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=1h
+OnBootSec=15min
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target