prometheus: Monitor all the mirrors backing our GeoIP mirror

Monitoring just geo.mirror.pkgbuild.com would only monitor the mirror
returned for that geographical area, with this commit we monitor all the
potential returned mirrors.

Co-authored-by: Evangelos Foutras <evangelos@foutrelis.com>

Fix #443

roles/prometheus/defaults/main.yml

@@ -14,6 +14,9 @@ prometheus_remote_write_relabel_configs:
 # for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/  - https:\/\//"
 blackbox_targets:
   http_prometheus:
+    - targets: "{{ groups['geo_mirrors'] }}"
+      hostname: geo.mirror.pkgbuild.com
+      secure: true
     - http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
     - https://accounts.archlinux.org
     - https://america.archive.pkgbuild.com
@@ -31,7 +34,6 @@ blackbox_targets:
     - https://dev.archlinux.org
     - https://europe.archive.pkgbuild.com
     - https://europe.mirror.pkgbuild.com
-    - https://geo.mirror.pkgbuild.com
     - https://gitlab.archlinux.org
     - https://ipxe.archlinux.org
     - https://lists.archlinux.org

roles/prometheus/templates/prometheus.yml.j2

@@ -164,10 +164,20 @@ scrape_configs:
       module: [{{ module }}]
     static_configs:
     - targets:
-    {% for target in blackbox_targets[module] %}
+    {% for target in blackbox_targets[module] | select('string') %}
       - {{ target }}
     {% endfor %}
 
+    {% for vhost in blackbox_targets[module] | select('mapping') %}
+
+    - labels:
+        vhost: {{ vhost.hostname }}
+      targets:
+    {% for target in vhost.targets %}
+      - http{{ 's' if vhost.secure }}://{{ target }}
+    {% endfor %}
+    {% endfor %}
+
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target
@@ -175,7 +185,10 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 127.0.0.1:9115
+      - source_labels: [vhost]
+        target_label: __param_hostname
 {% endfor %}
+
 {% for endpoint in matrix_metrics_endpoints %}
 
   - job_name: 'synapse_{{ endpoint }}'