Merge branch 'move-crt.sh-lookups-into-script' into 'master'

prometheus: split crt.sh lookups into new script

See merge request archlinux/infrastructure!591

misc/find-arch-on-crt.sh

+#!/bin/bash
+
+# crt.sh lookup script to generate part of the blackbox_targets.http_prometheus
+# list (stored in roles/prometheus/defaults/main.yml) based on SSL certificates
+
+set -eo pipefail
+
+readonly DOMAINS=(
+  archlinux.org
+  pkgbuild.com
+)
+readonly LOOKUP_URLS=(
+  "${DOMAINS[@]/#/https://crt.sh/?exclude=expired&deduplicate=Y&output=json&q=}"
+)
+
+names() {
+  curl -sf "${LOOKUP_URLS[@]}" | jq -r '.[].name_value' | sort -u
+}
+
+prometheus_targets() {
+  names \
+  | sed 's|^|https://|' \
+  | sed '/monitoring\|dashboards/ s|$|/healthz|' \
+  | sed '/mta-sts/ s|$|/.well-known/mta-sts.txt|' \
+  | sed '/openpgpkey/ s|openpgpkey\.\(.*\)|&/.well-known/openpgpkey/\1/policy|' \
+  | sed '/repos\.arch/ s|$|/lastupdate|' \
+  | sed '/static\.conf/ s|$|/README.md|' \
+  | sed -r '/geo\.mirror|(bugs-old|coc|git|status)\.arch/d' \
+  | xargs -P8 -I{} sh -c 'curl -m 10 -sf -o /dev/null {} && echo "    "- {}' \
+  | sort
+}
+
+if [[ $1 = targets ]]; then
+  prometheus_targets
+elif [[ -n $1 ]]; then
+  echo >&2 'error: the first argument can only be empty or "targets"'
+  exit 1
+else
+  names
+fi

roles/prometheus/defaults/main.yml

@@ -11,13 +11,14 @@ prometheus_remote_write_relabel_configs:
   - label: __name__
     regex: "archive_directory_size_bytes|archive_total_packages|rebuilderd_results|rebuilderd_workers|rebuilderd_queue_length|repository_directory_size_bytes|aur_.+"
 
-# for d in $(curl -sf "https://crt.sh/?q=archlinux.org&output=json" "https://crt.sh/?q=pkgbuild.com&output=json" | jq -r ".[].name_value" | sort -u); do if curl -o /dev/null -sS "https://$d"; then echo $d; fi; done | grep -v "\@" | sort | sed "s/^/  - https:\/\//"
 blackbox_targets:
   http_prometheus:
     - targets: "{{ groups['geo_mirrors'] }}"
       hostname: geo.mirror.pkgbuild.com
       secure: true
     - http://{{ hostvars['monitoring.archlinux.org']['wireguard_address'] }}
+
+    # regenerate the list below with: ./misc/find-arch-on-crt.sh targets
     - https://accounts.archlinux.org
     - https://america.archive.pkgbuild.com
     - https://america.mirror.pkgbuild.com