Merge branch 'inventory-tweaks' into 'master'

Remove [node_exporters]/[wireguard] from inventory + Replace dynamic hcloud inventory with host entries

See merge request archlinux/infrastructure!572

.gitlab-ci.yml

@@ -8,7 +8,6 @@ ansible-lint:
     # This probably happens due to gitlab-runner mounting the git repo into the container
     - chmod o-w .
     # Fix syntax-check rule (https://github.com/ansible-community/ansible-lint/issues/1350#issuecomment-778764110)
-    - sed "s/,hcloud_inventory.py//" -i ansible.cfg
     - sed "/^vault_identity_list/d" -i ansible.cfg
     - sed "/misc\/vaults\/vault_/d" -i playbooks/*.yml
     # Fix load-failure: Failed to load or parse file

README.md

@@ -40,13 +40,6 @@ locally signed with `--lsign-key`. This is necessary for running any of the
 `reencrypt-vault-default-key`, `reencrypt-vault-super-key `or `fetch-borg-keys`
 tasks.
 
-#### Note about Ansible dynamic inventories
-
-We use a dynamic inventory script in order to automatically get information for
-all servers directly from hcloud. You don't really have to do anything to make
-this work but you should keep in mind to NOT add hcloud servers to `hosts`!
-They'll be available automatically.
-
 #### Note about packer
 
 We use packer to build snapshots on hcloud to use as server base images.

ansible.cfg

 [defaults]
-inventory = hosts,hcloud_inventory.py
+inventory = hosts
 library = library
 remote_tmp = $HOME/.ansible/tmp
 remote_user = root

docs/monitoring.md

@@ -5,7 +5,6 @@ To access our monitoring system, go to https://monitoring.archlinux and log in v
 
 ## Adding a new host to monitoring
 
-* Add $host to node_exporters in `hosts`
 * Rollout exporter on host: `ansible-playbook playbooks/host.yml -t prometheus_exporters`
 * Rollout changes on monitoring host: `ansible-playbook playbooks/monitoring.archlinux.org.yml -t prometheus`
 

hcloud_inventory.py

-#!/usr/bin/env python
-#
-# Dynamic inventory script for getting infrastructure information from hcloud
-
-import argparse
-import json
-import sys
-
-from hcloud import Client
-
-from misc.get_key import load_vault
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Hcloud dynamic inventory script")
-    group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument('--list', action='store_true')
-    group.add_argument('--host')
-    return parser.parse_args()
-
-
-def get_host_details(server):
-    return {'ansible_host': server.public_net.ipv4.ip,
-            'ansible_port': 22,
-            'ansible_user': "root"}
-
-
-def main():
-    args = parse_args()
-    loaded = load_vault('misc/vaults/vault_hcloud.yml')
-    client = Client(token=loaded["hcloud_api_key_readonly"])
-    servers = client.servers.get_all()
-
-    hostvars = {server.name: get_host_details(server) for server in servers}
-    if args.list:
-        hosts = [server.name for server in servers]
-        json.dump({'hcloud': hosts, '_meta': {'hostvars': hostvars}}, sys.stdout)
-    else:
-        json.dump(hostvars[args.host], sys.stdout)
-
-
-if __name__ == '__main__':
-    main()

hosts

@@ -100,90 +100,43 @@ patchwork.archlinux.org
 [prometheus]
 monitoring.archlinux.org
 
-[node_exporters]
-archlinux.org
-aur.archlinux.org
-monitoring.archlinux.org
-gitlab.archlinux.org
-reproducible.archlinux.org
-runner2.archlinux.org
-secure-runner1.archlinux.org
-phrik.archlinux.org
-quassel.archlinux.org
-state.archlinux.org
-mirror.pkgbuild.com
-homedir.archlinux.org
-mail.archlinux.org
-matrix.archlinux.org
-accounts.archlinux.org
-bbs.archlinux.org
-build.archlinux.org
-bugs.archlinux.org
-gemini.archlinux.org
-repro1.pkgbuild.com
-wiki.archlinux.org
-patchwork.archlinux.org
-security.archlinux.org
-redirect.archlinux.org
+[kape_servers]
 asia.mirror.pkgbuild.com
 america.mirror.pkgbuild.com
 europe.mirror.pkgbuild.com
 repro2.pkgbuild.com
 runner1.archlinux.org
-md.archlinux.org
-man.archlinux.org
-dashboards.archlinux.org
-lists.archlinux.org
-gluebuddy.archlinux.org
-debuginfod.archlinux.org
 
-[wireguard]
+[dedicated_servers]
+gemini.archlinux.org
+build.archlinux.org
+runner1.archlinux.org
+runner2.archlinux.org
+secure-runner1.archlinux.org
+
+[hcloud]
+accounts.archlinux.org
 archlinux.org
 aur.archlinux.org
-monitoring.archlinux.org
+bbs.archlinux.org
+bugs.archlinux.org
+dashboards.archlinux.org
+debuginfod.archlinux.org
 gitlab.archlinux.org
-reproducible.archlinux.org
-runner2.archlinux.org
-secure-runner1.archlinux.org
-phrik.archlinux.org
-quassel.archlinux.org
-state.archlinux.org
-mirror.pkgbuild.com
+gluebuddy.archlinux.org
 homedir.archlinux.org
+lists.archlinux.org
 mail.archlinux.org
+man.archlinux.org
 matrix.archlinux.org
-accounts.archlinux.org
-bbs.archlinux.org
-build.archlinux.org
-bugs.archlinux.org
-gemini.archlinux.org
-repro1.pkgbuild.com
-wiki.archlinux.org
+md.archlinux.org
+mirror.pkgbuild.com
+monitoring.archlinux.org
 patchwork.archlinux.org
-security.archlinux.org
+phrik.archlinux.org
+quassel.archlinux.org
 redirect.archlinux.org
-asia.mirror.pkgbuild.com
-america.mirror.pkgbuild.com
-europe.mirror.pkgbuild.com
-repro2.pkgbuild.com
-runner1.archlinux.org
-md.archlinux.org
-man.archlinux.org
-dashboards.archlinux.org
-lists.archlinux.org
-gluebuddy.archlinux.org
-debuginfod.archlinux.org
-
-[kape_servers]
-asia.mirror.pkgbuild.com
-america.mirror.pkgbuild.com
-europe.mirror.pkgbuild.com
-repro2.pkgbuild.com
-runner1.archlinux.org
-
-[dedicated_servers]
-gemini.archlinux.org
-build.archlinux.org
-runner1.archlinux.org
-runner2.archlinux.org
-secure-runner1.archlinux.org
+reproducible.archlinux.org
+security.archlinux.org
+state.archlinux.org
+wiki.archlinux.org

roles/prometheus/templates/prometheus.yml.j2

@@ -52,7 +52,7 @@ scrape_configs:
 
   - job_name: 'node_exporter'
     static_configs:
-      {% for host in groups['node_exporters'] %}
+      {% for host in groups['all'] %}
       - targets: ['{{ host }}:{{ prometheus_exporter_port }}']
         labels:
           instance: "{{ host }}"
@@ -60,7 +60,7 @@ scrape_configs:
 
   - job_name: 'promtail'
     static_configs:
-      {% for host in groups['node_exporters'] %}
+      {% for host in groups['all'] %}
       - targets: ['{{ hostvars[host]['wireguard_address'] }}:9080']
         labels:
           instance: "{{ host }}"

roles/wireguard/templates/wg0.netdev.j2

@@ -6,7 +6,7 @@ Kind=wireguard
 ListenPort=51820
 PrivateKey={{ vault_wireguard_private_key }}
 
-{% for host in groups['wireguard'] if host != inventory_hostname %}
+{% for host in groups['all'] if host != inventory_hostname %}
 [WireGuardPeer]
 PublicKey={{ hostvars[host]['wireguard_public_key'] }}
 AllowedIPs={{ hostvars[host]['wireguard_address'] }}/32