Introduce prometheus exporters role for collection

Add a new role called prometheus_exporters which should be run on every
machine we have and starts different collectors depending on what group
the machine is in. Currently supported our the gitlab runner exporter,
rebuilder textcollector, mysqld-exporter, borg textcollector and an
node/arch exporter. The arch exporter monitors the security status and
pacman out of date packages gauge.

group_vars/all/common.yml

@@ -10,3 +10,6 @@ zabbix_agent_templates:
 # this is used by the maintenance role to get the ip address
 # of the machine running the playbook
 maintenance_remote_machine: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CLIENT.split(' ')[0] }}"
+
+# prometheus-node-exporter port
+prometheus_exporter_port: '9100'

group_vars/all/vault_gitlab.yml

 $ANSIBLE_VAULT;1.1;AES256
-35393938326563366437646365633563303031393034626433333163373838613535333136356132
-3139393330383337376633313739643431636337343263310a616263613665356437383862663134
-34643230613832643332323634613561313634386636373937373533653338313030633339653235
-6330646665656530350a623165393139306136636433356262626534376436313539646664336266
-30353562633266383138373334656238396464326231313132643461313763356537386535326462
-64643862383638373930373632663462663865346664636635653937303837656239653739343266
-31346133383133656665363034653330633735663862393930303862306566643533633635343935
-31376266356139376630316566613732663231393136316163356433666366613466653664643162
-62393833303265303539333637613662346465326131656139663633386439633032313138353637
-32663337656661303264633862333530336430306433613032353263666365396439616466313665
-65653765386362383633363737316337643763653634663664363661633032303866346566656338
-63383537633038303632303331646637323566323266363035343538383962656464633338633835
-32313164663634326436616563313236316361353936336662366162393236303939333932316261
-66343931333432333766353239393261373936336534393163663864643336633364666665333731
-62386465616466636162303030663634303963663133633864363638383836306637616166326530
-62306535326565373132323739663933653162383366623931373833316363633737616364326565
-33303933633561336437663661306139663931303232323639353930353734633334323966353364
-36623036646161303135616436656364303338343130616132396436313963383638373834353463
-63636632353263656431393839356438363261353132663964646262306365383830393636613238
-36643039663333303764633161343561306439353462386638323534333563333739646336343537
-38326465393438346536343666346231343234613639373439376665653738353965353330393262
-32353464646538366631663039663530383562306430623938376231393639313932376331333735
-62343164613832623038353762303238386538663537303534616435323837386637623338343439
-31643965653266383137313635356565653635333563343435663561306262363533653363376634
-34333464316161303162313136333931383161366561303634356238613131613462386538623662
-65666664343164646135363566373565333138313666613731633366623831396261613165393364
-32663534303730633264396435613630613730316463646131326134333763363235646435366630
-39336364643933653363316562386265373132303139396532626531393166366261316465316366
-64356233326538616339303735373832666564626332636236653632333930303466653639623561
-33623730343833626363633434616365383662646266663238623166313238373364343338383765
-61363039306266336539386333633166313138663862323335343031666162333964646565616139
-62623536623438343839316366656562343631316265616431663638366130343465326437613236
-39386662613634373263326637633036623239343733643437666661363632653838333832623839
-37306361623434303432303934666639353230653631386338303337373332373439363566366631
-39363964393637646364383262633561303830336134383562343264663531643064636439323832
-63346238626632303638643639666139336364373430363966353939386262376633343061336466
-37303437343530633034353437373432393434326330643835316538303666313833363761336132
-62333834396439623936323432363730306263376166326434393835346561653937643663336261
-66623434376566653830616666363338353861323038373063663935366233333137613466353137
-33383930623532343264666239363466393565336437306532616433643031333835323930643564
-33316362313965353539393934623361373431306632616266623265623738383765643838663037
-30636635356532353564386637326539393064333636623866353032383030353064653931663233
-33636134656663323036353534643734623039663130333562353739646161313638363363393237
-33663531373965653862366666626435383165343464323463346435393238633663663237316337
-62623833303864633562663764656332323436323931393363363337666133643430633838656336
-34653934313166353864363061663164396532303565623636666430313637313830376139313764
-63343863333833343632323366616463613233336331303232363335396338323461363665343136
-61386238343931633739623134363362656436643764356130333431613465623136383161643766
-33353330303437623064363637346262353165343765643365346265633431393530613938356463
-38313166646162303332633736323035353933666563316637633731643331643639643663323331
-32616535643034396535316130636238373937643462316137646562643833343732396234333062
-33623064336561623336636431366365383664636631636434356461383333363738623861376164
-37343464323738316234313831633032316464326265363530333037633833313739663231323031
-64616537333064323364353362626161366130353662613334336635346231393666663839646135
-62636538653563336463373631626563363461323865666333306461393735366566373931616537
-64396563633663373566643637313230393430663364636662623536313264393265393065343964
-35623530643833643764646530326533353635396161613832616263383036383739643261333930
-61333132343362626163663735323231343930343639636563393764643863646261396362383238
-62373835626465626335643865663765653861613462323861316366393536383839313032303261
-34646661376534333638613764323738613438373236393534396661376465373361646662616165
-64663833366366313439303831333933306662366361353531646133623532323262616632306362
-35643066306232333035633430376165303633653764363630323263316338646530653831303934
-34316534306539303265343937366563613233366537386634636631646562653263323365663865
-35663364663762306266353534643563633365383063373531303864643938633530306538313437
-34633234323830323661383537303931336262356239646661343764393766666336306662616334
-36373231366130633438366365623539326232656234376163633935393939393934376664623863
-30636535366661366330303563326138323364613736313833376638303663316433323430336165
-32326534313533623938636164376435333965663633383236376632393564383132666238303338
-37653364346531636564306137633030373232353464356365306333613630623937323462646165
-32393233633531363863646234636138303034663333396161633162393136636635306536386238
-61616433623565313532396238393437356462316365643438313966613438303461666463653966
-37323839333136313331383464396163396666313063613332373832656532636334613332373066
-39303339333662336539353032396233313865653030613339383263313038366566623537336632
-66396637306230336563663363656163376430663331363331643438313130623764616232396632
-64353565643565663061646631383839636164393134306232363934653534356236343031383261
-36326563633134626465613532376635326139393563383065383437376433316632663165623361
-39613362373736396532333739303737613736366138633031613961313135353466306365393630
-39313863663762666361373430613963613331386336623765353865323437656630646132343761
-61393035316331393133636633393661616661373363326463643837663462376239316432323930
-38306435396538646636353265356461396166376530366436353864356538643663303230663535
-37306439373263343065346537656430643061623439313738646135623838383438333336353561
-66313330653438343937656562653338396639346464303636333736643533356262656233363164
-33376264623262376334383134303730313762313639643864376132666661303533613031373962
-36623239623466636434366662306237393062616665333234646336653863313331613662333261
-35646437316432393062313261646231356362336639643733366563643934643730
+38326435396539336139383762626634363962383031303030633464323433656634343639366564
+3139366461336631626537623866336362336337633062320a326136373934633930656538363633
+39636137336436363233643038663935386633383433353533383134636532353139303239326332
+3465626434646334620a356161303939666163306564663937303532333363376631626463613132
+64336337663939633538346165653839383366643137666635323239316431353332386661306436
+64323239386263323865643232633635323264613863636465653062373937383036366233383035
+32363061333938656639626432333538343130656165653330646461663665386264356136343564
+61643730353136616533613536313539386636366262666439323666393661373030303637636537
+32623634353830326337346131613739306336656562313339363230353630633334363635333566
+38303462663132323635616138363435633766623461663962376262623130646637636364386563
+33636230383939336531363562323065363634353731373662663639373734356362636530313166
+65656333383739356564363365643337633230616563366461386439376238363062373637393762
+37356566326531303939333664396133326634366461333632636664626261333064343335646263
+32663335663665316538313662383039626433613739363230643561313734316263626464396133
+33303335353832306232656664636437663163373535666136643635663930663562313065363062
+38383837396631333939363539363237333261313330373464303339633961356239386132313763
+36393566663535633138616137373266366465616537353461623566643961326530626366383631
+35316336303562643439373166646639323837336531373133393464323236353235663235346161
+35323634373135313865303133646366356630363034386235656531386465383430316432343932
+62376537323962373935313830613437386138396339643437633966363930366339396462353736
+61336631373062343439316261383265356135643238363036343435303932353033393431616634
+39376464396432633635616133323435633461393537656634663139653032313065626538383961
+30316638633065383836353061336437663462643765613863653863313863613830613531643030
+30653133373365313530316637653838636237336532313835363838616639366331303266663632
+62646561656231653536626231323364313836643633623132613239626433323433366138636332
+32653264643736626462323065393163373261663561373236306133646337386335656262313262
+30346265303161323265363431376266313864653763306661316632623333306531303732326561
+63313537636462623366376235373237636536366238613434363737616465636162313832633738
+61323732373566356133646566316536393931303439303863656361616664366133306435633237
+66316339383537376130623537303563333331353937623536376138646566303564303632343363
+63633561643465333331623736353263616438376431376431356538373766356465616130393430
+31646539306564343734303664616132376130313164663431333566366165643132303632373063
+62326463386363656462363830656666336365396165643235613436353665323230313765383266
+34383433343166316366363665333762376463623334626565613530653035646565353135333736
+65643632366565363636613765356162376261643633346661613636323435306135353330386262
+36306165653165333732343032636565366662613131323332306666633835613631393662666633
+65393735333734356539323663373230366265343138646565663631303332323361663435343331
+30623935616366643837383138393764393763663763646238303865613939303965376639663465
+30653432333332643135613934346361613332323732643765656333366135623232353535663862
+63353461343638313738316234313961353864643832383265633561306136333966633338373932
+31393237316138376531363836333239353066353561633539613636643562373537316666306434
+34343339396435646262373134323130353431623435383137373761616164353662313138316235
+66646134616237353031356339643064343936336433663938353834633861613762323862396533
+64613333306436353839613231373736333331323266643532396335663336663530396138393861
+32613163646335343564666634316636353437656466393431356334356164393861363562663836
+38656430333339303161306134303637336535613233653634303235383136393931376262613738
+65343337306433316339616632373032363835646439373934616662383566303063393538383334
+61356236313762326262316265613266626533643238626630663162386532333264356133663366
+35383539353962633062656463616537363137616130353465313639663037336565363831326139
+35313363326634336361303834373764363666383834346365633563373762363965646533643062
+65623130626537653836623262343832323165323539356130616461356131353636316465366662
+39623432356432326330303766316365323237653064646565343833396435333134623635356439
+34613637333939393432616261383439636238323566353138626631643937666165343330663761
+66663064653139663633303736343433633363633062653339666331373862393732303761383639
+37373735643432303366663238383562396534356433663439376563366639613934306430653633
+65626233636639613332633435363066393862363232666236383830646438646463626539393838
+66323364643265333336633263383433656330353033343033356565366361323862353439356435
+34616633666439666661653630616630373032623235336430326366306430633732666337313435
+30656665323736313363663036646462646430663266316231653238643133643134363639613435
+63353239353735653438323964386135326663323939306534306663393931363163623831626631
+31643132376463643633643163613862633832313535343561323734373733623435336339613939
+39306432313638613235386361343738663633613133643964353163366435646337396231656463
+39393435303232393763343237393661626334626263343038343237353736343431306431646432
+66623732386561316436396162383130663065303531313935633864323261363666383364363130
+35303764633265386538383263363039383866376234373536376563366537343833626235666462
+38356666313964666139346632363733396439626363313665643135663061316166636339376230
+35366132613631646130393135613030623962306538323134643539323133393934636666643762
+64326636363865323936636135383565656466663037346235313230366164313361386635316562
+63353038316430373737616231353262653033343730643363353530643936333166313866323933
+63363562346332396164363330346162366333663736643761316361653039343763626133323030
+33333831323363316333663961343661373666353065616165663232346230393966633063366263
+30636362666463643036326562313833653431663262373432383232396561303961333536613762
+62343032646164303762333635376465643735353038303337633865663830643133613039636432
+66393762346336323339383233393539353739663961346362656161306238353766343930656363
+62663830323833326366353539623863333962363366393137363831616537613932346534393764
+35643239373330313866393639393364343939313931633339663932376461613039396430626263
+32666238393565323562663336363339303235333064656130313434643934663566306563626166
+66343032616463393666366361653631373661363938656462373364656636303638396532396365
+30616539656364373032396231303634653261323433333734646262323639323330616264346539
+30623965363335323164646263346332353566396366306639303265373863663062653732386339
+31643862653134613532616236313730366431353962396133613565363564326463616131613739
+37626331643633623934646465653237326366623734623433363534386661343537663736386133
+61383961643066646534653331626330623831383935383361353630383236393632333162343861
+35333533343762643933363234643836333138643566636236663661376562616133626330326435
+37653933313537353562336237653732653431626266663332653764666538363335313161353539
+30633961306337656636663435303535613234303661353438353364323331643861396163393365
+63396433643661373138636664313237633762333731393963383763626631363163633430323633
+35636666383866633231303538346438363932326665323231376162643335636639613866633639
+66333736313233393362636165396233306633386530313737376236613432313833386332613936
+35643633383830346233616162353435323238386364323663626537393738623032323032303638
+61316161356238666235343861373531613236363437613131653761356234643938306430336463
+39373537643431323138

group_vars/gitlab_runners.yml

+gitlab_runner_exporter_port: 9252
+
 fail2ban_jails:
   sshd: true
   postfix: false

hosts

@@ -100,3 +100,13 @@ aur-dev.archlinux.org
 
 [prometheus]
 monitoring.archlinux.org
+
+[node_exporters]
+aur.archlinux.org
+monitoring.archlinux.org
+gitlab.archlinux.org
+reproducible.archlinux.org
+runner1.archlinux.org
+runner2.archlinux.org
+secure-runner1.archlinux.org
+secure-runner2.archlinux.org

playbooks/aur.archlinux.org.yml

@@ -8,6 +8,7 @@
     - { role: tools }
     - { role: sshd, sshd_enable_includes: true }
     - { role: root_ssh }
+    - { role: prometheus_exporters }
     - { role: certbot }
     - { role: nginx }
     - { role: mariadb, mariadb_innodb_buffer_pool_size: '64M', mariadb_table_open_cache: '256', mariadb_query_cache_type: '0',

playbooks/gitlab-runners.yml

@@ -9,4 +9,5 @@
     - { role: sshd }
     - { role: root_ssh }
     - { role: fail2ban }
+    - { role: prometheus_exporters }
     - { role: gitlab_runner }

playbooks/gitlab.archlinux.org.yml

@@ -11,3 +11,4 @@
     - { role: root_ssh }
     - { role: gitlab, gitlab_domain: "gitlab.archlinux.org" }
     - { role: borg_client, tags: ["borg"] }
+    - { role: prometheus_exporters }

playbooks/monitoring.archlinux.org.yml

@@ -10,5 +10,6 @@
     - { role: hardening }
     - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
     - { role: prometheus }
+    - { role: prometheus_exporters }
     - { role: certbot }
     - { role: nginx }

playbooks/reproducible.archlinux.org.yml

@@ -14,3 +14,4 @@
     - { role: certbot }
     - { role: nginx }
     - { role: rebuilderd }
+    - { role: prometheus_exporters }

roles/gitlab_runner/tasks/main.yml

@@ -40,5 +40,12 @@
     line: concurrent = 100
   notify: restart gitlab-runner
 
+- name: enable prometheus exporter
+  lineinfile:
+    path: /etc/gitlab-runner/config.toml
+    insertbefore: '^concurrent'
+    line: listen_address = ":{{ gitlab_runner_exporter_port }}"
+  notify: restart gitlab-runner
+
 - name: enable and start gitlab runner service
   systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes

roles/prometheus/defaults/main.yml

 monitoring_domain: monitoring.archlinux.org
+gitlab_runner_exporter_port: '9252'

roles/prometheus/templates/prometheus.yml.j2

@@ -13,3 +13,69 @@ alerting:
        - localhost:9093
 
 scrape_configs:
+  - job_name: 'node_exporter'
+    static_configs:
+    {% for host in groups['node_exporters'] %}
+
+    - targets: ['{{ host }}:{{ prometheus_exporter_port }}']
+      labels:
+        instance: "{{ host }}"
+
+    {% endfor %}
+
+  - job_name: 'gitlab_runner_exporter'
+    static_configs:
+    {% for host in groups['gitlab_runners'] %}
+
+    - targets: ['{{ host }}:{{ gitlab_runner_exporter_port }}']
+      labels:
+        instance: "{{ host }}"
+
+    {% endfor %}
+
+  - job_name: 'keycloak'
+    scheme: https
+    metrics_path: "/auth/realms/master/metrics"
+    basic_auth:
+       username: "{{ vault_keycloak_nginx_user }}"
+       password: "{{ vault_keycloak_nginx_passwd }}"
+    static_configs:
+    - targets: ['accounts.archlinux.org:443']
+      labels:
+        instance: "accounts.archlinux.org"
+
+  - job_name: 'gitlab_exporter'
+    scheme: https
+    metrics_path: "-/metrics"
+    params:
+      token: ["{{ vault_gitlab_prometheus_token }}"]
+    static_configs:
+    - targets: ['gitlab.archlinux.org:443']
+      labels:
+        instance: "gitlab.archlinux.org"
+
+  - job_name: 'mysqld_exporter'
+    static_configs:
+
+    - targets: ['aur.archlinux.org:9104']
+      labels:
+        instance: "aur.archlinux.org"
+
+  - job_name: 'blackbox'
+    metrics_path: /probe
+    scrape_interval: 15s
+    params:
+      module: [http_prometheus]
+    static_configs:
+    - targets:
+    {% for target in blackbox_targets %}
+      - {{ target }}
+    {% endfor %}
+
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115

roles/prometheus_exporters/defaults/main.yml

+---
+
+prometheus_domain: monitoring.archlinux.org
+
+prometheus_textfile_dir: /var/lib/node_exporter
+
+gitlab_runner_exporter_port: '9252'
+
+prometheus_mysqld_user: mysqld_exporter
+prometheus_mysqld_exporter_port: '9104'

roles/prometheus_exporters/files/arch-textcollector.sh

+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+HOSTNAME=$(hostname)
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/pacman.prom
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+updates=$(/usr/bin/checkupdates | wc -l)
+secupdates=$(/usr/bin/arch-audit -u | wc -l)
+
+echo "# HELP pacman_updates_pending number of pending updates from pacman" >> $TMP_FILE
+echo "# TYPE pacman_updates_pending gauge" >> $TMP_FILE
+echo "pacman_updates_pending{host=\"${HOSTNAME}\"} $updates" >> $TMP_FILE
+
+echo "# HELP pacman_security_updates_pending number of pending updates from pacman" >> $TMP_FILE
+echo "# TYPE pacman_security_updates_pending gauge" >> $TMP_FILE
+echo "pacman_security_updates_pending{host=\"${HOSTNAME}\"} $secupdates" >> $TMP_FILE
+
+mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/files/borg-textcollector.sh

+#!/usr/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+HOSTNAME=$(hostname)
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/borg.prom
+
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+# Hetzner borg
+if [[ -f /usr/local/bin/borg ]]; then
+  LAST_ARCHIVE=$(/usr/local/bin/borg list --last 1)
+  LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}')
+  LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}')
+  LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s")
+
+  echo "# HELP borg_hetzner_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE
+  echo "# TYPE borg_hetzner_last_archive_timestamp counter" >> $TMP_FILE
+  echo "borg_hetzner_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE;
+fi
+
+# rsync.net borg
+if [[ -f /usr/local/bin/borg-offsite ]]; then
+  LAST_ARCHIVE=$(/usr/local/bin/borg-offsite list --last 1)
+  LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}')
+  LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}')
+  LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s")
+
+  echo "# HELP borg_offsite_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE
+  echo "# TYPE borg_offsite_last_archive_timestamp counter" >> $TMP_FILE
+  echo "borg_offsite_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE;
+fi
+
+mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/tasks/main.yml

+---
+
+- name: install prometheus-node-exporter
+  pacman: name=prometheus-node-exporter,arch-audit,pacman-contrib state=present
+
+- name: install prometheus-memcached-exporter
+  pacman: name=prometheus-memcached-exporter state=present
+  when: "'memcached' in group_names"
+
+- name: install prometheus-mysqld-exporter
+  pacman: name=prometheus-mysqld-exporter state=present
+  when: "'mysql_servers' in group_names"
+
+- name: create prometheus mysqld database user
+  mysql_user:
+    name: '{{ prometheus_mysqld_user }}'
+    password: '{{ vault_prometheus_mysql_password }}'
+    priv: "*.*:PROCESS,REPLICATION CLIENT"
+    state: present
+  when: "'mysql_servers' in group_names"
+
+# TODO: with ansible 2.10 this can be set by mysql_user https://github.com/ansible/ansible/issues/26581
+- name: set max_user_connections for prometheus mysqld user
+  command: mysql -u root -e "ALTER USER '{{ prometheus_mysqld_user }}'@'localhost' WITH MAX_USER_CONNECTIONS 3;"
+  when: "'mysql_servers' in group_names"
+
+- name: copy prometheus mysqld exporter configuration
+  template: src=prometheus-mysqld-exporter.j2 dest=/etc/conf.d/prometheus-mysqld-exporter owner=root group=root mode=600
+  when: "'mysql_servers' in group_names"
+
+- name: enable prometheus-mysqld-exporter service
+  systemd: name=prometheus-mysqld-exporter enabled=yes daemon_reload=yes state=started
+  when: "'mysql_servers' in group_names"
+
+- name: install node exporter configuration
+  template: src=prometheus-node-exporter.env.j2 dest=/etc/conf.d/prometheus-node-exporter owner=root group=root mode=600
+
+- name: create textcollector directory
+  file: path="{{ prometheus_textfile_dir }}" state=directory owner=node_exporter group=node_exporter mode=700
+
+- name: install node exporter textcollector scripts
+  copy: src={{ item }} dest=/usr/local/bin/{{ item }} owner=root group=root mode=0755
+  with_items:
+    - arch-textcollector.sh
+    - borg-textcollector.sh
+
+- name: install arch textcollector service
+  template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600
+
+- name: install arch textcollector timer
+  template: src=prometheus-arch-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.timer owner=root group=root mode=600
+
+- name: enable and start prometheus arch textcollector timer
+  systemd: name=prometheus-arch-textcollector.timer enabled=yes daemon_reload=yes state=started
+
+- name: install borg textcollector service
+  template: src=prometheus-borg-textcollector.service.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.service owner=root group=root mode=600
+  when: "'borg_clients' in group_names"
+
+- name: install borg textcollector timer
+  template: src=prometheus-borg-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.timer owner=root group=root mode=600
+  when: "'borg_clients' in group_names"
+
+- name: enable and start prometheus borg textcollector timer
+  systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "'borg_clients' in group_names"
+
+- name: enable prometheus-node-exporter service
+  systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started
+
+- name: enable prometheus-memcached-exporter service
+  systemd: name=prometheus-memcached-exporter enabled=yes daemon_reload=yes state=started
+  when: "'memcached' in group_names"
+
+- name: open prometheus-node-exporter ipv4 port for monitoring.archlinux.org
+  firewalld: state=enabled permanent=true immediate=yes
+        rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_exporter_port }} accept"
+  when: "'prometheus' not in group_names"
+
+- name: open gitlab exporter ipv4 port for monitoring.archlinux.org
+  firewalld: state=enabled permanent=true immediate=yes
+        rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ gitlab_runner_exporter_port }} accept"
+  when: "'gitlab_runners' in group_names"
+
+- name: open prometheus mysqld exporter ipv4 port for monitoring.archlinux.org
+  firewalld: state=enabled permanent=true immediate=yes
+        rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_mysqld_exporter_port }} accept"
+  when: "'mysql_servers' in group_names"

roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2

+[Unit]
+Description=Prometheus Arch Exporter
+After=network.target
+
+[Service]
+Type=oneshot
+User=node_exporter
+ExecStart=/usr/local/bin/arch-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+CapabilityBoundingSet=
+UMask=077
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native

roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2

+[Unit]
+Description=Prometheus Arch Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=60m
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target

roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2

+[Unit]
+Description=Prometheus Borg Exporter TextCollector
+After=network.target
+ConditionPathExistsGlob=!/root/.cache/borg/*/lock.roster
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/borg-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths={{ prometheus_textfile_dir }} /root/.cache/borg
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_PACKET
+RestrictAddressFamilies=~AF_NETLINK
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native

roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2

+[Unit]
+Description=Prometheus Borg Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=1h
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target