diff --git a/group_vars/all/common.yml b/group_vars/all/common.yml
index 56a6bf8708642e1f99ca0f558309c47996f9ae93..664f5ddce40ea5d4f07c98ed76f44e2188e5517f 100644
--- a/group_vars/all/common.yml
+++ b/group_vars/all/common.yml
@@ -10,3 +10,6 @@ zabbix_agent_templates:
 # this is used by the maintenance role to get the ip address
 # of the machine running the playbook
 maintenance_remote_machine: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CLIENT.split(' ')[0] }}"
+
+# prometheus-node-exporter port
+prometheus_exporter_port: '9100'
diff --git a/group_vars/all/vault_gitlab.yml b/group_vars/all/vault_gitlab.yml
index 8f4e0e010d353530c52a22796577cc02d2f98d18..64145b7e620d1f24c60355cc0a8ee03181e64a6b 100644
--- a/group_vars/all/vault_gitlab.yml
+++ b/group_vars/all/vault_gitlab.yml
@@ -1,86 +1,93 @@
 $ANSIBLE_VAULT;1.1;AES256
-35393938326563366437646365633563303031393034626433333163373838613535333136356132
-3139393330383337376633313739643431636337343263310a616263613665356437383862663134
-34643230613832643332323634613561313634386636373937373533653338313030633339653235
-6330646665656530350a623165393139306136636433356262626534376436313539646664336266
-30353562633266383138373334656238396464326231313132643461313763356537386535326462
-64643862383638373930373632663462663865346664636635653937303837656239653739343266
-31346133383133656665363034653330633735663862393930303862306566643533633635343935
-31376266356139376630316566613732663231393136316163356433666366613466653664643162
-62393833303265303539333637613662346465326131656139663633386439633032313138353637
-32663337656661303264633862333530336430306433613032353263666365396439616466313665
-65653765386362383633363737316337643763653634663664363661633032303866346566656338
-63383537633038303632303331646637323566323266363035343538383962656464633338633835
-32313164663634326436616563313236316361353936336662366162393236303939333932316261
-66343931333432333766353239393261373936336534393163663864643336633364666665333731
-62386465616466636162303030663634303963663133633864363638383836306637616166326530
-62306535326565373132323739663933653162383366623931373833316363633737616364326565
-33303933633561336437663661306139663931303232323639353930353734633334323966353364
-36623036646161303135616436656364303338343130616132396436313963383638373834353463
-63636632353263656431393839356438363261353132663964646262306365383830393636613238
-36643039663333303764633161343561306439353462386638323534333563333739646336343537
-38326465393438346536343666346231343234613639373439376665653738353965353330393262
-32353464646538366631663039663530383562306430623938376231393639313932376331333735
-62343164613832623038353762303238386538663537303534616435323837386637623338343439
-31643965653266383137313635356565653635333563343435663561306262363533653363376634
-34333464316161303162313136333931383161366561303634356238613131613462386538623662
-65666664343164646135363566373565333138313666613731633366623831396261613165393364
-32663534303730633264396435613630613730316463646131326134333763363235646435366630
-39336364643933653363316562386265373132303139396532626531393166366261316465316366
-64356233326538616339303735373832666564626332636236653632333930303466653639623561
-33623730343833626363633434616365383662646266663238623166313238373364343338383765
-61363039306266336539386333633166313138663862323335343031666162333964646565616139
-62623536623438343839316366656562343631316265616431663638366130343465326437613236
-39386662613634373263326637633036623239343733643437666661363632653838333832623839
-37306361623434303432303934666639353230653631386338303337373332373439363566366631
-39363964393637646364383262633561303830336134383562343264663531643064636439323832
-63346238626632303638643639666139336364373430363966353939386262376633343061336466
-37303437343530633034353437373432393434326330643835316538303666313833363761336132
-62333834396439623936323432363730306263376166326434393835346561653937643663336261
-66623434376566653830616666363338353861323038373063663935366233333137613466353137
-33383930623532343264666239363466393565336437306532616433643031333835323930643564
-33316362313965353539393934623361373431306632616266623265623738383765643838663037
-30636635356532353564386637326539393064333636623866353032383030353064653931663233
-33636134656663323036353534643734623039663130333562353739646161313638363363393237
-33663531373965653862366666626435383165343464323463346435393238633663663237316337
-62623833303864633562663764656332323436323931393363363337666133643430633838656336
-34653934313166353864363061663164396532303565623636666430313637313830376139313764
-63343863333833343632323366616463613233336331303232363335396338323461363665343136
-61386238343931633739623134363362656436643764356130333431613465623136383161643766
-33353330303437623064363637346262353165343765643365346265633431393530613938356463
-38313166646162303332633736323035353933666563316637633731643331643639643663323331
-32616535643034396535316130636238373937643462316137646562643833343732396234333062
-33623064336561623336636431366365383664636631636434356461383333363738623861376164
-37343464323738316234313831633032316464326265363530333037633833313739663231323031
-64616537333064323364353362626161366130353662613334336635346231393666663839646135
-62636538653563336463373631626563363461323865666333306461393735366566373931616537
-64396563633663373566643637313230393430663364636662623536313264393265393065343964
-35623530643833643764646530326533353635396161613832616263383036383739643261333930
-61333132343362626163663735323231343930343639636563393764643863646261396362383238
-62373835626465626335643865663765653861613462323861316366393536383839313032303261
-34646661376534333638613764323738613438373236393534396661376465373361646662616165
-64663833366366313439303831333933306662366361353531646133623532323262616632306362
-35643066306232333035633430376165303633653764363630323263316338646530653831303934
-34316534306539303265343937366563613233366537386634636631646562653263323365663865
-35663364663762306266353534643563633365383063373531303864643938633530306538313437
-34633234323830323661383537303931336262356239646661343764393766666336306662616334
-36373231366130633438366365623539326232656234376163633935393939393934376664623863
-30636535366661366330303563326138323364613736313833376638303663316433323430336165
-32326534313533623938636164376435333965663633383236376632393564383132666238303338
-37653364346531636564306137633030373232353464356365306333613630623937323462646165
-32393233633531363863646234636138303034663333396161633162393136636635306536386238
-61616433623565313532396238393437356462316365643438313966613438303461666463653966
-37323839333136313331383464396163396666313063613332373832656532636334613332373066
-39303339333662336539353032396233313865653030613339383263313038366566623537336632
-66396637306230336563663363656163376430663331363331643438313130623764616232396632
-64353565643565663061646631383839636164393134306232363934653534356236343031383261
-36326563633134626465613532376635326139393563383065383437376433316632663165623361
-39613362373736396532333739303737613736366138633031613961313135353466306365393630
-39313863663762666361373430613963613331386336623765353865323437656630646132343761
-61393035316331393133636633393661616661373363326463643837663462376239316432323930
-38306435396538646636353265356461396166376530366436353864356538643663303230663535
-37306439373263343065346537656430643061623439313738646135623838383438333336353561
-66313330653438343937656562653338396639346464303636333736643533356262656233363164
-33376264623262376334383134303730313762313639643864376132666661303533613031373962
-36623239623466636434366662306237393062616665333234646336653863313331613662333261
-35646437316432393062313261646231356362336639643733366563643934643730
+38326435396539336139383762626634363962383031303030633464323433656634343639366564
+3139366461336631626537623866336362336337633062320a326136373934633930656538363633
+39636137336436363233643038663935386633383433353533383134636532353139303239326332
+3465626434646334620a356161303939666163306564663937303532333363376631626463613132
+64336337663939633538346165653839383366643137666635323239316431353332386661306436
+64323239386263323865643232633635323264613863636465653062373937383036366233383035
+32363061333938656639626432333538343130656165653330646461663665386264356136343564
+61643730353136616533613536313539386636366262666439323666393661373030303637636537
+32623634353830326337346131613739306336656562313339363230353630633334363635333566
+38303462663132323635616138363435633766623461663962376262623130646637636364386563
+33636230383939336531363562323065363634353731373662663639373734356362636530313166
+65656333383739356564363365643337633230616563366461386439376238363062373637393762
+37356566326531303939333664396133326634366461333632636664626261333064343335646263
+32663335663665316538313662383039626433613739363230643561313734316263626464396133
+33303335353832306232656664636437663163373535666136643635663930663562313065363062
+38383837396631333939363539363237333261313330373464303339633961356239386132313763
+36393566663535633138616137373266366465616537353461623566643961326530626366383631
+35316336303562643439373166646639323837336531373133393464323236353235663235346161
+35323634373135313865303133646366356630363034386235656531386465383430316432343932
+62376537323962373935313830613437386138396339643437633966363930366339396462353736
+61336631373062343439316261383265356135643238363036343435303932353033393431616634
+39376464396432633635616133323435633461393537656634663139653032313065626538383961
+30316638633065383836353061336437663462643765613863653863313863613830613531643030
+30653133373365313530316637653838636237336532313835363838616639366331303266663632
+62646561656231653536626231323364313836643633623132613239626433323433366138636332
+32653264643736626462323065393163373261663561373236306133646337386335656262313262
+30346265303161323265363431376266313864653763306661316632623333306531303732326561
+63313537636462623366376235373237636536366238613434363737616465636162313832633738
+61323732373566356133646566316536393931303439303863656361616664366133306435633237
+66316339383537376130623537303563333331353937623536376138646566303564303632343363
+63633561643465333331623736353263616438376431376431356538373766356465616130393430
+31646539306564343734303664616132376130313164663431333566366165643132303632373063
+62326463386363656462363830656666336365396165643235613436353665323230313765383266
+34383433343166316366363665333762376463623334626565613530653035646565353135333736
+65643632366565363636613765356162376261643633346661613636323435306135353330386262
+36306165653165333732343032636565366662613131323332306666633835613631393662666633
+65393735333734356539323663373230366265343138646565663631303332323361663435343331
+30623935616366643837383138393764393763663763646238303865613939303965376639663465
+30653432333332643135613934346361613332323732643765656333366135623232353535663862
+63353461343638313738316234313961353864643832383265633561306136333966633338373932
+31393237316138376531363836333239353066353561633539613636643562373537316666306434
+34343339396435646262373134323130353431623435383137373761616164353662313138316235
+66646134616237353031356339643064343936336433663938353834633861613762323862396533
+64613333306436353839613231373736333331323266643532396335663336663530396138393861
+32613163646335343564666634316636353437656466393431356334356164393861363562663836
+38656430333339303161306134303637336535613233653634303235383136393931376262613738
+65343337306433316339616632373032363835646439373934616662383566303063393538383334
+61356236313762326262316265613266626533643238626630663162386532333264356133663366
+35383539353962633062656463616537363137616130353465313639663037336565363831326139
+35313363326634336361303834373764363666383834346365633563373762363965646533643062
+65623130626537653836623262343832323165323539356130616461356131353636316465366662
+39623432356432326330303766316365323237653064646565343833396435333134623635356439
+34613637333939393432616261383439636238323566353138626631643937666165343330663761
+66663064653139663633303736343433633363633062653339666331373862393732303761383639
+37373735643432303366663238383562396534356433663439376563366639613934306430653633
+65626233636639613332633435363066393862363232666236383830646438646463626539393838
+66323364643265333336633263383433656330353033343033356565366361323862353439356435
+34616633666439666661653630616630373032623235336430326366306430633732666337313435
+30656665323736313363663036646462646430663266316231653238643133643134363639613435
+63353239353735653438323964386135326663323939306534306663393931363163623831626631
+31643132376463643633643163613862633832313535343561323734373733623435336339613939
+39306432313638613235386361343738663633613133643964353163366435646337396231656463
+39393435303232393763343237393661626334626263343038343237353736343431306431646432
+66623732386561316436396162383130663065303531313935633864323261363666383364363130
+35303764633265386538383263363039383866376234373536376563366537343833626235666462
+38356666313964666139346632363733396439626363313665643135663061316166636339376230
+35366132613631646130393135613030623962306538323134643539323133393934636666643762
+64326636363865323936636135383565656466663037346235313230366164313361386635316562
+63353038316430373737616231353262653033343730643363353530643936333166313866323933
+63363562346332396164363330346162366333663736643761316361653039343763626133323030
+33333831323363316333663961343661373666353065616165663232346230393966633063366263
+30636362666463643036326562313833653431663262373432383232396561303961333536613762
+62343032646164303762333635376465643735353038303337633865663830643133613039636432
+66393762346336323339383233393539353739663961346362656161306238353766343930656363
+62663830323833326366353539623863333962363366393137363831616537613932346534393764
+35643239373330313866393639393364343939313931633339663932376461613039396430626263
+32666238393565323562663336363339303235333064656130313434643934663566306563626166
+66343032616463393666366361653631373661363938656462373364656636303638396532396365
+30616539656364373032396231303634653261323433333734646262323639323330616264346539
+30623965363335323164646263346332353566396366306639303265373863663062653732386339
+31643862653134613532616236313730366431353962396133613565363564326463616131613739
+37626331643633623934646465653237326366623734623433363534386661343537663736386133
+61383961643066646534653331626330623831383935383361353630383236393632333162343861
+35333533343762643933363234643836333138643566636236663661376562616133626330326435
+37653933313537353562336237653732653431626266663332653764666538363335313161353539
+30633961306337656636663435303535613234303661353438353364323331643861396163393365
+63396433643661373138636664313237633762333731393963383763626631363163633430323633
+35636666383866633231303538346438363932326665323231376162643335636639613866633639
+66333736313233393362636165396233306633386530313737376236613432313833386332613936
+35643633383830346233616162353435323238386364323663626537393738623032323032303638
+61316161356238666235343861373531613236363437613131653761356234643938306430336463
+39373537643431323138
diff --git a/group_vars/gitlab_runners.yml b/group_vars/gitlab_runners.yml
index 1c98fb00e6ee21fa0f9948475bea59c92a9eb800..80777b00c8970bc60cf98c6121d5cbe5d768fa10 100644
--- a/group_vars/gitlab_runners.yml
+++ b/group_vars/gitlab_runners.yml
@@ -1,3 +1,5 @@
+gitlab_runner_exporter_port: 9252
+
 fail2ban_jails:
   sshd: true
   postfix: false
diff --git a/hosts b/hosts
index 43a89c29524d41c5a2d6c7eecacda6cf8e1299ae..bc91b96b014eb2dd451e59d03ee7734f82af732a 100644
--- a/hosts
+++ b/hosts
@@ -100,3 +100,13 @@ aur-dev.archlinux.org
 
 [prometheus]
 monitoring.archlinux.org
+
+[node_exporters]
+aur.archlinux.org
+monitoring.archlinux.org
+gitlab.archlinux.org
+reproducible.archlinux.org
+runner1.archlinux.org
+runner2.archlinux.org
+secure-runner1.archlinux.org
+secure-runner2.archlinux.org
diff --git a/playbooks/aur.archlinux.org.yml b/playbooks/aur.archlinux.org.yml
index e2fcabf57c54961a8c6056f66821b4a6226bbc44..6a13585a289c71ce27277ded3cf9ada8d6bcd7eb 100644
--- a/playbooks/aur.archlinux.org.yml
+++ b/playbooks/aur.archlinux.org.yml
@@ -8,6 +8,7 @@
     - { role: tools }
     - { role: sshd, sshd_enable_includes: true }
     - { role: root_ssh }
+    - { role: prometheus_exporters }
     - { role: certbot }
     - { role: nginx }
     - { role: mariadb, mariadb_innodb_buffer_pool_size: '64M', mariadb_table_open_cache: '256', mariadb_query_cache_type: '0',
diff --git a/playbooks/gitlab-runners.yml b/playbooks/gitlab-runners.yml
index b2d09a3713d6377f02a4348ec161b92f42cdfc4f..3980aed0be513d5b2f8e0cb5293f7ad87f4c8e05 100644
--- a/playbooks/gitlab-runners.yml
+++ b/playbooks/gitlab-runners.yml
@@ -9,4 +9,5 @@
     - { role: sshd }
     - { role: root_ssh }
     - { role: fail2ban }
+    - { role: prometheus_exporters }
     - { role: gitlab_runner }
diff --git a/playbooks/gitlab.archlinux.org.yml b/playbooks/gitlab.archlinux.org.yml
index 3f47c753c1622c7737a8a865805a8ddf0ce6e879..d5e7eeeeed83b4388870f660fe45644b2f294b15 100644
--- a/playbooks/gitlab.archlinux.org.yml
+++ b/playbooks/gitlab.archlinux.org.yml
@@ -11,3 +11,4 @@
     - { role: root_ssh }
     - { role: gitlab, gitlab_domain: "gitlab.archlinux.org" }
     - { role: borg_client, tags: ["borg"] }
+    - { role: prometheus_exporters }
diff --git a/playbooks/monitoring.archlinux.org.yml b/playbooks/monitoring.archlinux.org.yml
index ec5aa7945277c62840f1ccf51783f3a874ac00b4..2fc94d7b286c6ec343bd6cd826b01d194713294f 100644
--- a/playbooks/monitoring.archlinux.org.yml
+++ b/playbooks/monitoring.archlinux.org.yml
@@ -10,5 +10,6 @@
     - { role: hardening }
     - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
     - { role: prometheus }
+    - { role: prometheus_exporters }
     - { role: certbot }
     - { role: nginx }
diff --git a/playbooks/reproducible.archlinux.org.yml b/playbooks/reproducible.archlinux.org.yml
index d194e102df544760cefe6eaf7510748bca2c10e8..ce800050a9201ea66f561a550262725f0a9f8f70 100644
--- a/playbooks/reproducible.archlinux.org.yml
+++ b/playbooks/reproducible.archlinux.org.yml
@@ -14,3 +14,4 @@
     - { role: certbot }
     - { role: nginx }
     - { role: rebuilderd }
+    - { role: prometheus_exporters }
diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml
index c5e18fab1998a96f2988a348b9d63dffc19aee20..f1a70b46cb7035311966dd3974cf9de978016026 100644
--- a/roles/gitlab_runner/tasks/main.yml
+++ b/roles/gitlab_runner/tasks/main.yml
@@ -40,5 +40,12 @@
     line: concurrent = 100
   notify: restart gitlab-runner
 
+- name: enable prometheus exporter
+  lineinfile:
+    path: /etc/gitlab-runner/config.toml
+    insertbefore: '^concurrent'
+    line: listen_address = ":{{ gitlab_runner_exporter_port }}"
+  notify: restart gitlab-runner
+
 - name: enable and start gitlab runner service
   systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
index 3b75be8a27d4a8258a72c033aa01383f4ae32fa7..47660d86680e1f8ed3b1adc6c5074c317e281418 100644
--- a/roles/prometheus/defaults/main.yml
+++ b/roles/prometheus/defaults/main.yml
@@ -1 +1,2 @@
 monitoring_domain: monitoring.archlinux.org
+gitlab_runner_exporter_port: '9252'
diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2
index c868c7545ac9888ec7d53983aa17b6f6c6d200c0..1da101edbc1922fb856e830fac10aa9579e1aa05 100644
--- a/roles/prometheus/templates/prometheus.yml.j2
+++ b/roles/prometheus/templates/prometheus.yml.j2
@@ -13,3 +13,69 @@ alerting:
        - localhost:9093
 
 scrape_configs:
+  - job_name: 'node_exporter'
+    static_configs:
+    {% for host in groups['node_exporters'] %}
+
+    - targets: ['{{ host }}:{{ prometheus_exporter_port }}']
+      labels:
+        instance: "{{ host }}"
+
+    {% endfor %}
+
+  - job_name: 'gitlab_runner_exporter'
+    static_configs:
+    {% for host in groups['gitlab_runners'] %}
+
+    - targets: ['{{ host }}:{{ gitlab_runner_exporter_port }}']
+      labels:
+        instance: "{{ host }}"
+
+    {% endfor %}
+
+  - job_name: 'keycloak'
+    scheme: https
+    metrics_path: "/auth/realms/master/metrics"
+    basic_auth:
+       username: "{{ vault_keycloak_nginx_user }}"
+       password: "{{ vault_keycloak_nginx_passwd }}"
+    static_configs:
+    - targets: ['accounts.archlinux.org:443']
+      labels:
+        instance: "accounts.archlinux.org"
+
+  - job_name: 'gitlab_exporter'
+    scheme: https
+    metrics_path: "-/metrics"
+    params:
+      token: ["{{ vault_gitlab_prometheus_token }}"]
+    static_configs:
+    - targets: ['gitlab.archlinux.org:443']
+      labels:
+        instance: "gitlab.archlinux.org"
+
+  - job_name: 'mysqld_exporter'
+    static_configs:
+
+    - targets: ['aur.archlinux.org:9104']
+      labels:
+        instance: "aur.archlinux.org"
+
+  - job_name: 'blackbox'
+    metrics_path: /probe
+    scrape_interval: 15s
+    params:
+      module: [http_prometheus]
+    static_configs:
+    - targets:
+    {% for target in blackbox_targets %}
+      - {{ target }}
+    {% endfor %}
+
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115
diff --git a/roles/prometheus_exporters/defaults/main.yml b/roles/prometheus_exporters/defaults/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b66b5c4602da8ff860fd2d17f6aa9d73816a54f8
--- /dev/null
+++ b/roles/prometheus_exporters/defaults/main.yml
@@ -0,0 +1,10 @@
+---
+
+prometheus_domain: monitoring.archlinux.org
+
+prometheus_textfile_dir: /var/lib/node_exporter
+
+gitlab_runner_exporter_port: '9252'
+
+prometheus_mysqld_user: mysqld_exporter
+prometheus_mysqld_exporter_port: '9104'
diff --git a/roles/prometheus_exporters/files/arch-textcollector.sh b/roles/prometheus_exporters/files/arch-textcollector.sh
new file mode 100755
index 0000000000000000000000000000000000000000..963a851a9e2538acfae9aab6eb28770d70a6747e
--- /dev/null
+++ b/roles/prometheus_exporters/files/arch-textcollector.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+HOSTNAME=$(hostname)
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/pacman.prom
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+updates=$(/usr/bin/checkupdates | wc -l)
+secupdates=$(/usr/bin/arch-audit -u | wc -l)
+
+echo "# HELP pacman_updates_pending number of pending updates from pacman" >> $TMP_FILE
+echo "# TYPE pacman_updates_pending gauge" >> $TMP_FILE
+echo "pacman_updates_pending{host=\"${HOSTNAME}\"} $updates" >> $TMP_FILE
+
+echo "# HELP pacman_security_updates_pending number of pending updates from pacman" >> $TMP_FILE
+echo "# TYPE pacman_security_updates_pending gauge" >> $TMP_FILE
+echo "pacman_security_updates_pending{host=\"${HOSTNAME}\"} $secupdates" >> $TMP_FILE
+
+mv -f $TMP_FILE $PROM_FILE
diff --git a/roles/prometheus_exporters/files/borg-textcollector.sh b/roles/prometheus_exporters/files/borg-textcollector.sh
new file mode 100755
index 0000000000000000000000000000000000000000..14de62849c3ab737020a159288bb9b865fa16b34
--- /dev/null
+++ b/roles/prometheus_exporters/files/borg-textcollector.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+HOSTNAME=$(hostname)
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/borg.prom
+
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+# Hetzner borg
+if [[ -f /usr/local/bin/borg ]]; then
+  LAST_ARCHIVE=$(/usr/local/bin/borg list --last 1)
+  LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}')
+  LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}')
+  LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s")
+
+  echo "# HELP borg_hetzner_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE
+  echo "# TYPE borg_hetzner_last_archive_timestamp counter" >> $TMP_FILE
+  echo "borg_hetzner_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE;
+fi
+
+# rsync.net borg
+if [[ -f /usr/local/bin/borg-offsite ]]; then
+  LAST_ARCHIVE=$(/usr/local/bin/borg-offsite list --last 1)
+  LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}')
+  LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}')
+  LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s")
+
+  echo "# HELP borg_offsite_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE
+  echo "# TYPE borg_offsite_last_archive_timestamp counter" >> $TMP_FILE
+  echo "borg_offsite_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE;
+fi
+
+mv -f $TMP_FILE $PROM_FILE
diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..60ea1ee36387a3e35b1d352ef642e4d30e2d53be
--- /dev/null
+++ b/roles/prometheus_exporters/tasks/main.yml
@@ -0,0 +1,88 @@
+---
+
+- name: install prometheus-node-exporter
+  pacman: name=prometheus-node-exporter,arch-audit,pacman-contrib state=present
+
+- name: install prometheus-memcached-exporter
+  pacman: name=prometheus-memcached-exporter state=present
+  when: "'memcached' in group_names"
+
+- name: install prometheus-mysqld-exporter
+  pacman: name=prometheus-mysqld-exporter state=present
+  when: "'mysql_servers' in group_names"
+
+- name: create prometheus mysqld database user
+  mysql_user:
+    name: '{{ prometheus_mysqld_user }}'
+    password: '{{ vault_prometheus_mysql_password }}'
+    priv: "*.*:PROCESS,REPLICATION CLIENT"
+    state: present
+  when: "'mysql_servers' in group_names"
+
+# TODO: with ansible 2.10 this can be set by mysql_user https://github.com/ansible/ansible/issues/26581
+- name: set max_user_connections for prometheus mysqld user
+  command: mysql -u root -e "ALTER USER '{{ prometheus_mysqld_user }}'@'localhost' WITH MAX_USER_CONNECTIONS 3;"
+  when: "'mysql_servers' in group_names"
+
+- name: copy prometheus mysqld exporter configuration
+  template: src=prometheus-mysqld-exporter.j2 dest=/etc/conf.d/prometheus-mysqld-exporter owner=root group=root mode=600
+  when: "'mysql_servers' in group_names"
+
+- name: enable prometheus-mysqld-exporter service
+  systemd: name=prometheus-mysqld-exporter enabled=yes daemon_reload=yes state=started
+  when: "'mysql_servers' in group_names"
+
+- name: install node exporter configuration
+  template: src=prometheus-node-exporter.env.j2 dest=/etc/conf.d/prometheus-node-exporter owner=root group=root mode=600
+
+- name: create textcollector directory
+  file: path="{{ prometheus_textfile_dir }}" state=directory owner=node_exporter group=node_exporter mode=700
+
+- name: install node exporter textcollector scripts
+  copy: src={{ item }} dest=/usr/local/bin/{{ item }} owner=root group=root mode=0755
+  with_items:
+    - arch-textcollector.sh
+    - borg-textcollector.sh
+
+- name: install arch textcollector service
+  template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600
+
+- name: install arch textcollector timer
+  template: src=prometheus-arch-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.timer owner=root group=root mode=600
+
+- name: enable and start prometheus arch textcollector timer
+  systemd: name=prometheus-arch-textcollector.timer enabled=yes daemon_reload=yes state=started
+
+- name: install borg textcollector service
+  template: src=prometheus-borg-textcollector.service.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.service owner=root group=root mode=600
+  when: "'borg_clients' in group_names"
+
+- name: install borg textcollector timer
+  template: src=prometheus-borg-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.timer owner=root group=root mode=600
+  when: "'borg_clients' in group_names"
+
+- name: enable and start prometheus borg textcollector timer
+  systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "'borg_clients' in group_names"
+
+- name: enable prometheus-node-exporter service
+  systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started
+
+- name: enable prometheus-memcached-exporter service
+  systemd: name=prometheus-memcached-exporter enabled=yes daemon_reload=yes state=started
+  when: "'memcached' in group_names"
+
+- name: open prometheus-node-exporter ipv4 port for monitoring.archlinux.org
+  firewalld: state=enabled permanent=true immediate=yes
+        rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_exporter_port }} accept"
+  when: "'prometheus' not in group_names"
+
+- name: open gitlab exporter ipv4 port for monitoring.archlinux.org
+  firewalld: state=enabled permanent=true immediate=yes
+        rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ gitlab_runner_exporter_port }} accept"
+  when: "'gitlab_runners' in group_names"
+
+- name: open prometheus mysqld exporter ipv4 port for monitoring.archlinux.org
+  firewalld: state=enabled permanent=true immediate=yes
+        rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_mysqld_exporter_port }} accept"
+  when: "'mysql_servers' in group_names"
diff --git a/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2
new file mode 100644
index 0000000000000000000000000000000000000000..5edb6dc89f1b22f9479cdf3b8e5bc8c62a687b0b
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2
@@ -0,0 +1,37 @@
+[Unit]
+Description=Prometheus Arch Exporter
+After=network.target
+
+[Service]
+Type=oneshot
+User=node_exporter
+ExecStart=/usr/local/bin/arch-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+CapabilityBoundingSet=
+UMask=077
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native
diff --git a/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2
new file mode 100644
index 0000000000000000000000000000000000000000..6afeacf4dc4428cdd3b2ca2a16d7f1a2e052ec50
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Prometheus Arch Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=60m
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2
new file mode 100644
index 0000000000000000000000000000000000000000..593a774eef4860048b1bc2049752d69cdb443531
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2
@@ -0,0 +1,35 @@
+[Unit]
+Description=Prometheus Borg Exporter TextCollector
+After=network.target
+ConditionPathExistsGlob=!/root/.cache/borg/*/lock.roster
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/borg-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths={{ prometheus_textfile_dir }} /root/.cache/borg
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_PACKET
+RestrictAddressFamilies=~AF_NETLINK
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native
diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2
new file mode 100644
index 0000000000000000000000000000000000000000..ca8a197e20efe0879cb411fe4fc720324a24785f
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Prometheus Borg Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=1h
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 b/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2
new file mode 100644
index 0000000000000000000000000000000000000000..c74feee70fcb6a05aac0775046c1463796b258ab
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2
@@ -0,0 +1,3 @@
+DATA_SOURCE_NAME="{{ prometheus_mysqld_user }}:{{ vault_prometheus_mysql_password }}@(localhost:3306)/"
+# TODO: review these settings
+MYSQLD_EXPORTER_ARGS="--collect.binlog_size --collect.info_schema.processlist --collect.info_schema.userstats"
diff --git a/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 b/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2
new file mode 100644
index 0000000000000000000000000000000000000000..88dd42d6d790b9f87e13c1ea4715bcabb8cc79fa
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2
@@ -0,0 +1 @@
+NODE_EXPORTER_ARGS="--collector.systemd --collector.textfile.directory={{ prometheus_textfile_dir }}"