From 23564b29a6eec7b477f5441661c1a45adf99c77c Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Mon, 31 Aug 2020 18:18:53 +0200 Subject: [PATCH 1/3] Introduce prometheus exporters role for collection Add a new role called prometheus_exporters which should be run on every machine we have and starts different collectors depending on what group the machine is in. Currently supported our the gitlab runner exporter, rebuilder textcollector, mysqld-exporter, borg textcollector and an node/arch exporter. The arch exporter monitors the security status and pacman out of date packages gauge. --- group_vars/all/common.yml | 3 + group_vars/all/vault_gitlab.yml | 177 +++++++++--------- group_vars/gitlab_runners.yml | 2 + hosts | 10 + playbooks/aur.archlinux.org.yml | 1 + playbooks/gitlab-runners.yml | 1 + playbooks/gitlab.archlinux.org.yml | 1 + playbooks/monitoring.archlinux.org.yml | 1 + playbooks/reproducible.archlinux.org.yml | 1 + roles/gitlab_runner/tasks/main.yml | 7 + roles/prometheus/defaults/main.yml | 1 + roles/prometheus/templates/prometheus.yml.j2 | 66 +++++++ roles/prometheus_exporters/defaults/main.yml | 10 + .../files/arch-textcollector.sh | 31 +++ .../files/borg-textcollector.sh | 46 +++++ roles/prometheus_exporters/tasks/main.yml | 88 +++++++++ .../prometheus-arch-textcollector.service.j2 | 37 ++++ .../prometheus-arch-textcollector.timer.j2 | 10 + .../prometheus-borg-textcollector.service.j2 | 35 ++++ .../prometheus-borg-textcollector.timer.j2 | 10 + .../templates/prometheus-mysqld-exporter.j2 | 3 + .../templates/prometheus-node-exporter.env.j2 | 1 + 22 files changed, 457 insertions(+), 85 deletions(-) create mode 100644 roles/prometheus_exporters/defaults/main.yml create mode 100755 roles/prometheus_exporters/files/arch-textcollector.sh create mode 100755 roles/prometheus_exporters/files/borg-textcollector.sh create mode 100644 roles/prometheus_exporters/tasks/main.yml create mode 100644 roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 create mode 100644 roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 create mode 100644 roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 create mode 100644 roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 create mode 100644 roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 create mode 100644 roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 diff --git a/group_vars/all/common.yml b/group_vars/all/common.yml index 56a6bf87..664f5ddc 100644 --- a/group_vars/all/common.yml +++ b/group_vars/all/common.yml @@ -10,3 +10,6 @@ zabbix_agent_templates: # this is used by the maintenance role to get the ip address # of the machine running the playbook maintenance_remote_machine: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CLIENT.split(' ')[0] }}" + +# prometheus-node-exporter port +prometheus_exporter_port: '9100' diff --git a/group_vars/all/vault_gitlab.yml b/group_vars/all/vault_gitlab.yml index 8f4e0e01..64145b7e 100644 --- a/group_vars/all/vault_gitlab.yml +++ b/group_vars/all/vault_gitlab.yml @@ -1,86 +1,93 @@ $ANSIBLE_VAULT;1.1;AES256 -35393938326563366437646365633563303031393034626433333163373838613535333136356132 -3139393330383337376633313739643431636337343263310a616263613665356437383862663134 -34643230613832643332323634613561313634386636373937373533653338313030633339653235 -6330646665656530350a623165393139306136636433356262626534376436313539646664336266 -30353562633266383138373334656238396464326231313132643461313763356537386535326462 -64643862383638373930373632663462663865346664636635653937303837656239653739343266 -31346133383133656665363034653330633735663862393930303862306566643533633635343935 -31376266356139376630316566613732663231393136316163356433666366613466653664643162 -62393833303265303539333637613662346465326131656139663633386439633032313138353637 -32663337656661303264633862333530336430306433613032353263666365396439616466313665 -65653765386362383633363737316337643763653634663664363661633032303866346566656338 -63383537633038303632303331646637323566323266363035343538383962656464633338633835 -32313164663634326436616563313236316361353936336662366162393236303939333932316261 -66343931333432333766353239393261373936336534393163663864643336633364666665333731 -62386465616466636162303030663634303963663133633864363638383836306637616166326530 -62306535326565373132323739663933653162383366623931373833316363633737616364326565 -33303933633561336437663661306139663931303232323639353930353734633334323966353364 -36623036646161303135616436656364303338343130616132396436313963383638373834353463 -63636632353263656431393839356438363261353132663964646262306365383830393636613238 -36643039663333303764633161343561306439353462386638323534333563333739646336343537 -38326465393438346536343666346231343234613639373439376665653738353965353330393262 -32353464646538366631663039663530383562306430623938376231393639313932376331333735 -62343164613832623038353762303238386538663537303534616435323837386637623338343439 -31643965653266383137313635356565653635333563343435663561306262363533653363376634 -34333464316161303162313136333931383161366561303634356238613131613462386538623662 -65666664343164646135363566373565333138313666613731633366623831396261613165393364 -32663534303730633264396435613630613730316463646131326134333763363235646435366630 -39336364643933653363316562386265373132303139396532626531393166366261316465316366 -64356233326538616339303735373832666564626332636236653632333930303466653639623561 -33623730343833626363633434616365383662646266663238623166313238373364343338383765 -61363039306266336539386333633166313138663862323335343031666162333964646565616139 -62623536623438343839316366656562343631316265616431663638366130343465326437613236 -39386662613634373263326637633036623239343733643437666661363632653838333832623839 -37306361623434303432303934666639353230653631386338303337373332373439363566366631 -39363964393637646364383262633561303830336134383562343264663531643064636439323832 -63346238626632303638643639666139336364373430363966353939386262376633343061336466 -37303437343530633034353437373432393434326330643835316538303666313833363761336132 -62333834396439623936323432363730306263376166326434393835346561653937643663336261 -66623434376566653830616666363338353861323038373063663935366233333137613466353137 -33383930623532343264666239363466393565336437306532616433643031333835323930643564 -33316362313965353539393934623361373431306632616266623265623738383765643838663037 -30636635356532353564386637326539393064333636623866353032383030353064653931663233 -33636134656663323036353534643734623039663130333562353739646161313638363363393237 -33663531373965653862366666626435383165343464323463346435393238633663663237316337 -62623833303864633562663764656332323436323931393363363337666133643430633838656336 -34653934313166353864363061663164396532303565623636666430313637313830376139313764 -63343863333833343632323366616463613233336331303232363335396338323461363665343136 -61386238343931633739623134363362656436643764356130333431613465623136383161643766 -33353330303437623064363637346262353165343765643365346265633431393530613938356463 -38313166646162303332633736323035353933666563316637633731643331643639643663323331 -32616535643034396535316130636238373937643462316137646562643833343732396234333062 -33623064336561623336636431366365383664636631636434356461383333363738623861376164 -37343464323738316234313831633032316464326265363530333037633833313739663231323031 -64616537333064323364353362626161366130353662613334336635346231393666663839646135 -62636538653563336463373631626563363461323865666333306461393735366566373931616537 -64396563633663373566643637313230393430663364636662623536313264393265393065343964 -35623530643833643764646530326533353635396161613832616263383036383739643261333930 -61333132343362626163663735323231343930343639636563393764643863646261396362383238 -62373835626465626335643865663765653861613462323861316366393536383839313032303261 -34646661376534333638613764323738613438373236393534396661376465373361646662616165 -64663833366366313439303831333933306662366361353531646133623532323262616632306362 -35643066306232333035633430376165303633653764363630323263316338646530653831303934 -34316534306539303265343937366563613233366537386634636631646562653263323365663865 -35663364663762306266353534643563633365383063373531303864643938633530306538313437 -34633234323830323661383537303931336262356239646661343764393766666336306662616334 -36373231366130633438366365623539326232656234376163633935393939393934376664623863 -30636535366661366330303563326138323364613736313833376638303663316433323430336165 -32326534313533623938636164376435333965663633383236376632393564383132666238303338 -37653364346531636564306137633030373232353464356365306333613630623937323462646165 -32393233633531363863646234636138303034663333396161633162393136636635306536386238 -61616433623565313532396238393437356462316365643438313966613438303461666463653966 -37323839333136313331383464396163396666313063613332373832656532636334613332373066 -39303339333662336539353032396233313865653030613339383263313038366566623537336632 -66396637306230336563663363656163376430663331363331643438313130623764616232396632 -64353565643565663061646631383839636164393134306232363934653534356236343031383261 -36326563633134626465613532376635326139393563383065383437376433316632663165623361 -39613362373736396532333739303737613736366138633031613961313135353466306365393630 -39313863663762666361373430613963613331386336623765353865323437656630646132343761 -61393035316331393133636633393661616661373363326463643837663462376239316432323930 -38306435396538646636353265356461396166376530366436353864356538643663303230663535 -37306439373263343065346537656430643061623439313738646135623838383438333336353561 -66313330653438343937656562653338396639346464303636333736643533356262656233363164 -33376264623262376334383134303730313762313639643864376132666661303533613031373962 -36623239623466636434366662306237393062616665333234646336653863313331613662333261 -35646437316432393062313261646231356362336639643733366563643934643730 +38326435396539336139383762626634363962383031303030633464323433656634343639366564 +3139366461336631626537623866336362336337633062320a326136373934633930656538363633 +39636137336436363233643038663935386633383433353533383134636532353139303239326332 +3465626434646334620a356161303939666163306564663937303532333363376631626463613132 +64336337663939633538346165653839383366643137666635323239316431353332386661306436 +64323239386263323865643232633635323264613863636465653062373937383036366233383035 +32363061333938656639626432333538343130656165653330646461663665386264356136343564 +61643730353136616533613536313539386636366262666439323666393661373030303637636537 +32623634353830326337346131613739306336656562313339363230353630633334363635333566 +38303462663132323635616138363435633766623461663962376262623130646637636364386563 +33636230383939336531363562323065363634353731373662663639373734356362636530313166 +65656333383739356564363365643337633230616563366461386439376238363062373637393762 +37356566326531303939333664396133326634366461333632636664626261333064343335646263 +32663335663665316538313662383039626433613739363230643561313734316263626464396133 +33303335353832306232656664636437663163373535666136643635663930663562313065363062 +38383837396631333939363539363237333261313330373464303339633961356239386132313763 +36393566663535633138616137373266366465616537353461623566643961326530626366383631 +35316336303562643439373166646639323837336531373133393464323236353235663235346161 +35323634373135313865303133646366356630363034386235656531386465383430316432343932 +62376537323962373935313830613437386138396339643437633966363930366339396462353736 +61336631373062343439316261383265356135643238363036343435303932353033393431616634 +39376464396432633635616133323435633461393537656634663139653032313065626538383961 +30316638633065383836353061336437663462643765613863653863313863613830613531643030 +30653133373365313530316637653838636237336532313835363838616639366331303266663632 +62646561656231653536626231323364313836643633623132613239626433323433366138636332 +32653264643736626462323065393163373261663561373236306133646337386335656262313262 +30346265303161323265363431376266313864653763306661316632623333306531303732326561 +63313537636462623366376235373237636536366238613434363737616465636162313832633738 +61323732373566356133646566316536393931303439303863656361616664366133306435633237 +66316339383537376130623537303563333331353937623536376138646566303564303632343363 +63633561643465333331623736353263616438376431376431356538373766356465616130393430 +31646539306564343734303664616132376130313164663431333566366165643132303632373063 +62326463386363656462363830656666336365396165643235613436353665323230313765383266 +34383433343166316366363665333762376463623334626565613530653035646565353135333736 +65643632366565363636613765356162376261643633346661613636323435306135353330386262 +36306165653165333732343032636565366662613131323332306666633835613631393662666633 +65393735333734356539323663373230366265343138646565663631303332323361663435343331 +30623935616366643837383138393764393763663763646238303865613939303965376639663465 +30653432333332643135613934346361613332323732643765656333366135623232353535663862 +63353461343638313738316234313961353864643832383265633561306136333966633338373932 +31393237316138376531363836333239353066353561633539613636643562373537316666306434 +34343339396435646262373134323130353431623435383137373761616164353662313138316235 +66646134616237353031356339643064343936336433663938353834633861613762323862396533 +64613333306436353839613231373736333331323266643532396335663336663530396138393861 +32613163646335343564666634316636353437656466393431356334356164393861363562663836 +38656430333339303161306134303637336535613233653634303235383136393931376262613738 +65343337306433316339616632373032363835646439373934616662383566303063393538383334 +61356236313762326262316265613266626533643238626630663162386532333264356133663366 +35383539353962633062656463616537363137616130353465313639663037336565363831326139 +35313363326634336361303834373764363666383834346365633563373762363965646533643062 +65623130626537653836623262343832323165323539356130616461356131353636316465366662 +39623432356432326330303766316365323237653064646565343833396435333134623635356439 +34613637333939393432616261383439636238323566353138626631643937666165343330663761 +66663064653139663633303736343433633363633062653339666331373862393732303761383639 +37373735643432303366663238383562396534356433663439376563366639613934306430653633 +65626233636639613332633435363066393862363232666236383830646438646463626539393838 +66323364643265333336633263383433656330353033343033356565366361323862353439356435 +34616633666439666661653630616630373032623235336430326366306430633732666337313435 +30656665323736313363663036646462646430663266316231653238643133643134363639613435 +63353239353735653438323964386135326663323939306534306663393931363163623831626631 +31643132376463643633643163613862633832313535343561323734373733623435336339613939 +39306432313638613235386361343738663633613133643964353163366435646337396231656463 +39393435303232393763343237393661626334626263343038343237353736343431306431646432 +66623732386561316436396162383130663065303531313935633864323261363666383364363130 +35303764633265386538383263363039383866376234373536376563366537343833626235666462 +38356666313964666139346632363733396439626363313665643135663061316166636339376230 +35366132613631646130393135613030623962306538323134643539323133393934636666643762 +64326636363865323936636135383565656466663037346235313230366164313361386635316562 +63353038316430373737616231353262653033343730643363353530643936333166313866323933 +63363562346332396164363330346162366333663736643761316361653039343763626133323030 +33333831323363316333663961343661373666353065616165663232346230393966633063366263 +30636362666463643036326562313833653431663262373432383232396561303961333536613762 +62343032646164303762333635376465643735353038303337633865663830643133613039636432 +66393762346336323339383233393539353739663961346362656161306238353766343930656363 +62663830323833326366353539623863333962363366393137363831616537613932346534393764 +35643239373330313866393639393364343939313931633339663932376461613039396430626263 +32666238393565323562663336363339303235333064656130313434643934663566306563626166 +66343032616463393666366361653631373661363938656462373364656636303638396532396365 +30616539656364373032396231303634653261323433333734646262323639323330616264346539 +30623965363335323164646263346332353566396366306639303265373863663062653732386339 +31643862653134613532616236313730366431353962396133613565363564326463616131613739 +37626331643633623934646465653237326366623734623433363534386661343537663736386133 +61383961643066646534653331626330623831383935383361353630383236393632333162343861 +35333533343762643933363234643836333138643566636236663661376562616133626330326435 +37653933313537353562336237653732653431626266663332653764666538363335313161353539 +30633961306337656636663435303535613234303661353438353364323331643861396163393365 +63396433643661373138636664313237633762333731393963383763626631363163633430323633 +35636666383866633231303538346438363932326665323231376162643335636639613866633639 +66333736313233393362636165396233306633386530313737376236613432313833386332613936 +35643633383830346233616162353435323238386364323663626537393738623032323032303638 +61316161356238666235343861373531613236363437613131653761356234643938306430336463 +39373537643431323138 diff --git a/group_vars/gitlab_runners.yml b/group_vars/gitlab_runners.yml index 1c98fb00..80777b00 100644 --- a/group_vars/gitlab_runners.yml +++ b/group_vars/gitlab_runners.yml @@ -1,3 +1,5 @@ +gitlab_runner_exporter_port: 9252 + fail2ban_jails: sshd: true postfix: false diff --git a/hosts b/hosts index 43a89c29..bc91b96b 100644 --- a/hosts +++ b/hosts @@ -100,3 +100,13 @@ aur-dev.archlinux.org [prometheus] monitoring.archlinux.org + +[node_exporters] +aur.archlinux.org +monitoring.archlinux.org +gitlab.archlinux.org +reproducible.archlinux.org +runner1.archlinux.org +runner2.archlinux.org +secure-runner1.archlinux.org +secure-runner2.archlinux.org diff --git a/playbooks/aur.archlinux.org.yml b/playbooks/aur.archlinux.org.yml index e2fcabf5..6a13585a 100644 --- a/playbooks/aur.archlinux.org.yml +++ b/playbooks/aur.archlinux.org.yml @@ -8,6 +8,7 @@ - { role: tools } - { role: sshd, sshd_enable_includes: true } - { role: root_ssh } + - { role: prometheus_exporters } - { role: certbot } - { role: nginx } - { role: mariadb, mariadb_innodb_buffer_pool_size: '64M', mariadb_table_open_cache: '256', mariadb_query_cache_type: '0', diff --git a/playbooks/gitlab-runners.yml b/playbooks/gitlab-runners.yml index b2d09a37..3980aed0 100644 --- a/playbooks/gitlab-runners.yml +++ b/playbooks/gitlab-runners.yml @@ -9,4 +9,5 @@ - { role: sshd } - { role: root_ssh } - { role: fail2ban } + - { role: prometheus_exporters } - { role: gitlab_runner } diff --git a/playbooks/gitlab.archlinux.org.yml b/playbooks/gitlab.archlinux.org.yml index 3f47c753..d5e7eeee 100644 --- a/playbooks/gitlab.archlinux.org.yml +++ b/playbooks/gitlab.archlinux.org.yml @@ -11,3 +11,4 @@ - { role: root_ssh } - { role: gitlab, gitlab_domain: "gitlab.archlinux.org" } - { role: borg_client, tags: ["borg"] } + - { role: prometheus_exporters } diff --git a/playbooks/monitoring.archlinux.org.yml b/playbooks/monitoring.archlinux.org.yml index ec5aa794..2fc94d7b 100644 --- a/playbooks/monitoring.archlinux.org.yml +++ b/playbooks/monitoring.archlinux.org.yml @@ -10,5 +10,6 @@ - { role: hardening } - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" } - { role: prometheus } + - { role: prometheus_exporters } - { role: certbot } - { role: nginx } diff --git a/playbooks/reproducible.archlinux.org.yml b/playbooks/reproducible.archlinux.org.yml index d194e102..ce800050 100644 --- a/playbooks/reproducible.archlinux.org.yml +++ b/playbooks/reproducible.archlinux.org.yml @@ -14,3 +14,4 @@ - { role: certbot } - { role: nginx } - { role: rebuilderd } + - { role: prometheus_exporters } diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml index c5e18fab..f1a70b46 100644 --- a/roles/gitlab_runner/tasks/main.yml +++ b/roles/gitlab_runner/tasks/main.yml @@ -40,5 +40,12 @@ line: concurrent = 100 notify: restart gitlab-runner +- name: enable prometheus exporter + lineinfile: + path: /etc/gitlab-runner/config.toml + insertbefore: '^concurrent' + line: listen_address = ":{{ gitlab_runner_exporter_port }}" + notify: restart gitlab-runner + - name: enable and start gitlab runner service systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 3b75be8a..47660d86 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1 +1,2 @@ monitoring_domain: monitoring.archlinux.org +gitlab_runner_exporter_port: '9252' diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index c868c754..1da101ed 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -13,3 +13,69 @@ alerting: - localhost:9093 scrape_configs: + - job_name: 'node_exporter' + static_configs: + {% for host in groups['node_exporters'] %} + + - targets: ['{{ host }}:{{ prometheus_exporter_port }}'] + labels: + instance: "{{ host }}" + + {% endfor %} + + - job_name: 'gitlab_runner_exporter' + static_configs: + {% for host in groups['gitlab_runners'] %} + + - targets: ['{{ host }}:{{ gitlab_runner_exporter_port }}'] + labels: + instance: "{{ host }}" + + {% endfor %} + + - job_name: 'keycloak' + scheme: https + metrics_path: "/auth/realms/master/metrics" + basic_auth: + username: "{{ vault_keycloak_nginx_user }}" + password: "{{ vault_keycloak_nginx_passwd }}" + static_configs: + - targets: ['accounts.archlinux.org:443'] + labels: + instance: "accounts.archlinux.org" + + - job_name: 'gitlab_exporter' + scheme: https + metrics_path: "-/metrics" + params: + token: ["{{ vault_gitlab_prometheus_token }}"] + static_configs: + - targets: ['gitlab.archlinux.org:443'] + labels: + instance: "gitlab.archlinux.org" + + - job_name: 'mysqld_exporter' + static_configs: + + - targets: ['aur.archlinux.org:9104'] + labels: + instance: "aur.archlinux.org" + + - job_name: 'blackbox' + metrics_path: /probe + scrape_interval: 15s + params: + module: [http_prometheus] + static_configs: + - targets: + {% for target in blackbox_targets %} + - {{ target }} + {% endfor %} + + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 diff --git a/roles/prometheus_exporters/defaults/main.yml b/roles/prometheus_exporters/defaults/main.yml new file mode 100644 index 00000000..b66b5c46 --- /dev/null +++ b/roles/prometheus_exporters/defaults/main.yml @@ -0,0 +1,10 @@ +--- + +prometheus_domain: monitoring.archlinux.org + +prometheus_textfile_dir: /var/lib/node_exporter + +gitlab_runner_exporter_port: '9252' + +prometheus_mysqld_user: mysqld_exporter +prometheus_mysqld_exporter_port: '9104' diff --git a/roles/prometheus_exporters/files/arch-textcollector.sh b/roles/prometheus_exporters/files/arch-textcollector.sh new file mode 100755 index 00000000..963a851a --- /dev/null +++ b/roles/prometheus_exporters/files/arch-textcollector.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +if (( $# != 1 )); then + echo "Missing textcollector directory argument" + exit 1 +fi + +HOSTNAME=$(hostname) +TEXTFILE_COLLECTOR_DIR=${1} +PROM_FILE=$TEXTFILE_COLLECTOR_DIR/pacman.prom + +TMP_FILE=$PROM_FILE.$$ +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +trap "rm -f $TMP_FILE" EXIT + +updates=$(/usr/bin/checkupdates | wc -l) +secupdates=$(/usr/bin/arch-audit -u | wc -l) + +echo "# HELP pacman_updates_pending number of pending updates from pacman" >> $TMP_FILE +echo "# TYPE pacman_updates_pending gauge" >> $TMP_FILE +echo "pacman_updates_pending{host=\"${HOSTNAME}\"} $updates" >> $TMP_FILE + +echo "# HELP pacman_security_updates_pending number of pending updates from pacman" >> $TMP_FILE +echo "# TYPE pacman_security_updates_pending gauge" >> $TMP_FILE +echo "pacman_security_updates_pending{host=\"${HOSTNAME}\"} $secupdates" >> $TMP_FILE + +mv -f $TMP_FILE $PROM_FILE diff --git a/roles/prometheus_exporters/files/borg-textcollector.sh b/roles/prometheus_exporters/files/borg-textcollector.sh new file mode 100755 index 00000000..14de6284 --- /dev/null +++ b/roles/prometheus_exporters/files/borg-textcollector.sh @@ -0,0 +1,46 @@ +#!/usr/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +if (( $# != 1 )); then + echo "Missing textcollector directory argument" + exit 1 +fi + +HOSTNAME=$(hostname) +TEXTFILE_COLLECTOR_DIR=${1} +PROM_FILE=$TEXTFILE_COLLECTOR_DIR/borg.prom + + +TMP_FILE=$PROM_FILE.$$ +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +trap "rm -f $TMP_FILE" EXIT + +# Hetzner borg +if [[ -f /usr/local/bin/borg ]]; then + LAST_ARCHIVE=$(/usr/local/bin/borg list --last 1) + LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}') + LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}') + LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s") + + echo "# HELP borg_hetzner_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE + echo "# TYPE borg_hetzner_last_archive_timestamp counter" >> $TMP_FILE + echo "borg_hetzner_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE; +fi + +# rsync.net borg +if [[ -f /usr/local/bin/borg-offsite ]]; then + LAST_ARCHIVE=$(/usr/local/bin/borg-offsite list --last 1) + LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}') + LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}') + LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s") + + echo "# HELP borg_offsite_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE + echo "# TYPE borg_offsite_last_archive_timestamp counter" >> $TMP_FILE + echo "borg_offsite_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE; +fi + +mv -f $TMP_FILE $PROM_FILE diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml new file mode 100644 index 00000000..60ea1ee3 --- /dev/null +++ b/roles/prometheus_exporters/tasks/main.yml @@ -0,0 +1,88 @@ +--- + +- name: install prometheus-node-exporter + pacman: name=prometheus-node-exporter,arch-audit,pacman-contrib state=present + +- name: install prometheus-memcached-exporter + pacman: name=prometheus-memcached-exporter state=present + when: "'memcached' in group_names" + +- name: install prometheus-mysqld-exporter + pacman: name=prometheus-mysqld-exporter state=present + when: "'mysql_servers' in group_names" + +- name: create prometheus mysqld database user + mysql_user: + name: '{{ prometheus_mysqld_user }}' + password: '{{ vault_prometheus_mysql_password }}' + priv: "*.*:PROCESS,REPLICATION CLIENT" + state: present + when: "'mysql_servers' in group_names" + +# TODO: with ansible 2.10 this can be set by mysql_user https://github.com/ansible/ansible/issues/26581 +- name: set max_user_connections for prometheus mysqld user + command: mysql -u root -e "ALTER USER '{{ prometheus_mysqld_user }}'@'localhost' WITH MAX_USER_CONNECTIONS 3;" + when: "'mysql_servers' in group_names" + +- name: copy prometheus mysqld exporter configuration + template: src=prometheus-mysqld-exporter.j2 dest=/etc/conf.d/prometheus-mysqld-exporter owner=root group=root mode=600 + when: "'mysql_servers' in group_names" + +- name: enable prometheus-mysqld-exporter service + systemd: name=prometheus-mysqld-exporter enabled=yes daemon_reload=yes state=started + when: "'mysql_servers' in group_names" + +- name: install node exporter configuration + template: src=prometheus-node-exporter.env.j2 dest=/etc/conf.d/prometheus-node-exporter owner=root group=root mode=600 + +- name: create textcollector directory + file: path="{{ prometheus_textfile_dir }}" state=directory owner=node_exporter group=node_exporter mode=700 + +- name: install node exporter textcollector scripts + copy: src={{ item }} dest=/usr/local/bin/{{ item }} owner=root group=root mode=0755 + with_items: + - arch-textcollector.sh + - borg-textcollector.sh + +- name: install arch textcollector service + template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600 + +- name: install arch textcollector timer + template: src=prometheus-arch-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.timer owner=root group=root mode=600 + +- name: enable and start prometheus arch textcollector timer + systemd: name=prometheus-arch-textcollector.timer enabled=yes daemon_reload=yes state=started + +- name: install borg textcollector service + template: src=prometheus-borg-textcollector.service.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.service owner=root group=root mode=600 + when: "'borg_clients' in group_names" + +- name: install borg textcollector timer + template: src=prometheus-borg-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.timer owner=root group=root mode=600 + when: "'borg_clients' in group_names" + +- name: enable and start prometheus borg textcollector timer + systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started + when: "'borg_clients' in group_names" + +- name: enable prometheus-node-exporter service + systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started + +- name: enable prometheus-memcached-exporter service + systemd: name=prometheus-memcached-exporter enabled=yes daemon_reload=yes state=started + when: "'memcached' in group_names" + +- name: open prometheus-node-exporter ipv4 port for monitoring.archlinux.org + firewalld: state=enabled permanent=true immediate=yes + rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_exporter_port }} accept" + when: "'prometheus' not in group_names" + +- name: open gitlab exporter ipv4 port for monitoring.archlinux.org + firewalld: state=enabled permanent=true immediate=yes + rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ gitlab_runner_exporter_port }} accept" + when: "'gitlab_runners' in group_names" + +- name: open prometheus mysqld exporter ipv4 port for monitoring.archlinux.org + firewalld: state=enabled permanent=true immediate=yes + rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_mysqld_exporter_port }} accept" + when: "'mysql_servers' in group_names" diff --git a/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 new file mode 100644 index 00000000..5edb6dc8 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 @@ -0,0 +1,37 @@ +[Unit] +Description=Prometheus Arch Exporter +After=network.target + +[Service] +Type=oneshot +User=node_exporter +ExecStart=/usr/local/bin/arch-textcollector.sh {{ prometheus_textfile_dir }} + +NoNewPrivileges=true +LockPersonality=true +CapabilityBoundingSet= +UMask=077 + +PrivateDevices=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths={{ prometheus_textfile_dir }} + +MemoryDenyWriteExecute=true +RemoveIPC=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictSUIDSGID=true + +RestrictAddressFamilies=~AF_NETLINK +RestrictAddressFamilies=~AF_PACKET + +ProtectHostname=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectClock=true + +SystemCallArchitectures=native diff --git a/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 new file mode 100644 index 00000000..6afeacf4 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus Arch Exporter TextCollector Timer + +[Timer] +OnUnitActiveSec=60m +OnBootSec=15min +RandomizedDelaySec=1min + +[Install] +WantedBy=timers.target diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 new file mode 100644 index 00000000..593a774e --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 @@ -0,0 +1,35 @@ +[Unit] +Description=Prometheus Borg Exporter TextCollector +After=network.target +ConditionPathExistsGlob=!/root/.cache/borg/*/lock.roster + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/borg-textcollector.sh {{ prometheus_textfile_dir }} + +NoNewPrivileges=true +LockPersonality=true + +PrivateDevices=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={{ prometheus_textfile_dir }} /root/.cache/borg + +MemoryDenyWriteExecute=true +RemoveIPC=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictSUIDSGID=true + +RestrictAddressFamilies=~AF_PACKET +RestrictAddressFamilies=~AF_NETLINK + +ProtectHostname=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectClock=true + +SystemCallArchitectures=native diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 new file mode 100644 index 00000000..ca8a197e --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus Borg Exporter TextCollector Timer + +[Timer] +OnUnitActiveSec=1h +OnBootSec=15min +RandomizedDelaySec=1min + +[Install] +WantedBy=timers.target diff --git a/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 b/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 new file mode 100644 index 00000000..c74feee7 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 @@ -0,0 +1,3 @@ +DATA_SOURCE_NAME="{{ prometheus_mysqld_user }}:{{ vault_prometheus_mysql_password }}@(localhost:3306)/" +# TODO: review these settings +MYSQLD_EXPORTER_ARGS="--collect.binlog_size --collect.info_schema.processlist --collect.info_schema.userstats" diff --git a/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 b/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 new file mode 100644 index 00000000..88dd42d6 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 @@ -0,0 +1 @@ +NODE_EXPORTER_ARGS="--collector.systemd --collector.textfile.directory={{ prometheus_textfile_dir }}" -- GitLab From 3fd36ddb177c42476ee3c194a553f723954507d8 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sun, 9 Aug 2020 19:26:10 +0200 Subject: [PATCH 2/3] Add blackbox exporter for https status checking Run the blackbox exporter on monitoring.archlinux.org to monitor other machines http status for public services we provide. Also has an alert for when a certificate is about to expire in 3 days. --- roles/prometheus/defaults/main.yml | 11 ++++++++ roles/prometheus/files/node.rules.yml | 28 +++++++++++++++++++ roles/prometheus_exporters/files/blackbox.yml | 5 ++++ roles/prometheus_exporters/tasks/main.yml | 12 ++++++++ 4 files changed, 56 insertions(+) create mode 100644 roles/prometheus_exporters/files/blackbox.yml diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 47660d86..7193311a 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1,2 +1,13 @@ monitoring_domain: monitoring.archlinux.org gitlab_runner_exporter_port: '9252' + +blackbox_targets: + - https://archlinux.org + - https://wiki.archlinux.org + - https://gitlab.archlinux.org + - https://bbs.archlinux.org + - https://bugs.archlinux.org + - https://aur.archlinux.org + - https://archive.archlinux.org + - https://mirror.pkgbuild.com + - https://pkgbuild.com diff --git a/roles/prometheus/files/node.rules.yml b/roles/prometheus/files/node.rules.yml index d1eba8ba..4fb0d23f 100644 --- a/roles/prometheus/files/node.rules.yml +++ b/roles/prometheus/files/node.rules.yml @@ -366,3 +366,31 @@ groups: annotations: summary: Workhorse has high error rates description: Workhorse route {{ $labels.route }} method {{ $labels.method }} has more than 10% errors ({{ $value | printf "%.1f" }}%) for the last 60 minutes. + +- name: blackbox + interval: 15s + rules: + - alert: BlackboxProbeFailed + expr: probe_success == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Blackbox probe failed (instance {{ $labels.instance }})" + description: "Probe failed\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + - alert: BlackboxProbeHttpFailure + expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400 + for: 5m + labels: + severity: critical + annotations: + summary: "Blackbox probe HTTP failure (instance {{ $labels.instance }})" + description: "HTTP status code is not 200-399\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 + for: 5m + labels: + severity: critical + annotations: + summary: "Blackbox SSL certificate will expire soon (instance {{ $labels.instance }})" + description: "SSL certificate expires in 3 days\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" diff --git a/roles/prometheus_exporters/files/blackbox.yml b/roles/prometheus_exporters/files/blackbox.yml new file mode 100644 index 00000000..a537090b --- /dev/null +++ b/roles/prometheus_exporters/files/blackbox.yml @@ -0,0 +1,5 @@ +modules: + http_prometheus: + prober: http + timeout: 5s + http: diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml index 60ea1ee3..65772aac 100644 --- a/roles/prometheus_exporters/tasks/main.yml +++ b/roles/prometheus_exporters/tasks/main.yml @@ -3,6 +3,10 @@ - name: install prometheus-node-exporter pacman: name=prometheus-node-exporter,arch-audit,pacman-contrib state=present +- name: install prometheus-blackbox-exporter + pacman: name=prometheus-blackbox-exporter state=present + when: "'prometheus' in group_names" + - name: install prometheus-memcached-exporter pacman: name=prometheus-memcached-exporter state=present when: "'memcached' in group_names" @@ -65,9 +69,17 @@ systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started when: "'borg_clients' in group_names" +- name: install blackbox exporter configuration + copy: src=blackbox.yml dest=/etc/prometheus/blackbox.yml owner=root group=root mode=0755 + when: "'prometheus' in group_names" + - name: enable prometheus-node-exporter service systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started +- name: enable prometheus-blackbox-exporter service + systemd: name=prometheus-blackbox-exporter enabled=yes daemon_reload=yes state=started + when: "'prometheus' in group_names" + - name: enable prometheus-memcached-exporter service systemd: name=prometheus-memcached-exporter enabled=yes daemon_reload=yes state=started when: "'memcached' in group_names" -- GitLab From cd4b284446d2f19b0dcf866f7a911fa0eb122cea Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sat, 15 Aug 2020 14:38:20 +0200 Subject: [PATCH 3/3] Add rebuilderd build queue length textcollector Record the rebuilderd queue length in prometheus so we can generate an alert for when the queue length keeps rising. As this could be an indication that the rebuilders have builds which are stuck. --- hosts | 3 ++ roles/prometheus/files/node.rules.yml | 22 +++++++++++ .../prometheus/templates/alertmanager.yml.j2 | 11 +++++- .../files/rebuilderd-textcollector.sh | 31 ++++++++++++++++ roles/prometheus_exporters/tasks/main.yml | 21 +++++++++++ ...etheus-rebuilderd-textcollector.service.j2 | 37 +++++++++++++++++++ ...ometheus-rebuilderd-textcollector.timer.j2 | 10 +++++ 7 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 roles/prometheus_exporters/files/rebuilderd-textcollector.sh create mode 100644 roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.service.j2 create mode 100644 roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.timer.j2 diff --git a/hosts b/hosts index bc91b96b..bf25cecf 100644 --- a/hosts +++ b/hosts @@ -93,6 +93,9 @@ secure-runner2.archlinux.org repro1.pkgbuild.com repro3.pkgbuild.com +[rebuilderd] +reproducible.archlinux.org + [memcached] apollo.archlinux.org aur.archlinux.org diff --git a/roles/prometheus/files/node.rules.yml b/roles/prometheus/files/node.rules.yml index 4fb0d23f..10989074 100644 --- a/roles/prometheus/files/node.rules.yml +++ b/roles/prometheus/files/node.rules.yml @@ -394,3 +394,25 @@ groups: annotations: summary: "Blackbox SSL certificate will expire soon (instance {{ $labels.instance }})" description: "SSL certificate expires in 3 days\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + +- name: rebuilderd + interval: 15m + rules: + - alert: RebuilderdQueueNotEmpty + expr: rebuilderd_queue_length > 2000 + for: 24h + labels: + severity: warning + service: rebuilderd + annotations: + summary: "Rebuilderd queue length is not empty {{ $labels.instance }})" + description: "Rebuilderd's queue length is now: {{ $value }}" + - alert: RebuilderdWorkersOffline + expr: rebuilderd_workers < 4 + for: 5m + labels: + severity: warning + service: rebuilderd + annotations: + summary: "Rebuilderd workers offline {{ $labels.instance }})" + description: "Not all rebuilder-workers are online, currently {{ $value }} workers are online" diff --git a/roles/prometheus/templates/alertmanager.yml.j2 b/roles/prometheus/templates/alertmanager.yml.j2 index 73083d67..de998095 100644 --- a/roles/prometheus/templates/alertmanager.yml.j2 +++ b/roles/prometheus/templates/alertmanager.yml.j2 @@ -11,9 +11,16 @@ route: group_wait: 30s group_interval: 5m repeat_interval: 24h - receiver: 'devops' + receiver: 'default-receiver' + routes: + - receiver: 'rebuilderd' + match_re: + service: rebuilderd receivers: - - name: 'devops' + - name: 'default-receiver' email_configs: - to: 'alerts@archlinux.org' + - name: 'rebuilderd' + email_configs: + - to: 'jelle@archlinux.org' diff --git a/roles/prometheus_exporters/files/rebuilderd-textcollector.sh b/roles/prometheus_exporters/files/rebuilderd-textcollector.sh new file mode 100644 index 00000000..635d8207 --- /dev/null +++ b/roles/prometheus_exporters/files/rebuilderd-textcollector.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +if (( $# != 1 )); then + echo "Missing textcollector directory argument" + exit 1 +fi + +HOSTNAME=$(hostname) +TEXTFILE_COLLECTOR_DIR=${1} +PROM_FILE=$TEXTFILE_COLLECTOR_DIR/rebuilderd.prom + +TMP_FILE=$PROM_FILE.$$ +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +trap "rm -f $TMP_FILE" EXIT + +queuelength=$(rebuildctl queue ls --json | jq '.queue | length') +workers=$(rebuildctl status | wc -l) + +echo "# HELP rebuilderd_queue_length number of packages in rebuilderd queue" >> $TMP_FILE +echo "# TYPE rebuilderd_queue_length gauge" >> $TMP_FILE +echo "rebuilderd_queue_length{host=\"${HOSTNAME}\"} $queuelength" >> $TMP_FILE + +echo "# HELP rebuilderd_workers number of rebuilderd-workers available" >> $TMP_FILE +echo "# TYPE rebuilderd_workers gauge" >> $TMP_FILE +echo "rebuilderd_workers{host=\"${HOSTNAME}\"} $workers" >> $TMP_FILE + +mv -f $TMP_FILE $PROM_FILE diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml index 65772aac..51c305d1 100644 --- a/roles/prometheus_exporters/tasks/main.yml +++ b/roles/prometheus_exporters/tasks/main.yml @@ -11,6 +11,14 @@ pacman: name=prometheus-memcached-exporter state=present when: "'memcached' in group_names" +- name: install jq for rebuilderd-textcollector + pacman: name=jq state=present + when: "'rebuilderd' in group_names" + +- name: add node_exporter to rebuilderd group + user: name=node_exporter groups=rebuilderd append=yes + when: "'rebuilderd' in group_names" + - name: install prometheus-mysqld-exporter pacman: name=prometheus-mysqld-exporter state=present when: "'mysql_servers' in group_names" @@ -47,6 +55,7 @@ with_items: - arch-textcollector.sh - borg-textcollector.sh + - rebuilderd-textcollector.sh - name: install arch textcollector service template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600 @@ -73,6 +82,18 @@ copy: src=blackbox.yml dest=/etc/prometheus/blackbox.yml owner=root group=root mode=0755 when: "'prometheus' in group_names" +- name: install rebuilderd textcollector service + template: src=prometheus-rebuilderd-textcollector.service.j2 dest=/etc/systemd/system/prometheus-rebuilderd-textcollector.service owner=root group=root mode=600 + when: "'rebuilderd' in group_names" + +- name: install rebuilderd textcollector timer + template: src=prometheus-rebuilderd-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-rebuilderd-textcollector.timer owner=root group=root mode=600 + when: "'rebuilderd' in group_names" + +- name: enable and start prometheus rebuilderd textcollector timer + systemd: name=prometheus-rebuilderd-textcollector.timer enabled=yes daemon_reload=yes state=started + when: "'rebuilderd' in group_names" + - name: enable prometheus-node-exporter service systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started diff --git a/roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.service.j2 new file mode 100644 index 00000000..7b219b2a --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.service.j2 @@ -0,0 +1,37 @@ +[Unit] +Description=Prometheus Rebuilderd Exporter +After=network.target + +[Service] +Type=oneshot +User=node_exporter +ExecStart=/usr/local/bin/rebuilderd-textcollector.sh {{ prometheus_textfile_dir }} + +NoNewPrivileges=true +LockPersonality=true +CapabilityBoundingSet= +UMask=077 + +PrivateDevices=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths={{ prometheus_textfile_dir }} + +MemoryDenyWriteExecute=true +RemoveIPC=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictSUIDSGID=true + +RestrictAddressFamilies=~AF_NETLINK +RestrictAddressFamilies=~AF_PACKET + +ProtectHostname=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectClock=true + +SystemCallArchitectures=native diff --git a/roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.timer.j2 new file mode 100644 index 00000000..5b3115f5 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-rebuilderd-textcollector.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus Rebuilderd Exporter TextCollector Timer + +[Timer] +OnUnitActiveSec=60m +OnBootSec=15min +RandomizedDelaySec=1min + +[Install] +WantedBy=timers.target -- GitLab