diff --git a/group_vars/all/common.yml b/group_vars/all/common.yml index 56a6bf8708642e1f99ca0f558309c47996f9ae93..664f5ddce40ea5d4f07c98ed76f44e2188e5517f 100644 --- a/group_vars/all/common.yml +++ b/group_vars/all/common.yml @@ -10,3 +10,6 @@ zabbix_agent_templates: # this is used by the maintenance role to get the ip address # of the machine running the playbook maintenance_remote_machine: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CLIENT.split(' ')[0] }}" + +# prometheus-node-exporter port +prometheus_exporter_port: '9100' diff --git a/group_vars/all/vault_gitlab.yml b/group_vars/all/vault_gitlab.yml index 8f4e0e010d353530c52a22796577cc02d2f98d18..64145b7e620d1f24c60355cc0a8ee03181e64a6b 100644 --- a/group_vars/all/vault_gitlab.yml +++ b/group_vars/all/vault_gitlab.yml @@ -1,86 +1,93 @@ $ANSIBLE_VAULT;1.1;AES256 -35393938326563366437646365633563303031393034626433333163373838613535333136356132 -3139393330383337376633313739643431636337343263310a616263613665356437383862663134 -34643230613832643332323634613561313634386636373937373533653338313030633339653235 -6330646665656530350a623165393139306136636433356262626534376436313539646664336266 -30353562633266383138373334656238396464326231313132643461313763356537386535326462 -64643862383638373930373632663462663865346664636635653937303837656239653739343266 -31346133383133656665363034653330633735663862393930303862306566643533633635343935 -31376266356139376630316566613732663231393136316163356433666366613466653664643162 -62393833303265303539333637613662346465326131656139663633386439633032313138353637 -32663337656661303264633862333530336430306433613032353263666365396439616466313665 -65653765386362383633363737316337643763653634663664363661633032303866346566656338 -63383537633038303632303331646637323566323266363035343538383962656464633338633835 -32313164663634326436616563313236316361353936336662366162393236303939333932316261 -66343931333432333766353239393261373936336534393163663864643336633364666665333731 -62386465616466636162303030663634303963663133633864363638383836306637616166326530 -62306535326565373132323739663933653162383366623931373833316363633737616364326565 -33303933633561336437663661306139663931303232323639353930353734633334323966353364 -36623036646161303135616436656364303338343130616132396436313963383638373834353463 -63636632353263656431393839356438363261353132663964646262306365383830393636613238 -36643039663333303764633161343561306439353462386638323534333563333739646336343537 -38326465393438346536343666346231343234613639373439376665653738353965353330393262 -32353464646538366631663039663530383562306430623938376231393639313932376331333735 -62343164613832623038353762303238386538663537303534616435323837386637623338343439 -31643965653266383137313635356565653635333563343435663561306262363533653363376634 -34333464316161303162313136333931383161366561303634356238613131613462386538623662 -65666664343164646135363566373565333138313666613731633366623831396261613165393364 -32663534303730633264396435613630613730316463646131326134333763363235646435366630 -39336364643933653363316562386265373132303139396532626531393166366261316465316366 -64356233326538616339303735373832666564626332636236653632333930303466653639623561 -33623730343833626363633434616365383662646266663238623166313238373364343338383765 -61363039306266336539386333633166313138663862323335343031666162333964646565616139 -62623536623438343839316366656562343631316265616431663638366130343465326437613236 -39386662613634373263326637633036623239343733643437666661363632653838333832623839 -37306361623434303432303934666639353230653631386338303337373332373439363566366631 -39363964393637646364383262633561303830336134383562343264663531643064636439323832 -63346238626632303638643639666139336364373430363966353939386262376633343061336466 -37303437343530633034353437373432393434326330643835316538303666313833363761336132 -62333834396439623936323432363730306263376166326434393835346561653937643663336261 -66623434376566653830616666363338353861323038373063663935366233333137613466353137 -33383930623532343264666239363466393565336437306532616433643031333835323930643564 -33316362313965353539393934623361373431306632616266623265623738383765643838663037 -30636635356532353564386637326539393064333636623866353032383030353064653931663233 -33636134656663323036353534643734623039663130333562353739646161313638363363393237 -33663531373965653862366666626435383165343464323463346435393238633663663237316337 -62623833303864633562663764656332323436323931393363363337666133643430633838656336 -34653934313166353864363061663164396532303565623636666430313637313830376139313764 -63343863333833343632323366616463613233336331303232363335396338323461363665343136 -61386238343931633739623134363362656436643764356130333431613465623136383161643766 -33353330303437623064363637346262353165343765643365346265633431393530613938356463 -38313166646162303332633736323035353933666563316637633731643331643639643663323331 -32616535643034396535316130636238373937643462316137646562643833343732396234333062 -33623064336561623336636431366365383664636631636434356461383333363738623861376164 -37343464323738316234313831633032316464326265363530333037633833313739663231323031 -64616537333064323364353362626161366130353662613334336635346231393666663839646135 -62636538653563336463373631626563363461323865666333306461393735366566373931616537 -64396563633663373566643637313230393430663364636662623536313264393265393065343964 -35623530643833643764646530326533353635396161613832616263383036383739643261333930 -61333132343362626163663735323231343930343639636563393764643863646261396362383238 -62373835626465626335643865663765653861613462323861316366393536383839313032303261 -34646661376534333638613764323738613438373236393534396661376465373361646662616165 -64663833366366313439303831333933306662366361353531646133623532323262616632306362 -35643066306232333035633430376165303633653764363630323263316338646530653831303934 -34316534306539303265343937366563613233366537386634636631646562653263323365663865 -35663364663762306266353534643563633365383063373531303864643938633530306538313437 -34633234323830323661383537303931336262356239646661343764393766666336306662616334 -36373231366130633438366365623539326232656234376163633935393939393934376664623863 -30636535366661366330303563326138323364613736313833376638303663316433323430336165 -32326534313533623938636164376435333965663633383236376632393564383132666238303338 -37653364346531636564306137633030373232353464356365306333613630623937323462646165 -32393233633531363863646234636138303034663333396161633162393136636635306536386238 -61616433623565313532396238393437356462316365643438313966613438303461666463653966 -37323839333136313331383464396163396666313063613332373832656532636334613332373066 -39303339333662336539353032396233313865653030613339383263313038366566623537336632 -66396637306230336563663363656163376430663331363331643438313130623764616232396632 -64353565643565663061646631383839636164393134306232363934653534356236343031383261 -36326563633134626465613532376635326139393563383065383437376433316632663165623361 -39613362373736396532333739303737613736366138633031613961313135353466306365393630 -39313863663762666361373430613963613331386336623765353865323437656630646132343761 -61393035316331393133636633393661616661373363326463643837663462376239316432323930 -38306435396538646636353265356461396166376530366436353864356538643663303230663535 -37306439373263343065346537656430643061623439313738646135623838383438333336353561 -66313330653438343937656562653338396639346464303636333736643533356262656233363164 -33376264623262376334383134303730313762313639643864376132666661303533613031373962 -36623239623466636434366662306237393062616665333234646336653863313331613662333261 -35646437316432393062313261646231356362336639643733366563643934643730 +38326435396539336139383762626634363962383031303030633464323433656634343639366564 +3139366461336631626537623866336362336337633062320a326136373934633930656538363633 +39636137336436363233643038663935386633383433353533383134636532353139303239326332 +3465626434646334620a356161303939666163306564663937303532333363376631626463613132 +64336337663939633538346165653839383366643137666635323239316431353332386661306436 +64323239386263323865643232633635323264613863636465653062373937383036366233383035 +32363061333938656639626432333538343130656165653330646461663665386264356136343564 +61643730353136616533613536313539386636366262666439323666393661373030303637636537 +32623634353830326337346131613739306336656562313339363230353630633334363635333566 +38303462663132323635616138363435633766623461663962376262623130646637636364386563 +33636230383939336531363562323065363634353731373662663639373734356362636530313166 +65656333383739356564363365643337633230616563366461386439376238363062373637393762 +37356566326531303939333664396133326634366461333632636664626261333064343335646263 +32663335663665316538313662383039626433613739363230643561313734316263626464396133 +33303335353832306232656664636437663163373535666136643635663930663562313065363062 +38383837396631333939363539363237333261313330373464303339633961356239386132313763 +36393566663535633138616137373266366465616537353461623566643961326530626366383631 +35316336303562643439373166646639323837336531373133393464323236353235663235346161 +35323634373135313865303133646366356630363034386235656531386465383430316432343932 +62376537323962373935313830613437386138396339643437633966363930366339396462353736 +61336631373062343439316261383265356135643238363036343435303932353033393431616634 +39376464396432633635616133323435633461393537656634663139653032313065626538383961 +30316638633065383836353061336437663462643765613863653863313863613830613531643030 +30653133373365313530316637653838636237336532313835363838616639366331303266663632 +62646561656231653536626231323364313836643633623132613239626433323433366138636332 +32653264643736626462323065393163373261663561373236306133646337386335656262313262 +30346265303161323265363431376266313864653763306661316632623333306531303732326561 +63313537636462623366376235373237636536366238613434363737616465636162313832633738 +61323732373566356133646566316536393931303439303863656361616664366133306435633237 +66316339383537376130623537303563333331353937623536376138646566303564303632343363 +63633561643465333331623736353263616438376431376431356538373766356465616130393430 +31646539306564343734303664616132376130313164663431333566366165643132303632373063 +62326463386363656462363830656666336365396165643235613436353665323230313765383266 +34383433343166316366363665333762376463623334626565613530653035646565353135333736 +65643632366565363636613765356162376261643633346661613636323435306135353330386262 +36306165653165333732343032636565366662613131323332306666633835613631393662666633 +65393735333734356539323663373230366265343138646565663631303332323361663435343331 +30623935616366643837383138393764393763663763646238303865613939303965376639663465 +30653432333332643135613934346361613332323732643765656333366135623232353535663862 +63353461343638313738316234313961353864643832383265633561306136333966633338373932 +31393237316138376531363836333239353066353561633539613636643562373537316666306434 +34343339396435646262373134323130353431623435383137373761616164353662313138316235 +66646134616237353031356339643064343936336433663938353834633861613762323862396533 +64613333306436353839613231373736333331323266643532396335663336663530396138393861 +32613163646335343564666634316636353437656466393431356334356164393861363562663836 +38656430333339303161306134303637336535613233653634303235383136393931376262613738 +65343337306433316339616632373032363835646439373934616662383566303063393538383334 +61356236313762326262316265613266626533643238626630663162386532333264356133663366 +35383539353962633062656463616537363137616130353465313639663037336565363831326139 +35313363326634336361303834373764363666383834346365633563373762363965646533643062 +65623130626537653836623262343832323165323539356130616461356131353636316465366662 +39623432356432326330303766316365323237653064646565343833396435333134623635356439 +34613637333939393432616261383439636238323566353138626631643937666165343330663761 +66663064653139663633303736343433633363633062653339666331373862393732303761383639 +37373735643432303366663238383562396534356433663439376563366639613934306430653633 +65626233636639613332633435363066393862363232666236383830646438646463626539393838 +66323364643265333336633263383433656330353033343033356565366361323862353439356435 +34616633666439666661653630616630373032623235336430326366306430633732666337313435 +30656665323736313363663036646462646430663266316231653238643133643134363639613435 +63353239353735653438323964386135326663323939306534306663393931363163623831626631 +31643132376463643633643163613862633832313535343561323734373733623435336339613939 +39306432313638613235386361343738663633613133643964353163366435646337396231656463 +39393435303232393763343237393661626334626263343038343237353736343431306431646432 +66623732386561316436396162383130663065303531313935633864323261363666383364363130 +35303764633265386538383263363039383866376234373536376563366537343833626235666462 +38356666313964666139346632363733396439626363313665643135663061316166636339376230 +35366132613631646130393135613030623962306538323134643539323133393934636666643762 +64326636363865323936636135383565656466663037346235313230366164313361386635316562 +63353038316430373737616231353262653033343730643363353530643936333166313866323933 +63363562346332396164363330346162366333663736643761316361653039343763626133323030 +33333831323363316333663961343661373666353065616165663232346230393966633063366263 +30636362666463643036326562313833653431663262373432383232396561303961333536613762 +62343032646164303762333635376465643735353038303337633865663830643133613039636432 +66393762346336323339383233393539353739663961346362656161306238353766343930656363 +62663830323833326366353539623863333962363366393137363831616537613932346534393764 +35643239373330313866393639393364343939313931633339663932376461613039396430626263 +32666238393565323562663336363339303235333064656130313434643934663566306563626166 +66343032616463393666366361653631373661363938656462373364656636303638396532396365 +30616539656364373032396231303634653261323433333734646262323639323330616264346539 +30623965363335323164646263346332353566396366306639303265373863663062653732386339 +31643862653134613532616236313730366431353962396133613565363564326463616131613739 +37626331643633623934646465653237326366623734623433363534386661343537663736386133 +61383961643066646534653331626330623831383935383361353630383236393632333162343861 +35333533343762643933363234643836333138643566636236663661376562616133626330326435 +37653933313537353562336237653732653431626266663332653764666538363335313161353539 +30633961306337656636663435303535613234303661353438353364323331643861396163393365 +63396433643661373138636664313237633762333731393963383763626631363163633430323633 +35636666383866633231303538346438363932326665323231376162643335636639613866633639 +66333736313233393362636165396233306633386530313737376236613432313833386332613936 +35643633383830346233616162353435323238386364323663626537393738623032323032303638 +61316161356238666235343861373531613236363437613131653761356234643938306430336463 +39373537643431323138 diff --git a/group_vars/gitlab_runners.yml b/group_vars/gitlab_runners.yml index 1c98fb00e6ee21fa0f9948475bea59c92a9eb800..80777b00c8970bc60cf98c6121d5cbe5d768fa10 100644 --- a/group_vars/gitlab_runners.yml +++ b/group_vars/gitlab_runners.yml @@ -1,3 +1,5 @@ +gitlab_runner_exporter_port: 9252 + fail2ban_jails: sshd: true postfix: false diff --git a/hosts b/hosts index 43a89c29524d41c5a2d6c7eecacda6cf8e1299ae..bc91b96b014eb2dd451e59d03ee7734f82af732a 100644 --- a/hosts +++ b/hosts @@ -100,3 +100,13 @@ aur-dev.archlinux.org [prometheus] monitoring.archlinux.org + +[node_exporters] +aur.archlinux.org +monitoring.archlinux.org +gitlab.archlinux.org +reproducible.archlinux.org +runner1.archlinux.org +runner2.archlinux.org +secure-runner1.archlinux.org +secure-runner2.archlinux.org diff --git a/playbooks/aur.archlinux.org.yml b/playbooks/aur.archlinux.org.yml index e2fcabf57c54961a8c6056f66821b4a6226bbc44..6a13585a289c71ce27277ded3cf9ada8d6bcd7eb 100644 --- a/playbooks/aur.archlinux.org.yml +++ b/playbooks/aur.archlinux.org.yml @@ -8,6 +8,7 @@ - { role: tools } - { role: sshd, sshd_enable_includes: true } - { role: root_ssh } + - { role: prometheus_exporters } - { role: certbot } - { role: nginx } - { role: mariadb, mariadb_innodb_buffer_pool_size: '64M', mariadb_table_open_cache: '256', mariadb_query_cache_type: '0', diff --git a/playbooks/gitlab-runners.yml b/playbooks/gitlab-runners.yml index b2d09a3713d6377f02a4348ec161b92f42cdfc4f..3980aed0be513d5b2f8e0cb5293f7ad87f4c8e05 100644 --- a/playbooks/gitlab-runners.yml +++ b/playbooks/gitlab-runners.yml @@ -9,4 +9,5 @@ - { role: sshd } - { role: root_ssh } - { role: fail2ban } + - { role: prometheus_exporters } - { role: gitlab_runner } diff --git a/playbooks/gitlab.archlinux.org.yml b/playbooks/gitlab.archlinux.org.yml index 3f47c753c1622c7737a8a865805a8ddf0ce6e879..d5e7eeeeed83b4388870f660fe45644b2f294b15 100644 --- a/playbooks/gitlab.archlinux.org.yml +++ b/playbooks/gitlab.archlinux.org.yml @@ -11,3 +11,4 @@ - { role: root_ssh } - { role: gitlab, gitlab_domain: "gitlab.archlinux.org" } - { role: borg_client, tags: ["borg"] } + - { role: prometheus_exporters } diff --git a/playbooks/monitoring.archlinux.org.yml b/playbooks/monitoring.archlinux.org.yml index ec5aa7945277c62840f1ccf51783f3a874ac00b4..2fc94d7b286c6ec343bd6cd826b01d194713294f 100644 --- a/playbooks/monitoring.archlinux.org.yml +++ b/playbooks/monitoring.archlinux.org.yml @@ -10,5 +10,6 @@ - { role: hardening } - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" } - { role: prometheus } + - { role: prometheus_exporters } - { role: certbot } - { role: nginx } diff --git a/playbooks/reproducible.archlinux.org.yml b/playbooks/reproducible.archlinux.org.yml index d194e102df544760cefe6eaf7510748bca2c10e8..ce800050a9201ea66f561a550262725f0a9f8f70 100644 --- a/playbooks/reproducible.archlinux.org.yml +++ b/playbooks/reproducible.archlinux.org.yml @@ -14,3 +14,4 @@ - { role: certbot } - { role: nginx } - { role: rebuilderd } + - { role: prometheus_exporters } diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml index c5e18fab1998a96f2988a348b9d63dffc19aee20..f1a70b46cb7035311966dd3974cf9de978016026 100644 --- a/roles/gitlab_runner/tasks/main.yml +++ b/roles/gitlab_runner/tasks/main.yml @@ -40,5 +40,12 @@ line: concurrent = 100 notify: restart gitlab-runner +- name: enable prometheus exporter + lineinfile: + path: /etc/gitlab-runner/config.toml + insertbefore: '^concurrent' + line: listen_address = ":{{ gitlab_runner_exporter_port }}" + notify: restart gitlab-runner + - name: enable and start gitlab runner service systemd: name=gitlab-runner state=started enabled=yes daemon_reload=yes diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 3b75be8a27d4a8258a72c033aa01383f4ae32fa7..47660d86680e1f8ed3b1adc6c5074c317e281418 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1 +1,2 @@ monitoring_domain: monitoring.archlinux.org +gitlab_runner_exporter_port: '9252' diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index c868c7545ac9888ec7d53983aa17b6f6c6d200c0..1da101edbc1922fb856e830fac10aa9579e1aa05 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -13,3 +13,69 @@ alerting: - localhost:9093 scrape_configs: + - job_name: 'node_exporter' + static_configs: + {% for host in groups['node_exporters'] %} + + - targets: ['{{ host }}:{{ prometheus_exporter_port }}'] + labels: + instance: "{{ host }}" + + {% endfor %} + + - job_name: 'gitlab_runner_exporter' + static_configs: + {% for host in groups['gitlab_runners'] %} + + - targets: ['{{ host }}:{{ gitlab_runner_exporter_port }}'] + labels: + instance: "{{ host }}" + + {% endfor %} + + - job_name: 'keycloak' + scheme: https + metrics_path: "/auth/realms/master/metrics" + basic_auth: + username: "{{ vault_keycloak_nginx_user }}" + password: "{{ vault_keycloak_nginx_passwd }}" + static_configs: + - targets: ['accounts.archlinux.org:443'] + labels: + instance: "accounts.archlinux.org" + + - job_name: 'gitlab_exporter' + scheme: https + metrics_path: "-/metrics" + params: + token: ["{{ vault_gitlab_prometheus_token }}"] + static_configs: + - targets: ['gitlab.archlinux.org:443'] + labels: + instance: "gitlab.archlinux.org" + + - job_name: 'mysqld_exporter' + static_configs: + + - targets: ['aur.archlinux.org:9104'] + labels: + instance: "aur.archlinux.org" + + - job_name: 'blackbox' + metrics_path: /probe + scrape_interval: 15s + params: + module: [http_prometheus] + static_configs: + - targets: + {% for target in blackbox_targets %} + - {{ target }} + {% endfor %} + + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 diff --git a/roles/prometheus_exporters/defaults/main.yml b/roles/prometheus_exporters/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..b66b5c4602da8ff860fd2d17f6aa9d73816a54f8 --- /dev/null +++ b/roles/prometheus_exporters/defaults/main.yml @@ -0,0 +1,10 @@ +--- + +prometheus_domain: monitoring.archlinux.org + +prometheus_textfile_dir: /var/lib/node_exporter + +gitlab_runner_exporter_port: '9252' + +prometheus_mysqld_user: mysqld_exporter +prometheus_mysqld_exporter_port: '9104' diff --git a/roles/prometheus_exporters/files/arch-textcollector.sh b/roles/prometheus_exporters/files/arch-textcollector.sh new file mode 100755 index 0000000000000000000000000000000000000000..963a851a9e2538acfae9aab6eb28770d70a6747e --- /dev/null +++ b/roles/prometheus_exporters/files/arch-textcollector.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +if (( $# != 1 )); then + echo "Missing textcollector directory argument" + exit 1 +fi + +HOSTNAME=$(hostname) +TEXTFILE_COLLECTOR_DIR=${1} +PROM_FILE=$TEXTFILE_COLLECTOR_DIR/pacman.prom + +TMP_FILE=$PROM_FILE.$$ +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +trap "rm -f $TMP_FILE" EXIT + +updates=$(/usr/bin/checkupdates | wc -l) +secupdates=$(/usr/bin/arch-audit -u | wc -l) + +echo "# HELP pacman_updates_pending number of pending updates from pacman" >> $TMP_FILE +echo "# TYPE pacman_updates_pending gauge" >> $TMP_FILE +echo "pacman_updates_pending{host=\"${HOSTNAME}\"} $updates" >> $TMP_FILE + +echo "# HELP pacman_security_updates_pending number of pending updates from pacman" >> $TMP_FILE +echo "# TYPE pacman_security_updates_pending gauge" >> $TMP_FILE +echo "pacman_security_updates_pending{host=\"${HOSTNAME}\"} $secupdates" >> $TMP_FILE + +mv -f $TMP_FILE $PROM_FILE diff --git a/roles/prometheus_exporters/files/borg-textcollector.sh b/roles/prometheus_exporters/files/borg-textcollector.sh new file mode 100755 index 0000000000000000000000000000000000000000..14de62849c3ab737020a159288bb9b865fa16b34 --- /dev/null +++ b/roles/prometheus_exporters/files/borg-textcollector.sh @@ -0,0 +1,46 @@ +#!/usr/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +if (( $# != 1 )); then + echo "Missing textcollector directory argument" + exit 1 +fi + +HOSTNAME=$(hostname) +TEXTFILE_COLLECTOR_DIR=${1} +PROM_FILE=$TEXTFILE_COLLECTOR_DIR/borg.prom + + +TMP_FILE=$PROM_FILE.$$ +[ -e $TMP_FILE ] && rm -f $TMP_FILE + +trap "rm -f $TMP_FILE" EXIT + +# Hetzner borg +if [[ -f /usr/local/bin/borg ]]; then + LAST_ARCHIVE=$(/usr/local/bin/borg list --last 1) + LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}') + LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}') + LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s") + + echo "# HELP borg_hetzner_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE + echo "# TYPE borg_hetzner_last_archive_timestamp counter" >> $TMP_FILE + echo "borg_hetzner_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE; +fi + +# rsync.net borg +if [[ -f /usr/local/bin/borg-offsite ]]; then + LAST_ARCHIVE=$(/usr/local/bin/borg-offsite list --last 1) + LAST_ARCHIVE_NAME=$(echo $LAST_ARCHIVE | awk '{print $1}') + LAST_ARCHIVE_DATE=$(echo $LAST_ARCHIVE | awk '{print $3" "$4}') + LAST_ARCHIVE_TIMESTAMP=$(date -d "$LAST_ARCHIVE_DATE" +"%s") + + echo "# HELP borg_offsite_last_archive_timestamp timestamp of last backup in UTC" >> $TMP_FILE + echo "# TYPE borg_offsite_last_archive_timestamp counter" >> $TMP_FILE + echo "borg_offsite_last_archive_timestamp{host=\"${HOSTNAME}\"} $LAST_ARCHIVE_TIMESTAMP" >> $TMP_FILE; +fi + +mv -f $TMP_FILE $PROM_FILE diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..60ea1ee36387a3e35b1d352ef642e4d30e2d53be --- /dev/null +++ b/roles/prometheus_exporters/tasks/main.yml @@ -0,0 +1,88 @@ +--- + +- name: install prometheus-node-exporter + pacman: name=prometheus-node-exporter,arch-audit,pacman-contrib state=present + +- name: install prometheus-memcached-exporter + pacman: name=prometheus-memcached-exporter state=present + when: "'memcached' in group_names" + +- name: install prometheus-mysqld-exporter + pacman: name=prometheus-mysqld-exporter state=present + when: "'mysql_servers' in group_names" + +- name: create prometheus mysqld database user + mysql_user: + name: '{{ prometheus_mysqld_user }}' + password: '{{ vault_prometheus_mysql_password }}' + priv: "*.*:PROCESS,REPLICATION CLIENT" + state: present + when: "'mysql_servers' in group_names" + +# TODO: with ansible 2.10 this can be set by mysql_user https://github.com/ansible/ansible/issues/26581 +- name: set max_user_connections for prometheus mysqld user + command: mysql -u root -e "ALTER USER '{{ prometheus_mysqld_user }}'@'localhost' WITH MAX_USER_CONNECTIONS 3;" + when: "'mysql_servers' in group_names" + +- name: copy prometheus mysqld exporter configuration + template: src=prometheus-mysqld-exporter.j2 dest=/etc/conf.d/prometheus-mysqld-exporter owner=root group=root mode=600 + when: "'mysql_servers' in group_names" + +- name: enable prometheus-mysqld-exporter service + systemd: name=prometheus-mysqld-exporter enabled=yes daemon_reload=yes state=started + when: "'mysql_servers' in group_names" + +- name: install node exporter configuration + template: src=prometheus-node-exporter.env.j2 dest=/etc/conf.d/prometheus-node-exporter owner=root group=root mode=600 + +- name: create textcollector directory + file: path="{{ prometheus_textfile_dir }}" state=directory owner=node_exporter group=node_exporter mode=700 + +- name: install node exporter textcollector scripts + copy: src={{ item }} dest=/usr/local/bin/{{ item }} owner=root group=root mode=0755 + with_items: + - arch-textcollector.sh + - borg-textcollector.sh + +- name: install arch textcollector service + template: src=prometheus-arch-textcollector.service.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.service owner=root group=root mode=600 + +- name: install arch textcollector timer + template: src=prometheus-arch-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-arch-textcollector.timer owner=root group=root mode=600 + +- name: enable and start prometheus arch textcollector timer + systemd: name=prometheus-arch-textcollector.timer enabled=yes daemon_reload=yes state=started + +- name: install borg textcollector service + template: src=prometheus-borg-textcollector.service.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.service owner=root group=root mode=600 + when: "'borg_clients' in group_names" + +- name: install borg textcollector timer + template: src=prometheus-borg-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-borg-textcollector.timer owner=root group=root mode=600 + when: "'borg_clients' in group_names" + +- name: enable and start prometheus borg textcollector timer + systemd: name=prometheus-borg-textcollector.timer enabled=yes daemon_reload=yes state=started + when: "'borg_clients' in group_names" + +- name: enable prometheus-node-exporter service + systemd: name=prometheus-node-exporter enabled=yes daemon_reload=yes state=started + +- name: enable prometheus-memcached-exporter service + systemd: name=prometheus-memcached-exporter enabled=yes daemon_reload=yes state=started + when: "'memcached' in group_names" + +- name: open prometheus-node-exporter ipv4 port for monitoring.archlinux.org + firewalld: state=enabled permanent=true immediate=yes + rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_exporter_port }} accept" + when: "'prometheus' not in group_names" + +- name: open gitlab exporter ipv4 port for monitoring.archlinux.org + firewalld: state=enabled permanent=true immediate=yes + rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ gitlab_runner_exporter_port }} accept" + when: "'gitlab_runners' in group_names" + +- name: open prometheus mysqld exporter ipv4 port for monitoring.archlinux.org + firewalld: state=enabled permanent=true immediate=yes + rich_rule="rule family=ipv4 source address={{ hostvars['monitoring.archlinux.org']['ipv4_address'] }} port protocol=tcp port={{ prometheus_mysqld_exporter_port }} accept" + when: "'mysql_servers' in group_names" diff --git a/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..5edb6dc89f1b22f9479cdf3b8e5bc8c62a687b0b --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.service.j2 @@ -0,0 +1,37 @@ +[Unit] +Description=Prometheus Arch Exporter +After=network.target + +[Service] +Type=oneshot +User=node_exporter +ExecStart=/usr/local/bin/arch-textcollector.sh {{ prometheus_textfile_dir }} + +NoNewPrivileges=true +LockPersonality=true +CapabilityBoundingSet= +UMask=077 + +PrivateDevices=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths={{ prometheus_textfile_dir }} + +MemoryDenyWriteExecute=true +RemoveIPC=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictSUIDSGID=true + +RestrictAddressFamilies=~AF_NETLINK +RestrictAddressFamilies=~AF_PACKET + +ProtectHostname=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectClock=true + +SystemCallArchitectures=native diff --git a/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 new file mode 100644 index 0000000000000000000000000000000000000000..6afeacf4dc4428cdd3b2ca2a16d7f1a2e052ec50 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-arch-textcollector.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus Arch Exporter TextCollector Timer + +[Timer] +OnUnitActiveSec=60m +OnBootSec=15min +RandomizedDelaySec=1min + +[Install] +WantedBy=timers.target diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..593a774eef4860048b1bc2049752d69cdb443531 --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.service.j2 @@ -0,0 +1,35 @@ +[Unit] +Description=Prometheus Borg Exporter TextCollector +After=network.target +ConditionPathExistsGlob=!/root/.cache/borg/*/lock.roster + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/borg-textcollector.sh {{ prometheus_textfile_dir }} + +NoNewPrivileges=true +LockPersonality=true + +PrivateDevices=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={{ prometheus_textfile_dir }} /root/.cache/borg + +MemoryDenyWriteExecute=true +RemoveIPC=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictSUIDSGID=true + +RestrictAddressFamilies=~AF_PACKET +RestrictAddressFamilies=~AF_NETLINK + +ProtectHostname=true +ProtectControlGroups=true +ProtectKernelLogs=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectClock=true + +SystemCallArchitectures=native diff --git a/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 new file mode 100644 index 0000000000000000000000000000000000000000..ca8a197e20efe0879cb411fe4fc720324a24785f --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-borg-textcollector.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus Borg Exporter TextCollector Timer + +[Timer] +OnUnitActiveSec=1h +OnBootSec=15min +RandomizedDelaySec=1min + +[Install] +WantedBy=timers.target diff --git a/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 b/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 new file mode 100644 index 0000000000000000000000000000000000000000..c74feee70fcb6a05aac0775046c1463796b258ab --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-mysqld-exporter.j2 @@ -0,0 +1,3 @@ +DATA_SOURCE_NAME="{{ prometheus_mysqld_user }}:{{ vault_prometheus_mysql_password }}@(localhost:3306)/" +# TODO: review these settings +MYSQLD_EXPORTER_ARGS="--collect.binlog_size --collect.info_schema.processlist --collect.info_schema.userstats" diff --git a/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 b/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 new file mode 100644 index 0000000000000000000000000000000000000000..88dd42d6d790b9f87e13c1ea4715bcabb8cc79fa --- /dev/null +++ b/roles/prometheus_exporters/templates/prometheus-node-exporter.env.j2 @@ -0,0 +1 @@ +NODE_EXPORTER_ARGS="--collector.systemd --collector.textfile.directory={{ prometheus_textfile_dir }}"