From 8decc2e977dfd12304c941d969b04d5f6880801f Mon Sep 17 00:00:00 2001 From: Frederik Schwan Date: Tue, 29 Dec 2020 01:30:18 +0100 Subject: [PATCH] use fetchmail for donor import The former approach to export a maildir and iterate over it with a script broke when the mail server and the web server got on their own hosts. This will use IMAP IDLE to check for new mails and pass them instantly to the djange manage.py script without storing the mail locally. --- .../{archlinux.org => archlinux.org/misc} | 2 + host_vars/archlinux.org/vault_archlinux.yml | 8 ++++ .../vault_patchwork.yml | 14 +++--- playbooks/archlinux.org.yml | 1 + roles/archweb/defaults/main.yml | 2 - roles/archweb/tasks/main.yml | 46 ++----------------- .../templates/archweb-donor_import.service.j2 | 13 ------ .../templates/archweb-donor_import.timer.j2 | 10 ---- .../archweb-dump_donor_mail.service.j2 | 11 ----- .../archweb-dump_donor_mail.timer.j2 | 10 ---- .../templates/donor_import_wrapper.sh.j2 | 3 ++ roles/archweb/templates/dump-mails.sh.j2 | 24 ---------- .../templates/sudoers-fetchmail-archweb.j2 | 1 + roles/fetchmail/templates/fetchmailrc.j2 | 2 +- roles/patchwork/files/fetchmail-patchwork | 1 - roles/patchwork/tasks/main.yml | 2 +- .../templates/sudoers-fetchmail-patchwork.j2 | 1 + 17 files changed, 30 insertions(+), 121 deletions(-) rename host_vars/{archlinux.org => archlinux.org/misc} (60%) create mode 100644 host_vars/archlinux.org/vault_archlinux.yml delete mode 100644 roles/archweb/templates/archweb-donor_import.service.j2 delete mode 100644 roles/archweb/templates/archweb-donor_import.timer.j2 delete mode 100644 roles/archweb/templates/archweb-dump_donor_mail.service.j2 delete mode 100644 roles/archweb/templates/archweb-dump_donor_mail.timer.j2 create mode 100644 roles/archweb/templates/donor_import_wrapper.sh.j2 delete mode 100755 roles/archweb/templates/dump-mails.sh.j2 create mode 100644 roles/archweb/templates/sudoers-fetchmail-archweb.j2 delete mode 100644 roles/patchwork/files/fetchmail-patchwork create mode 100644 roles/patchwork/templates/sudoers-fetchmail-patchwork.j2 diff --git a/host_vars/archlinux.org b/host_vars/archlinux.org/misc similarity index 60% rename from host_vars/archlinux.org rename to host_vars/archlinux.org/misc index d8443e35..cc31925d 100644 --- a/host_vars/archlinux.org +++ b/host_vars/archlinux.org/misc @@ -1,5 +1,7 @@ --- filesystem: btrfs +fetchmail_user: "donate@archlinux.org" +fetchmail_delivery_cmd: "/usr/local/bin/donor_import_wrapper.sh" ipv4_address: "95.217.163.246" ipv6_address: "2a01:4f9:c010:6b1f::1" diff --git a/host_vars/archlinux.org/vault_archlinux.yml b/host_vars/archlinux.org/vault_archlinux.yml new file mode 100644 index 00000000..8dffcd10 --- /dev/null +++ b/host_vars/archlinux.org/vault_archlinux.yml @@ -0,0 +1,8 @@ +$ANSIBLE_VAULT;1.1;AES256 +30343863353037356134636339633536323666663861393166653963303537303365656531643830 +3862336361376565346630313638393632333931333939350a653338623261633739396464333236 +30613562663166653039616433353239313164653233373531343734643934643962646662366264 +3761663633333036310a363166343037616266613235343937343465646332666235333864396333 +30623733346138393864666436376236663263393661623762366237626331356630373762666138 +61386262623533326265613966356636666665643430386561343563306431623137313932643334 +303930343963306437396165663235353739 diff --git a/host_vars/patchwork.archlinux.org/vault_patchwork.yml b/host_vars/patchwork.archlinux.org/vault_patchwork.yml index b1873063..2bf5eb7a 100644 --- a/host_vars/patchwork.archlinux.org/vault_patchwork.yml +++ b/host_vars/patchwork.archlinux.org/vault_patchwork.yml @@ -1,8 +1,8 @@ $ANSIBLE_VAULT;1.1;AES256 -38343839616266323863666465663431343935333663613637656139646631336431376131663764 -3739393932303339353936313034313831633335316332620a383364313566333862376636373663 -31343439613834333338333633326232386263616665383561663936316665363935366165346165 -6433383735386261300a633862333661303639353734306333366233333237313162623562386130 -39383062333138343235613135663036386666636537626661336132643139323239323938643932 -35343034346534356437656638333862386230323063376661333939633532306539333035316537 -613739623366326235356137623633386137 +39323465346236343761323262393437336164613164653665626235386561383335396533373238 +6432656264613366353930343763386534646131656361370a366335356139383534373233333665 +37393032356238313932366663383237663162326164393637303866333862353032303130353362 +3933613433313964360a373739303336653333386332323638633335383337313334623635326266 +62363863643166396530666366313837346365306230363263643335393334303034636332356538 +35306532643439623235313933633562323264363431643232346662383639306438333238313766 +613838336337313665643837353135383063 diff --git a/playbooks/archlinux.org.yml b/playbooks/archlinux.org.yml index 6b63d70a..6bc84acc 100644 --- a/playbooks/archlinux.org.yml +++ b/playbooks/archlinux.org.yml @@ -34,6 +34,7 @@ - { role: sudo } - { role: uwsgi } - { role: memcached } + - { role: fetchmail } - { role: archweb, archweb_planet: true } - { role: fail2ban } - { role: prometheus_exporters } diff --git a/roles/archweb/defaults/main.yml b/roles/archweb/defaults/main.yml index d0deb7f4..c7b500d7 100644 --- a/roles/archweb/defaults/main.yml +++ b/roles/archweb/defaults/main.yml @@ -20,9 +20,7 @@ archweb_site: true archweb_mirrorcheck: false archweb_mirrorresolv: false archweb_pgp_import: false -archweb_donor_import: false archweb_planet: false -archweb_donor_maildir: '/var/lib/archweb/donate' archweb_populate_signoffs: false archweb_keyring: '/etc/pacman.d/gnupg/pubring.gpg' archweb_reporead: false diff --git a/roles/archweb/tasks/main.yml b/roles/archweb/tasks/main.yml index 4c03e5f9..6fa54119 100644 --- a/roles/archweb/tasks/main.yml +++ b/roles/archweb/tasks/main.yml @@ -204,37 +204,6 @@ - daemon reload when: archweb_site|bool -- name: install donor_import service and timer - template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644 - with_items: - - archweb-donor_import.service - - archweb-donor_import.timer - notify: - - daemon reload - when: archweb_donor_import|bool - -- name: create maildir dump dir - file: path={{ archweb_donor_maildir }}/{{ item }} recurse=yes state=directory owner=archweb group=users mode=0755 - with_items: - - cur - - new - when: archweb_donor_import|bool - -- name: install dump donor mail service and timer - template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644 - with_items: - - archweb-dump_donor_mail.service - - archweb-dump_donor_mail.timer - notify: - - daemon reload - when: archweb_donor_import|bool - -- name: install archweb donor mail dump script - template: src="dump-mails.sh.j2" dest="/usr/local/bin/dump-mails.sh" owner=root group=root mode=0755 - notify: - - daemon reload - when: archweb_donor_import|bool - - name: deploy archweb template: src=archweb.ini.j2 dest=/etc/uwsgi/vassals/archweb.ini owner=archweb group=http mode=0640 when: archweb_site|bool @@ -283,16 +252,11 @@ service: name="archweb-rebuilderd.timer" enabled=yes state=started when: archweb_site -- name: start and enable archweb donor_import and archweb-dump_donor_mail timer - systemd: - name: "{{ item }}" - enabled: yes - state: started - daemon_reload: yes - with_items: - - archweb-donor_import.timer - - archweb-dump_donor_mail.timer - when: archweb_donor_import|bool +- name: install donation import wrapper script + template: src=donor_import_wrapper.sh.j2 dest=/usr/local/bin/donor_import_wrapper.sh owner=root group=root mode=0755 + +- name: install sudoer rights for fetchmail to call archweb django scripts + template: src=sudoers-fetchmail-archweb.j2 dest=/etc/sudoers.d/fetchmail-archweb owner=root group=root mode=0440 - name: create retro dir file: state=directory owner=archweb group=archweb mode=0755 path="{{ archweb_retro_dir }}" diff --git a/roles/archweb/templates/archweb-donor_import.service.j2 b/roles/archweb/templates/archweb-donor_import.service.j2 deleted file mode 100644 index 19da4294..00000000 --- a/roles/archweb/templates/archweb-donor_import.service.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=archweb donor_import service -After=network.target - -[Service] -Type=oneshot -User=archweb -WorkingDirectory={{ archweb_dir }} -ExecStart={{ archweb_dir }}/env/bin/python manage.py donor_import {{ archweb_donor_maildir }} -Nice=5 - -[Install] -WantedBy=multi-user.target diff --git a/roles/archweb/templates/archweb-donor_import.timer.j2 b/roles/archweb/templates/archweb-donor_import.timer.j2 deleted file mode 100644 index 0fb976fd..00000000 --- a/roles/archweb/templates/archweb-donor_import.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=archweb donor_import timer - -[Timer] -OnUnitActiveSec=1w -OnBootSec=15min -RandomizedDelaySec=1h - -[Install] -WantedBy=timers.target diff --git a/roles/archweb/templates/archweb-dump_donor_mail.service.j2 b/roles/archweb/templates/archweb-dump_donor_mail.service.j2 deleted file mode 100644 index df9ddcf2..00000000 --- a/roles/archweb/templates/archweb-dump_donor_mail.service.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=dump donate mail service -After=network.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/dump-mails.sh -Nice=5 - -[Install] -WantedBy=multi-user.target diff --git a/roles/archweb/templates/archweb-dump_donor_mail.timer.j2 b/roles/archweb/templates/archweb-dump_donor_mail.timer.j2 deleted file mode 100644 index f7cdefdd..00000000 --- a/roles/archweb/templates/archweb-dump_donor_mail.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=dump donate mail timer - -[Timer] -OnUnitActiveSec=1w -OnBootSec=15min -RandomizedDelaySec=1h - -[Install] -WantedBy=timers.target diff --git a/roles/archweb/templates/donor_import_wrapper.sh.j2 b/roles/archweb/templates/donor_import_wrapper.sh.j2 new file mode 100644 index 00000000..8814025f --- /dev/null +++ b/roles/archweb/templates/donor_import_wrapper.sh.j2 @@ -0,0 +1,3 @@ +#!/bin/bash + +sudo -u archweb /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import diff --git a/roles/archweb/templates/dump-mails.sh.j2 b/roles/archweb/templates/dump-mails.sh.j2 deleted file mode 100755 index 6f48b5f0..00000000 --- a/roles/archweb/templates/dump-mails.sh.j2 +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -e -shopt -s nullglob - -dump_mails() { - target_dir="$1"; shift - date=$(date +%s) - searchquery=(all) - - for user in "$@"; do - mailcount=$(doveadm search -u "$user" "${searchquery[@]}" | wc -l) - if ((mailcount > 0)); then - echo "dumping $mailcount mails from $user ..." >&2 - doveadm search -u "$user" "${searchquery[@]}" | while read guid uid; do - doveadm fetch -u "$user" text mailbox-guid "$guid" uid "$uid" | sed '/^text:$/d; s#^\f$##' > "$target_dir/$user-$date-$uid.msg" - #doveadm expunge -u "$user" mailbox-guid "$guid" uid "$uid" - done - fi - done -} - - -dump_mails "{{ archweb_donor_maildir }}/cur" donate diff --git a/roles/archweb/templates/sudoers-fetchmail-archweb.j2 b/roles/archweb/templates/sudoers-fetchmail-archweb.j2 new file mode 100644 index 00000000..8444a41e --- /dev/null +++ b/roles/archweb/templates/sudoers-fetchmail-archweb.j2 @@ -0,0 +1 @@ +fetchmail ALL=(archweb) NOPASSWD: /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import diff --git a/roles/fetchmail/templates/fetchmailrc.j2 b/roles/fetchmail/templates/fetchmailrc.j2 index 2064648d..81d4bd04 100644 --- a/roles/fetchmail/templates/fetchmailrc.j2 +++ b/roles/fetchmail/templates/fetchmailrc.j2 @@ -8,6 +8,6 @@ poll mail.archlinux.org bad-header accept proto imap user {{ fetchmail_user }} -password {{ postfix_relay_password }} +password {{ fetchmail_password }} options idle sslcertck ssl sslproto "TLS1.2+" limitflush limit 25000000 fetchall mda "{{ fetchmail_delivery_cmd }}" diff --git a/roles/patchwork/files/fetchmail-patchwork b/roles/patchwork/files/fetchmail-patchwork deleted file mode 100644 index a8a981e4..00000000 --- a/roles/patchwork/files/fetchmail-patchwork +++ /dev/null @@ -1 +0,0 @@ -fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON=/srv/http/patchwork/env/bin/python /srv/http/patchwork/patchwork/bin/parsemail.sh diff --git a/roles/patchwork/tasks/main.yml b/roles/patchwork/tasks/main.yml index 0be35cf2..134d0b8b 100644 --- a/roles/patchwork/tasks/main.yml +++ b/roles/patchwork/tasks/main.yml @@ -101,7 +101,7 @@ template: src="patchwork-parsemail-wrapper.sh.j2" dest="/usr/local/bin/patchwork-parsemail-wrapper.sh" owner=root group=root mode=0755 - name: install sudoer rights for fetchmail to call patchwork - copy: src=fetchmail-patchwork dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440 + template: src=sudoers-fetchmail-patchwork.j2 dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440 - name: install patchwork memcached service template: src="patchwork-memcached.service.j2" dest="/etc/systemd/system/patchwork-memcached.service" owner=root group=root mode=0644 diff --git a/roles/patchwork/templates/sudoers-fetchmail-patchwork.j2 b/roles/patchwork/templates/sudoers-fetchmail-patchwork.j2 new file mode 100644 index 00000000..e0e2b2fd --- /dev/null +++ b/roles/patchwork/templates/sudoers-fetchmail-patchwork.j2 @@ -0,0 +1 @@ +fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON={{ patchwork_dir }}/env/bin/python {{ patchwork_dir }}/patchwork/bin/parsemail.sh -- GitLab