Verified Commit 8decc2e9 authored by Frederik Schwan's avatar Frederik Schwan
Browse files

use fetchmail for donor import

The former approach to export a maildir and iterate over it with a
script broke when the mail server and the web server got on their
own hosts. This will use IMAP IDLE to check for new mails and pass
them instantly to the djange manage.py script without storing the mail
locally.
parent 040595fe
---
filesystem: btrfs
fetchmail_user: "donate@archlinux.org"
fetchmail_delivery_cmd: "/usr/local/bin/donor_import_wrapper.sh"
ipv4_address: "95.217.163.246"
ipv6_address: "2a01:4f9:c010:6b1f::1"
......
$ANSIBLE_VAULT;1.1;AES256
30343863353037356134636339633536323666663861393166653963303537303365656531643830
3862336361376565346630313638393632333931333939350a653338623261633739396464333236
30613562663166653039616433353239313164653233373531343734643934643962646662366264
3761663633333036310a363166343037616266613235343937343465646332666235333864396333
30623733346138393864666436376236663263393661623762366237626331356630373762666138
61386262623533326265613966356636666665643430386561343563306431623137313932643334
303930343963306437396165663235353739
$ANSIBLE_VAULT;1.1;AES256
38343839616266323863666465663431343935333663613637656139646631336431376131663764
3739393932303339353936313034313831633335316332620a383364313566333862376636373663
31343439613834333338333633326232386263616665383561663936316665363935366165346165
6433383735386261300a633862333661303639353734306333366233333237313162623562386130
39383062333138343235613135663036386666636537626661336132643139323239323938643932
35343034346534356437656638333862386230323063376661333939633532306539333035316537
613739623366326235356137623633386137
39323465346236343761323262393437336164613164653665626235386561383335396533373238
6432656264613366353930343763386534646131656361370a366335356139383534373233333665
37393032356238313932366663383237663162326164393637303866333862353032303130353362
3933613433313964360a373739303336653333386332323638633335383337313334623635326266
62363863643166396530666366313837346365306230363263643335393334303034636332356538
35306532643439623235313933633562323264363431643232346662383639306438333238313766
613838336337313665643837353135383063
......@@ -34,6 +34,7 @@
- { role: sudo }
- { role: uwsgi }
- { role: memcached }
- { role: fetchmail }
- { role: archweb, archweb_planet: true }
- { role: fail2ban }
- { role: prometheus_exporters }
......@@ -20,9 +20,7 @@ archweb_site: true
archweb_mirrorcheck: false
archweb_mirrorresolv: false
archweb_pgp_import: false
archweb_donor_import: false
archweb_planet: false
archweb_donor_maildir: '/var/lib/archweb/donate'
archweb_populate_signoffs: false
archweb_keyring: '/etc/pacman.d/gnupg/pubring.gpg'
archweb_reporead: false
......
......@@ -204,37 +204,6 @@
- daemon reload
when: archweb_site|bool
- name: install donor_import service and timer
template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644
with_items:
- archweb-donor_import.service
- archweb-donor_import.timer
notify:
- daemon reload
when: archweb_donor_import|bool
- name: create maildir dump dir
file: path={{ archweb_donor_maildir }}/{{ item }} recurse=yes state=directory owner=archweb group=users mode=0755
with_items:
- cur
- new
when: archweb_donor_import|bool
- name: install dump donor mail service and timer
template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644
with_items:
- archweb-dump_donor_mail.service
- archweb-dump_donor_mail.timer
notify:
- daemon reload
when: archweb_donor_import|bool
- name: install archweb donor mail dump script
template: src="dump-mails.sh.j2" dest="/usr/local/bin/dump-mails.sh" owner=root group=root mode=0755
notify:
- daemon reload
when: archweb_donor_import|bool
- name: deploy archweb
template: src=archweb.ini.j2 dest=/etc/uwsgi/vassals/archweb.ini owner=archweb group=http mode=0640
when: archweb_site|bool
......@@ -283,16 +252,11 @@
service: name="archweb-rebuilderd.timer" enabled=yes state=started
when: archweb_site
- name: start and enable archweb donor_import and archweb-dump_donor_mail timer
systemd:
name: "{{ item }}"
enabled: yes
state: started
daemon_reload: yes
with_items:
- archweb-donor_import.timer
- archweb-dump_donor_mail.timer
when: archweb_donor_import|bool
- name: install donation import wrapper script
template: src=donor_import_wrapper.sh.j2 dest=/usr/local/bin/donor_import_wrapper.sh owner=root group=root mode=0755
- name: install sudoer rights for fetchmail to call archweb django scripts
template: src=sudoers-fetchmail-archweb.j2 dest=/etc/sudoers.d/fetchmail-archweb owner=root group=root mode=0440
- name: create retro dir
file: state=directory owner=archweb group=archweb mode=0755 path="{{ archweb_retro_dir }}"
......
[Unit]
Description=archweb donor_import service
After=network.target
[Service]
Type=oneshot
User=archweb
WorkingDirectory={{ archweb_dir }}
ExecStart={{ archweb_dir }}/env/bin/python manage.py donor_import {{ archweb_donor_maildir }}
Nice=5
[Install]
WantedBy=multi-user.target
[Unit]
Description=archweb donor_import timer
[Timer]
OnUnitActiveSec=1w
OnBootSec=15min
RandomizedDelaySec=1h
[Install]
WantedBy=timers.target
[Unit]
Description=dump donate mail service
After=network.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/dump-mails.sh
Nice=5
[Install]
WantedBy=multi-user.target
[Unit]
Description=dump donate mail timer
[Timer]
OnUnitActiveSec=1w
OnBootSec=15min
RandomizedDelaySec=1h
[Install]
WantedBy=timers.target
#!/bin/bash
sudo -u archweb /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import
#!/bin/bash
set -e
shopt -s nullglob
dump_mails() {
target_dir="$1"; shift
date=$(date +%s)
searchquery=(all)
for user in "$@"; do
mailcount=$(doveadm search -u "$user" "${searchquery[@]}" | wc -l)
if ((mailcount > 0)); then
echo "dumping $mailcount mails from $user ..." >&2
doveadm search -u "$user" "${searchquery[@]}" | while read guid uid; do
doveadm fetch -u "$user" text mailbox-guid "$guid" uid "$uid" | sed '/^text:$/d; s#^\f$##' > "$target_dir/$user-$date-$uid.msg"
#doveadm expunge -u "$user" mailbox-guid "$guid" uid "$uid"
done
fi
done
}
dump_mails "{{ archweb_donor_maildir }}/cur" donate
fetchmail ALL=(archweb) NOPASSWD: /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import
......@@ -8,6 +8,6 @@ poll mail.archlinux.org
bad-header accept
proto imap
user {{ fetchmail_user }}
password {{ postfix_relay_password }}
password {{ fetchmail_password }}
options idle sslcertck ssl sslproto "TLS1.2+" limitflush limit 25000000 fetchall
mda "{{ fetchmail_delivery_cmd }}"
fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON=/srv/http/patchwork/env/bin/python /srv/http/patchwork/patchwork/bin/parsemail.sh
......@@ -101,7 +101,7 @@
template: src="patchwork-parsemail-wrapper.sh.j2" dest="/usr/local/bin/patchwork-parsemail-wrapper.sh" owner=root group=root mode=0755
- name: install sudoer rights for fetchmail to call patchwork
copy: src=fetchmail-patchwork dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440
template: src=sudoers-fetchmail-patchwork.j2 dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440
- name: install patchwork memcached service
template: src="patchwork-memcached.service.j2" dest="/etc/systemd/system/patchwork-memcached.service" owner=root group=root mode=0644
......
fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON={{ patchwork_dir }}/env/bin/python {{ patchwork_dir }}/patchwork/bin/parsemail.sh
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment