Commit 62276551 authored by Jelle van der Waa's avatar Jelle van der Waa 🚧
Browse files

Merge branch 'fix-donate' into 'master'

use fetchmail for donor import

See merge request !248
parents 040595fe 8decc2e9
Pipeline #4288 passed with stage
in 45 seconds
--- ---
filesystem: btrfs filesystem: btrfs
fetchmail_user: "donate@archlinux.org"
fetchmail_delivery_cmd: "/usr/local/bin/donor_import_wrapper.sh"
ipv4_address: "95.217.163.246" ipv4_address: "95.217.163.246"
ipv6_address: "2a01:4f9:c010:6b1f::1" ipv6_address: "2a01:4f9:c010:6b1f::1"
......
$ANSIBLE_VAULT;1.1;AES256
30343863353037356134636339633536323666663861393166653963303537303365656531643830
3862336361376565346630313638393632333931333939350a653338623261633739396464333236
30613562663166653039616433353239313164653233373531343734643934643962646662366264
3761663633333036310a363166343037616266613235343937343465646332666235333864396333
30623733346138393864666436376236663263393661623762366237626331356630373762666138
61386262623533326265613966356636666665643430386561343563306431623137313932643334
303930343963306437396165663235353739
$ANSIBLE_VAULT;1.1;AES256 $ANSIBLE_VAULT;1.1;AES256
38343839616266323863666465663431343935333663613637656139646631336431376131663764 39323465346236343761323262393437336164613164653665626235386561383335396533373238
3739393932303339353936313034313831633335316332620a383364313566333862376636373663 6432656264613366353930343763386534646131656361370a366335356139383534373233333665
31343439613834333338333633326232386263616665383561663936316665363935366165346165 37393032356238313932366663383237663162326164393637303866333862353032303130353362
6433383735386261300a633862333661303639353734306333366233333237313162623562386130 3933613433313964360a373739303336653333386332323638633335383337313334623635326266
39383062333138343235613135663036386666636537626661336132643139323239323938643932 62363863643166396530666366313837346365306230363263643335393334303034636332356538
35343034346534356437656638333862386230323063376661333939633532306539333035316537 35306532643439623235313933633562323264363431643232346662383639306438333238313766
613739623366326235356137623633386137 613838336337313665643837353135383063
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
- { role: sudo } - { role: sudo }
- { role: uwsgi } - { role: uwsgi }
- { role: memcached } - { role: memcached }
- { role: fetchmail }
- { role: archweb, archweb_planet: true } - { role: archweb, archweb_planet: true }
- { role: fail2ban } - { role: fail2ban }
- { role: prometheus_exporters } - { role: prometheus_exporters }
...@@ -20,9 +20,7 @@ archweb_site: true ...@@ -20,9 +20,7 @@ archweb_site: true
archweb_mirrorcheck: false archweb_mirrorcheck: false
archweb_mirrorresolv: false archweb_mirrorresolv: false
archweb_pgp_import: false archweb_pgp_import: false
archweb_donor_import: false
archweb_planet: false archweb_planet: false
archweb_donor_maildir: '/var/lib/archweb/donate'
archweb_populate_signoffs: false archweb_populate_signoffs: false
archweb_keyring: '/etc/pacman.d/gnupg/pubring.gpg' archweb_keyring: '/etc/pacman.d/gnupg/pubring.gpg'
archweb_reporead: false archweb_reporead: false
......
...@@ -204,37 +204,6 @@ ...@@ -204,37 +204,6 @@
- daemon reload - daemon reload
when: archweb_site|bool when: archweb_site|bool
- name: install donor_import service and timer
template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644
with_items:
- archweb-donor_import.service
- archweb-donor_import.timer
notify:
- daemon reload
when: archweb_donor_import|bool
- name: create maildir dump dir
file: path={{ archweb_donor_maildir }}/{{ item }} recurse=yes state=directory owner=archweb group=users mode=0755
with_items:
- cur
- new
when: archweb_donor_import|bool
- name: install dump donor mail service and timer
template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644
with_items:
- archweb-dump_donor_mail.service
- archweb-dump_donor_mail.timer
notify:
- daemon reload
when: archweb_donor_import|bool
- name: install archweb donor mail dump script
template: src="dump-mails.sh.j2" dest="/usr/local/bin/dump-mails.sh" owner=root group=root mode=0755
notify:
- daemon reload
when: archweb_donor_import|bool
- name: deploy archweb - name: deploy archweb
template: src=archweb.ini.j2 dest=/etc/uwsgi/vassals/archweb.ini owner=archweb group=http mode=0640 template: src=archweb.ini.j2 dest=/etc/uwsgi/vassals/archweb.ini owner=archweb group=http mode=0640
when: archweb_site|bool when: archweb_site|bool
...@@ -283,16 +252,11 @@ ...@@ -283,16 +252,11 @@
service: name="archweb-rebuilderd.timer" enabled=yes state=started service: name="archweb-rebuilderd.timer" enabled=yes state=started
when: archweb_site when: archweb_site
- name: start and enable archweb donor_import and archweb-dump_donor_mail timer - name: install donation import wrapper script
systemd: template: src=donor_import_wrapper.sh.j2 dest=/usr/local/bin/donor_import_wrapper.sh owner=root group=root mode=0755
name: "{{ item }}"
enabled: yes - name: install sudoer rights for fetchmail to call archweb django scripts
state: started template: src=sudoers-fetchmail-archweb.j2 dest=/etc/sudoers.d/fetchmail-archweb owner=root group=root mode=0440
daemon_reload: yes
with_items:
- archweb-donor_import.timer
- archweb-dump_donor_mail.timer
when: archweb_donor_import|bool
- name: create retro dir - name: create retro dir
file: state=directory owner=archweb group=archweb mode=0755 path="{{ archweb_retro_dir }}" file: state=directory owner=archweb group=archweb mode=0755 path="{{ archweb_retro_dir }}"
......
[Unit]
Description=archweb donor_import service
After=network.target
[Service]
Type=oneshot
User=archweb
WorkingDirectory={{ archweb_dir }}
ExecStart={{ archweb_dir }}/env/bin/python manage.py donor_import {{ archweb_donor_maildir }}
Nice=5
[Install]
WantedBy=multi-user.target
[Unit]
Description=archweb donor_import timer
[Timer]
OnUnitActiveSec=1w
OnBootSec=15min
RandomizedDelaySec=1h
[Install]
WantedBy=timers.target
[Unit]
Description=dump donate mail service
After=network.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/dump-mails.sh
Nice=5
[Install]
WantedBy=multi-user.target
[Unit]
Description=dump donate mail timer
[Timer]
OnUnitActiveSec=1w
OnBootSec=15min
RandomizedDelaySec=1h
[Install]
WantedBy=timers.target
#!/bin/bash
sudo -u archweb /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import
#!/bin/bash
set -e
shopt -s nullglob
dump_mails() {
target_dir="$1"; shift
date=$(date +%s)
searchquery=(all)
for user in "$@"; do
mailcount=$(doveadm search -u "$user" "${searchquery[@]}" | wc -l)
if ((mailcount > 0)); then
echo "dumping $mailcount mails from $user ..." >&2
doveadm search -u "$user" "${searchquery[@]}" | while read guid uid; do
doveadm fetch -u "$user" text mailbox-guid "$guid" uid "$uid" | sed '/^text:$/d; s#^\f$##' > "$target_dir/$user-$date-$uid.msg"
#doveadm expunge -u "$user" mailbox-guid "$guid" uid "$uid"
done
fi
done
}
dump_mails "{{ archweb_donor_maildir }}/cur" donate
fetchmail ALL=(archweb) NOPASSWD: /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import
...@@ -8,6 +8,6 @@ poll mail.archlinux.org ...@@ -8,6 +8,6 @@ poll mail.archlinux.org
bad-header accept bad-header accept
proto imap proto imap
user {{ fetchmail_user }} user {{ fetchmail_user }}
password {{ postfix_relay_password }} password {{ fetchmail_password }}
options idle sslcertck ssl sslproto "TLS1.2+" limitflush limit 25000000 fetchall options idle sslcertck ssl sslproto "TLS1.2+" limitflush limit 25000000 fetchall
mda "{{ fetchmail_delivery_cmd }}" mda "{{ fetchmail_delivery_cmd }}"
fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON=/srv/http/patchwork/env/bin/python /srv/http/patchwork/patchwork/bin/parsemail.sh
...@@ -101,7 +101,7 @@ ...@@ -101,7 +101,7 @@
template: src="patchwork-parsemail-wrapper.sh.j2" dest="/usr/local/bin/patchwork-parsemail-wrapper.sh" owner=root group=root mode=0755 template: src="patchwork-parsemail-wrapper.sh.j2" dest="/usr/local/bin/patchwork-parsemail-wrapper.sh" owner=root group=root mode=0755
- name: install sudoer rights for fetchmail to call patchwork - name: install sudoer rights for fetchmail to call patchwork
copy: src=fetchmail-patchwork dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440 template: src=sudoers-fetchmail-patchwork.j2 dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440
- name: install patchwork memcached service - name: install patchwork memcached service
template: src="patchwork-memcached.service.j2" dest="/etc/systemd/system/patchwork-memcached.service" owner=root group=root mode=0644 template: src="patchwork-memcached.service.j2" dest="/etc/systemd/system/patchwork-memcached.service" owner=root group=root mode=0644
......
fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON={{ patchwork_dir }}/env/bin/python {{ patchwork_dir }}/patchwork/bin/parsemail.sh
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment