Merge branch 'fix-donate' into 'master'

use fetchmail for donor import

See merge request !248

host_vars/archlinux.org → host_vars/archlinux.org/misc

 ---
 filesystem: btrfs
+fetchmail_user: "donate@archlinux.org"
+fetchmail_delivery_cmd: "/usr/local/bin/donor_import_wrapper.sh"
 
 ipv4_address: "95.217.163.246"
 ipv6_address: "2a01:4f9:c010:6b1f::1"

host_vars/archlinux.org/vault_archlinux.yml

+$ANSIBLE_VAULT;1.1;AES256
+30343863353037356134636339633536323666663861393166653963303537303365656531643830
+3862336361376565346630313638393632333931333939350a653338623261633739396464333236
+30613562663166653039616433353239313164653233373531343734643934643962646662366264
+3761663633333036310a363166343037616266613235343937343465646332666235333864396333
+30623733346138393864666436376236663263393661623762366237626331356630373762666138
+61386262623533326265613966356636666665643430386561343563306431623137313932643334
+303930343963306437396165663235353739

host_vars/patchwork.archlinux.org/vault_patchwork.yml

 $ANSIBLE_VAULT;1.1;AES256
-38343839616266323863666465663431343935333663613637656139646631336431376131663764
-3739393932303339353936313034313831633335316332620a383364313566333862376636373663
-31343439613834333338333633326232386263616665383561663936316665363935366165346165
-6433383735386261300a633862333661303639353734306333366233333237313162623562386130
-39383062333138343235613135663036386666636537626661336132643139323239323938643932
-35343034346534356437656638333862386230323063376661333939633532306539333035316537
-613739623366326235356137623633386137
+39323465346236343761323262393437336164613164653665626235386561383335396533373238
+6432656264613366353930343763386534646131656361370a366335356139383534373233333665
+37393032356238313932366663383237663162326164393637303866333862353032303130353362
+3933613433313964360a373739303336653333386332323638633335383337313334623635326266
+62363863643166396530666366313837346365306230363263643335393334303034636332356538
+35306532643439623235313933633562323264363431643232346662383639306438333238313766
+613838336337313665643837353135383063

playbooks/archlinux.org.yml

@@ -34,6 +34,7 @@
     - { role: sudo }
     - { role: uwsgi }
     - { role: memcached }
+    - { role: fetchmail }
     - { role: archweb, archweb_planet: true }
     - { role: fail2ban }
     - { role: prometheus_exporters }

roles/archweb/defaults/main.yml

@@ -20,9 +20,7 @@ archweb_site: true
 archweb_mirrorcheck: false
 archweb_mirrorresolv: false
 archweb_pgp_import: false
-archweb_donor_import: false
 archweb_planet: false
-archweb_donor_maildir: '/var/lib/archweb/donate'
 archweb_populate_signoffs: false
 archweb_keyring: '/etc/pacman.d/gnupg/pubring.gpg'
 archweb_reporead: false

roles/archweb/tasks/main.yml

@@ -204,37 +204,6 @@
     - daemon reload
   when: archweb_site|bool
 
-- name: install donor_import service and timer
-  template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644
-  with_items:
-    - archweb-donor_import.service
-    - archweb-donor_import.timer
-  notify:
-    - daemon reload
-  when: archweb_donor_import|bool
-
-- name: create maildir dump dir
-  file: path={{ archweb_donor_maildir }}/{{ item }} recurse=yes state=directory owner=archweb group=users mode=0755
-  with_items:
-    - cur
-    - new
-  when: archweb_donor_import|bool
-
-- name: install dump donor mail service and timer
-  template: src="{{ item }}.j2" dest="/etc/systemd/system/{{ item }}" owner=root group=root mode=0644
-  with_items:
-    - archweb-dump_donor_mail.service
-    - archweb-dump_donor_mail.timer
-  notify:
-    - daemon reload
-  when: archweb_donor_import|bool
-
-- name: install archweb donor mail dump script
-  template: src="dump-mails.sh.j2" dest="/usr/local/bin/dump-mails.sh" owner=root group=root mode=0755
-  notify:
-    - daemon reload
-  when: archweb_donor_import|bool
-
 - name: deploy archweb
   template: src=archweb.ini.j2 dest=/etc/uwsgi/vassals/archweb.ini owner=archweb group=http mode=0640
   when: archweb_site|bool
@@ -283,16 +252,11 @@
   service: name="archweb-rebuilderd.timer" enabled=yes state=started
   when: archweb_site
 
-- name: start and enable archweb donor_import and archweb-dump_donor_mail timer
-  systemd:
-    name: "{{ item }}"
-    enabled: yes
-    state: started
-    daemon_reload: yes
-  with_items:
-    - archweb-donor_import.timer
-    - archweb-dump_donor_mail.timer
-  when: archweb_donor_import|bool
+- name: install donation import wrapper script
+  template: src=donor_import_wrapper.sh.j2 dest=/usr/local/bin/donor_import_wrapper.sh owner=root group=root mode=0755
+
+- name: install sudoer rights for fetchmail to call archweb django scripts
+  template: src=sudoers-fetchmail-archweb.j2 dest=/etc/sudoers.d/fetchmail-archweb owner=root group=root mode=0440
 
 - name: create retro dir
   file: state=directory owner=archweb group=archweb mode=0755 path="{{ archweb_retro_dir }}"

roles/archweb/templates/archweb-donor_import.service.j2

-[Unit]
-Description=archweb donor_import service
-After=network.target
-
-[Service]
-Type=oneshot
-User=archweb
-WorkingDirectory={{ archweb_dir }}
-ExecStart={{ archweb_dir }}/env/bin/python manage.py donor_import {{ archweb_donor_maildir }}
-Nice=5
-
-[Install]
-WantedBy=multi-user.target

roles/archweb/templates/archweb-donor_import.timer.j2

-[Unit]
-Description=archweb donor_import timer
-
-[Timer]
-OnUnitActiveSec=1w
-OnBootSec=15min
-RandomizedDelaySec=1h
-
-[Install]
-WantedBy=timers.target

roles/archweb/templates/archweb-dump_donor_mail.service.j2

-[Unit]
-Description=dump donate mail service
-After=network.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/dump-mails.sh
-Nice=5
-
-[Install]
-WantedBy=multi-user.target

roles/archweb/templates/archweb-dump_donor_mail.timer.j2

-[Unit]
-Description=dump donate mail timer
-
-[Timer]
-OnUnitActiveSec=1w
-OnBootSec=15min
-RandomizedDelaySec=1h
-
-[Install]
-WantedBy=timers.target

roles/archweb/templates/donor_import_wrapper.sh.j2

+#!/bin/bash
+
+sudo -u archweb /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import

roles/archweb/templates/dump-mails.sh.j2

-#!/bin/bash
-
-set -e
-shopt -s nullglob
-
-dump_mails() {
-        target_dir="$1"; shift
-        date=$(date +%s)
-        searchquery=(all)
-
-        for user in "$@"; do
-                mailcount=$(doveadm search -u "$user" "${searchquery[@]}" | wc -l)
-                if ((mailcount > 0)); then
-                        echo "dumping $mailcount mails from $user ..." >&2
-                        doveadm search -u "$user" "${searchquery[@]}" | while read guid uid; do
-                                doveadm fetch -u "$user" text mailbox-guid "$guid" uid "$uid" | sed '/^text:$/d; s#^\f$##' > "$target_dir/$user-$date-$uid.msg"
-                                #doveadm expunge -u "$user" mailbox-guid "$guid" uid "$uid"
-                        done
-                fi
-        done
-}
-
-
-dump_mails "{{ archweb_donor_maildir }}/cur" donate

roles/archweb/templates/sudoers-fetchmail-archweb.j2

+fetchmail ALL=(archweb) NOPASSWD: /usr/bin/env PW_PYTHON={{ archweb_dir }}/env/bin/python {{ archweb_dir }}/manage.py donor_import

roles/fetchmail/templates/fetchmailrc.j2

@@ -8,6 +8,6 @@ poll mail.archlinux.org
 bad-header accept
 proto imap
 user {{ fetchmail_user }}
-password {{ postfix_relay_password }}
+password {{ fetchmail_password }}
 options idle sslcertck ssl sslproto "TLS1.2+" limitflush limit 25000000 fetchall
 mda "{{ fetchmail_delivery_cmd }}"

roles/patchwork/files/fetchmail-patchwork

-fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON=/srv/http/patchwork/env/bin/python /srv/http/patchwork/patchwork/bin/parsemail.sh

roles/patchwork/tasks/main.yml

@@ -101,7 +101,7 @@
   template: src="patchwork-parsemail-wrapper.sh.j2" dest="/usr/local/bin/patchwork-parsemail-wrapper.sh" owner=root group=root mode=0755
 
 - name: install sudoer rights for fetchmail to call patchwork
-  copy: src=fetchmail-patchwork dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440
+  template: src=sudoers-fetchmail-patchwork.j2 dest=/etc/sudoers.d/fetchmail-patchwork owner=root group=root mode=0440
 
 - name: install patchwork memcached service
   template: src="patchwork-memcached.service.j2" dest="/etc/systemd/system/patchwork-memcached.service" owner=root group=root mode=0644

roles/patchwork/templates/sudoers-fetchmail-patchwork.j2

+fetchmail ALL=(patchwork) NOPASSWD: /usr/bin/env PW_PYTHON={{ patchwork_dir }}/env/bin/python {{ patchwork_dir }}/patchwork/bin/parsemail.sh