Add donor import systemd services/timer

playbooks/orion.yml

@@ -22,6 +22,6 @@
     - { role: nginx, tags: ["nginx"] }
     - { role: dbscripts, repos_domain: "repos.archlinux.org", repos_rsync_domain: "rsync.archlinux.org", svntogit_repos: "/srv/svntogit/repos", postgres_ssl: 'on', tags: ['dbscripts', 'archusers'] }
     - { role: sudo, tags: ['sudo', 'archusers'] }
-    - { role: archweb, archweb_site: false, archweb_services: true, archweb_donor_import: false, archweb_mirrorcheck_locations: [5, 6], tags: ['archweb'] }
+    - { role: archweb, archweb_site: false, archweb_services: true, archweb_donor_import: true, archweb_mirrorcheck_locations: [5, 6], tags: ['archweb'] }
     - { role: sources, sources_domain: "sources.archlinux.org", sources_dir: "/srv/sources", tags: ['sources'] }
     - { role: archive, archive_domain: "archive.archlinux.org", archive_dir: "/srv/archive", tags: ['archive'] }

roles/archweb/tasks/main.yml

@@ -197,6 +197,31 @@
     - daemon reload
   when: archweb_donor_import
 
+- name: create maildir dump dir
+  file: path={{ archweb_donor_maildir }}/{{ item }} recurse=yes state=directory
+  with_items:
+    - cur
+    - new
+  when: archweb_donor_import
+
+- name: install dump donor mail service
+  template: src="archweb-dump_donor_mail.service.j2" dest="/etc/systemd/system/archweb-dump_donor_mail.service" owner=root group=root mode=0644
+  notify:
+    - daemon reload
+  when: archweb_donor_import
+
+- name: install dump donor mail timer
+  template: src="archweb-dump_donor_mail.timer.j2" dest="/etc/systemd/system/archweb-dump_donor_mail.timer" owner=root group=root mode=0644
+  notify:
+    - daemon reload
+  when: archweb_donor_import
+
+- name: install archweb donor mail dump script
+  template: src="dump-mails.sh.j2" dest="/usr/local/bin/dump-mails.sh" owner=root group=root mode=0755
+  notify:
+    - daemon reload
+  when: archweb_donor_import
+
 - name: deploy archweb
   template: src=archweb.ini.j2 dest=/etc/uwsgi/vassals/archweb.ini owner=archweb group=http mode=0644
   when: archweb_site
@@ -235,3 +260,11 @@
 - name: start and enable archweb populate_signoffs timer
   service: name="archweb-populate_signoffs.timer" enabled=yes state=started
   when: archweb_services or archweb_populate_signoffs
+
+- name: start and enable archweb donor_import timer
+  service: name="archweb-donor_import.timer" enabled=yes state=started
+  when: archweb_donor_import
+
+- name: start and enable archweb dump donor timer
+  service: name="archweb-dump_donor_mail.timer" enabled=yes state=started
+  when: archweb_donor_import

roles/archweb/templates/archweb-donor_import.service.j2

+[Unit]
+Description=archweb donor_import service
+After=network.target
+
+[Service]
+Type=oneshot
+User=archweb
+WorkingDirectory={{ archweb_dir }}
+ExecStart={{ archweb_dir }}/env/bin/python manage.py donor_import {{ archweb_donor_maildir }}
+Nice=5
+
+[Install]
+WantedBy=multi-user.target

roles/archweb/templates/archweb-donor_import.timer.j2

+[Unit]
+Description=archweb donor_import timer
+
+[Timer]
+OnUnitActiveSec=1w
+OnBootSec=15min
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target

roles/archweb/templates/archweb-dump_donor_mail.service.j2

+[Unit]
+Description=dump donate mail service
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/dump-mails.sh
+Nice=5
+
+[Install]
+WantedBy=multi-user.target

roles/archweb/templates/archweb-dump_donor_mail.timer.j2

+[Unit]
+Description=dump donate mail timer
+
+[Timer]
+OnUnitActiveSec=1w
+OnBootSec=15min
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target

roles/archweb/templates/dump-mails.sh.j2

+#!/bin/bash
+
+set -e
+shopt -s nullglob
+
+dump_mails() {
+        target_dir="$1"; shift
+        date=$(date +%s)
+        searchquery=(all)
+
+        for user in "$@"; do
+                mailcount=$(doveadm search -u "$user" "${searchquery[@]}" | wc -l)
+                if ((mailcount > 0)); then
+                        echo "dumping $mailcount mails from $user ..." >&2
+                        doveadm search -u "$user" "${searchquery[@]}" | while read guid uid; do
+                                doveadm fetch -u "$user" text mailbox-guid "$guid" uid "$uid" | sed '/^text:$/d; s#^\f$##' > "$target_dir/$user-$date-$uid.msg"
+                                #doveadm expunge -u "$user" mailbox-guid "$guid" uid "$uid"
+                        done
+                fi
+        done
+}
+
+
+dump_mails "{{ archweb_donor_maildir }}/cur" donate