Setup mailman3 server

We want to migrate to mailman3 as mailman2 is basically unmaintained and
requires Python 2 which is EOL.

Because the mailman and mailman3 packages conflict and we don't want to
perform a big bang migration, mailman3 must be deployed on a separate
server. mailman-web (mailman3's web interface) hasn't been packaged yet,
so for now we are using my homebrewed PKGBUILD[1].

[1] https://gist.github.com/klausenbusk/5982063f95c503754a51ed2fefb8915e

Ref #59

docs/servers.md

@@ -151,6 +151,14 @@ Prometheus, and Grafana server which receives selected performance/metrics from
 
   Online collborative markdwown editor for Arch Linux Staff.
 
+## mailman3.archlinux.org
+
+This server runs mailman3 as mailman2 and mailman3 can't be installed on the same server. The HTTP and LMTP traffic is routed over WireGuard from lists.archlinux.org.
+
+### Services
+
+  - mailman3
+
 ### Services
   - [hedgedoc](https://hedgedoc.org/)
 

group_vars/all/vault_mailman3.yml

+$ANSIBLE_VAULT;1.1;AES256
+63633533303232373335663630346139613137616132393738383265663337636565663935386365
+3262636536383962333438653033323061306433323232610a623836643732616163383364316639
+37626134643334383432346465343734353566663261643334396563336132666133666431313563
+6365643566626635360a616139393131346566666266653737303562663664656231643836373038
+37316436643133333261313963356435353938393032313935353939613962303733623934313965
+64356635626561376130336134656436386638306538373635313638393932313337316636343533
+32666138613765326332373335366634313530656162383162633861666365333230303132346263
+63613031643230356361383638386230613231626135663763373630666362623536663165356335
+33333033376332653130626262633563336238383931393636346339333963326330373431363931
+61383733626363316539653638373562616335366363306365353166666335383037633830636263
+37313663636139666131623435383833313434396665663162623934646330626362346237363331
+65323537383536333763646431623061646337613761363861373261343638653235333038663239
+34636662663763363832643061313035316437633965346332363432653562613865623261613235
+61303239626136303736356533373739343566313464343931383962633232313263383230336438
+32653534623739616436346539616336373562376632303833323230643465666262303263383334
+64623362363863393866666461396237613934656239653262316438633338313036303436313236
+61623562376139616539646231376438636234656363666639646465663035326161346435396439
+63613839396163616135313537626535393039623866646431333239383263313931386131303464
+36353837303662343530663561363036633864346131343731643535386462316663353233636638
+36323134643230376239326637656537633337323333616630313531653239366263386238363333
+32336538613635613964366562383165616433363738623638393364363233636262643131653532
+62326363356333333563383139323366363462613031303566376365643439373163613166333339
+38353266616463396139336663353536336631666565656630396431363439333034653336316234
+61663232383136353937336431353131323933613462666233663464656166356161613039316436
+3136

host_vars/mailman3.archlinux.org/misc

+---
+filesystem: btrfs
+ipv4_address: 65.21.106.94
+wireguard_address: 10.0.0.37
+wireguard_public_key: obBFreFGNDLB17+PaJspE4qNeVX4o7ZPcJj3ZmJhahg=

host_vars/mailman3.archlinux.org/vault_wireguard.yml

+$ANSIBLE_VAULT;1.1;AES256
+32363065633737653663623334663139323638366462343630623765396636353932653932356261
+6239356162633731656330383436363861376231616462390a356432316532333632653839333230
+63636434373462643231323532633362363434646230323636333264393032373632343932616361
+6536383038313134300a363139313337646533626334333666326535623039323332666338306532
+33643430313864663833343765623138393165386564343636306363626232666436353665353235
+34623064363764336139633334663530376332633536383033313438613035303662333435313536
+34366663643130633064646161613065373532653235373730316439643165383635353761396639
+61656462333035666437

hosts

@@ -45,6 +45,7 @@ security.archlinux.org
 md.archlinux.org
 lists.archlinux.org
 gluebuddy.archlinux.org
+mailman3.archlinux.org
 
 [public_html]
 homedir.archlinux.org
@@ -127,6 +128,7 @@ gluebuddy.archlinux.org
 homedir.archlinux.org
 lists.archlinux.org
 mail.archlinux.org
+mailman3.archlinux.org
 man.archlinux.org
 matrix.archlinux.org
 md.archlinux.org

playbooks/mailman3.archlinux.org.yml

+- name: setup mailman3 server
+  hosts: mailman3.archlinux.org
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: firewalld }
+    - { role: wireguard }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: hardening }
+    - { role: borg_client, tags: ["borg"] }
+    - { role: prometheus_exporters }
+    - { role: promtail }
+    - { role: nginx, nginx_firewall_zone: wireguard }
+    - { role: uwsgi }
+    - { role: postgres }
+    - { role: mailman3 }

roles/mailman/handlers/main.yml

@@ -7,3 +7,9 @@
 
 - name: reload postfix
   service: name=postfix state=reloaded
+
+- name: run postmap
+  command: postmap /etc/postfix/{{ item }}
+  loop:
+    - aliases
+    - transport

roles/mailman/tasks/main.yml

@@ -21,10 +21,19 @@
   loop:
     - aliases
     - milter_header_checks
-  notify: reload postfix
+  notify: run postmap
+
+- name: install postfix templated maps
+  template: src={{ item }}.j2 dest=/etc/postfix/{{ item }} owner=root group=root mode=0644
+  loop:
+    - transport
+  notify: run postmap
 
 - name: open firewall holes for postfix
-  ansible.posix.firewalld: service=smtp permanent=true state=enabled immediate=yes
+  ansible.posix.firewalld: service=smtp zone={{ item }} permanent=true state=enabled immediate=yes
+  loop:
+    -
+    - wireguard
   when: configure_firewall
   tags:
     - firewall

roles/mailman/templates/main.cf.j2

@@ -18,6 +18,11 @@ smtp_tls_security_level = may
 mydomain = {{ lists_domain }}
 myorigin = {{ lists_domain }}
 mydestination = {{ lists_domain }}
+mynetworks =
+    127.0.0.0/8
+    [::1]/128
+    [fe80::]/64
+    {{ hostvars['mailman3.archlinux.org']['wireguard_address'] }}
 
 # fatal: configuration error: mailbox_size_limit is smaller than message_size_limit
 message_size_limit = 104857600
@@ -41,9 +46,10 @@ smtpd_reject_footer = For assistance contact <postmaster@archlinux.org>. Please
 smtpd_milters = inet:localhost:11332
 non_smtpd_milters = $smtpd_milters
 
-alias_maps = texthash:/etc/postfix/aliases hash:/var/lib/mailman/data/aliases
-local_recipient_maps = $alias_maps
+alias_maps = hash:/etc/postfix/aliases hash:/var/lib/mailman/data/aliases
+local_recipient_maps = hash:/etc/postfix/transport $alias_maps
 alias_database = $alias_maps
+transport_maps = hash:/etc/postfix/transport
 
 milter_header_checks = pcre:/etc/postfix/milter_header_checks
 

roles/mailman/templates/nginx.d.conf.j2

@@ -51,4 +51,10 @@ server {
         uwsgi_pass      unix:/run/uwsgi/mailman.sock;
     }
 
+    location ~ ^/(static|mailman3|archives|user-profile|accounts|admin3)($|/) {
+        proxy_pass http://{{ hostvars['mailman3.archlinux.org']['wireguard_address'] }};
+        proxy_set_header Host {{ lists_domain }};
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
 }

roles/mailman/templates/transport.j2

+# AUTOMATICALLY GENERATED BY MAILMAN ON 2021-07-03 22:52:50
+#
+# This file is generated by Mailman, and is kept in sync with the binary hash
+# file.  YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're
+# doing, and can keep the two files properly in sync.  If you screw it up,
+# you're on your own.
+    
+# Aliases which are visible only in the @lists.archlinux.org domain.

roles/mailman3/defaults/main.yml

+lists_domain: lists.archlinux.org

roles/mailman3/files/postfix.cfg

+[postfix]
+# Additional configuration variables for the postfix MTA.
+
+# This variable describe the program to use for regenerating the transport map
+# db file, from the associated plain text files.  The file being updated will
+# be appended to this string (with a separating space), so it must be
+# appropriate for os.system().
+postmap_command: /usr/bin/true
+
+# This variable describes the type of transport maps that will be generated by
+# mailman to be used with postfix for LMTP transport. By default, it is set to
+# hash, but mailman also supports `regex` tables.
+transport_file_type: hash

roles/mailman3/handlers/main.yml

+---
+- name: reload mailman
+  service: name=mailman3 state=reloaded
+
+- name: restart mailman-web
+  service: name=uwsgi@mailman\\x2dweb.service state=restarted

roles/mailman3/tasks/main.yml

+---
+- name: install mailman3 and related packages
+  pacman: name=mailman3,mailman3-hyperkitty,python-psycopg2,mailman-web,uwsgi-plugin-python state=present
+  register: install
+
+- name: install {mailman,mailman-web} configuration
+  template: src={{ item.src }} dest={{ item.dest }} owner=root group={{ item.group }} mode=0640
+  loop:
+    - {src: mailman.cfg.j2, dest: /etc/mailman.cfg, group: mailman}
+    - {src: mailman-hyperkitty.cfg.j2, dest: /etc/mailman-hyperkitty.cfg, group: mailman}
+    - {src: settings.py.j2, dest: /etc/mailman3/settings.py, group: mailman-web}
+    - {src: urls.py.j2, dest: /etc/mailman3/urls.py, group: mailman-web}
+  notify:
+    - reload mailman
+    - restart mailman-web
+
+- name: install mailman postfix.cfg configuration
+  copy: src=postfix.cfg dest=/etc/postfix.cfg owner=root group=root mode=0644
+  notify: reload mailman
+
+- name: make nginx log dir
+  file: path=/var/log/nginx/{{ lists_domain }} state=directory owner=root group=root mode=0755
+
+- name: set up nginx
+  template: src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/mailman.conf" owner=root group=root mode=644
+  notify: reload nginx
+  tags: ['nginx']
+
+- name: create postgres {mailman,mailman-web} user
+  postgresql_user: name={{ item.username }} password={{ item.password }}
+  loop:
+    - {username: "{{ vault_mailman_db_user }}", password: "{{ vault_mailman_db_password }}"}
+    - {username: "{{ vault_mailman_web_db_user }}", password: "{{ vault_mailman_web_db_password }}"}
+  become: true
+  become_user: postgres
+  become_method: su
+  no_log: true
+
+- name: create {mailman,mailman-web} db
+  postgresql_db: name={{ item.db }} owner={{ item.owner }}
+  loop:
+    - {db: mailman, owner: "{{ vault_mailman_db_user }}"}
+    - {db: mailman-web, owner: "{{ vault_mailman_web_db_user }}"}
+  become: true
+  become_user: postgres
+  become_method: su
+
+- name: run Django management tasks
+  command: django-admin {{ item }} --pythonpath /etc/mailman3 --settings settings
+  loop:
+    - migrate
+    - loaddata
+    - collectstatic
+    - compress
+  become: true
+  become_user: mailman-web
+  when: install.changed
+
+- name: open LMTP ipv4 port for lists.archlinux.org
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['lists.archlinux.org']['wireguard_address'] }} port protocol=tcp port=8024 accept"
+  tags:
+    - firewall
+
+- name: start and enable mailman{.service,-*.timer}
+  systemd: name={{ item }} enabled=yes daemon_reload=yes state=started
+  loop:
+    - mailman3.service
+    - mailman3-digests.timer
+    - mailman3-gatenews.timer
+    - mailman3-notify.timer
+    - uwsgi@mailman\x2dweb.service

roles/mailman3/templates/mailman.cfg.j2

+[mailman]
+site_owner: root@{{ lists_domain }}
+layout: fhs
+
+[database]
+class: mailman.database.postgresql.PostgreSQLDatabase
+url: postgres://{{ vault_mailman_db_user }}:{{ vault_mailman_db_password }}@/mailman
+
+[webservice]
+admin_user: {{ vault_mailman_admin_user }}
+admin_pass: {{ vault_mailman_admin_pass }}
+
+[mta]
+configuration: /etc/postfix.cfg
+lmtp_host: {{ hostvars['mailman3.archlinux.org']['wireguard_address'] }}
+lmtp_port: 8024
+smtp_host: {{ hostvars['lists.archlinux.org']['wireguard_address'] }}
+smtp_port: 25
+
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
+enable: yes
+configuration: /etc/mailman-hyperkitty.cfg

roles/mailman3/templates/nginx.d.conf.j2

+server {
+    listen       80;
+    listen       [::]:80;
+    server_name  {{ lists_domain }} localhost;
+
+    set_real_ip_from {{ hostvars['lists.archlinux.org']['wireguard_address'] }}/32;
+    real_ip_header   X-Forwarded-For;
+
+    access_log   /var/log/nginx/{{ lists_domain }}/access.log main;
+    access_log   /var/log/nginx/{{ lists_domain }}/access.log.json json_main;
+    error_log    /var/log/nginx/{{ lists_domain }}/error.log;
+
+    location /static {
+      alias /var/lib/mailman-web/static;
+    }
+
+    # include uwsgi_params
+    location / {
+      include /etc/nginx/uwsgi_params;
+      uwsgi_pass unix:/run/mailman-web/mailman-web.sock;
+    }
+}

roles/mailman3/templates/settings.py.j2

+# mailman-web config
+
+from mailman_web.settings.base import *
+from mailman_web.settings.mailman import *
+
+
+#: Default list of admins who receive the emails from error logging.
+ADMINS = (
+    ('Mailman Suite Admin', 'root@{{ lists_domain }}'),
+)
+
+# Postgresql datbase setup.
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'mailman-web',
+        'USER': '{{ vault_mailman_web_db_user }}',
+        'PASSWORD': '{{ vault_mailman_web_db_password }}',
+    }
+}
+
+#: See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [
+    "localhost",  # Archiving API from Mailman, keep it.
+    "{{ lists_domain }}",
+]
+
+#: Current Django Site being served. This is used to customize the web host
+#: being used to serve the current website. For more details about Django
+#: site, see: https://docs.djangoproject.com/en/dev/ref/contrib/sites/
+SITE_ID = 1
+
+SECRET_KEY = '{{ vault_mailman_web_secret_key }}'
+
+MAILMAN_REST_API_USER = '{{ vault_mailman_admin_user }}'
+MAILMAN_REST_API_PASS = '{{ vault_mailman_admin_pass }}'
+MAILMAN_ARCHIVER_KEY = '{{ vault_mailman_archiver_key }}'
+
+#: https://docs.djangoproject.com/en/3.2/topics/email/#smtp-backend
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST = '{{ hostvars['lists.archlinux.org']['wireguard_address'] }}'
+EMAIL_PORT = 25
+
+#: Sender in Emails sent out by Postorius.
+DEFAULT_FROM_EMAIL = 'postorius@{{ lists_domain }}'
+SERVER_EMAIL = 'root@{{ lists_domain }}'
+
+POSTORIUS_TEMPLATE_BASE_URL = 'http://localhost'
+HYPERKITTY_ALLOW_WEB_POSTING = False
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': '/var/lib/mailman-web/fulltext_index'
+    }
+}

roles/mailman3/templates/urls.py.j2

+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Postorius.
+#
+# Postorius is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Postorius is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Postorius.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from django.conf.urls import include, url
+from django.contrib import admin
+from django.urls import reverse_lazy
+from django.views.generic import RedirectView
+
+urlpatterns = [
+    url(r'^$', RedirectView.as_view(
+        url=reverse_lazy('list_index'),
+        permanent=True)),
+    url(r'^mailman3/', include('postorius.urls')),
+    url(r'^archives/', include('hyperkitty.urls')),
+    url(r'', include('django_mailman3.urls')),
+    url(r'^accounts/', include('allauth.urls')),
+    # Django admin
+    url(r'^admin3/', admin.site.urls),
+]

roles/nginx/defaults/main.yml

 ---
 letsencrypt_validation_dir: "/var/lib/letsencrypt"
+nginx_firewall_zone: