Setup mailman3 server

We want to migrate to mailman3 as mailman2 is basically unmaintained and
requires Python 2 which is EOL.

Because the mailman and mailman3 packages conflict and we don't want to
perform a big bang migration, mailman3 must be deployed on a separate
server. mailman-web (mailman3's web interface) hasn't been packaged yet,
so for now we are using my homebrewed PKGBUILD.

[1] https://gist.github.com/klausenbusk/5982063f95c503754a51ed2fefb8915e

Ref #59

docs/servers.md

@@ -151,6 +151,14 @@ Prometheus, and Grafana server which receives selected performance/metrics from
 
   Online collborative markdwown editor for Arch Linux Staff.
 
+## mailman3.archlinux.org
+
+This server runs mailman3 as mailman2 and mailman3 can't be installed on the same server. The HTTP and LMTP traffic is routed over WireGuard from lists.archlinux.org.
+
+### Services
+
+  - mailman3
+
 ### Services
   - [hedgedoc](https://hedgedoc.org/)
 

group_vars/all/vault_mailman3.yml

+$ANSIBLE_VAULT;1.1;AES256
+34383835323639306337626539636433613539636466653630383234333063623930666137646438
+6631373661313934656531363265363061313132633031650a326434306636396139663436623133
+38303738636465343430396461353533336165353033356430613036623136353137316338363364
+6563386438653733610a636566326462346364323362666639373430626663353935393763343638
+34393436616131376639653238326266336161393130386232373836303831646133303133313639
+65666136303433636632663261363930626161623938353033623761663264643831353063616432
+61373963636164663832666532313833656133363434323831323364643561616134616536646538
+30363632366661333936663139666166303737316166383738653038343034396265313466643532
+64646532393239613631653336633361346664343561643734316336363632383935653534323263
+38616463366235366636633839663464393562316138343534666430643266663439363838353239
+65643030666564343661343466646333313566333561646662623731656666353166626566333637
+31656432613439366661343834636239356139663263303965653430336662396431643966303739
+37316166663132363062323533646131623630666332653565303630613039666231323130386139
+34393639366537343463393133643832653133333065363839663333366333626132306636663136
+36316464623664303762316461386235643363353262323535383065346563613261623966313530
+37303466623834333537346265323766633934366332396436643962306637643262343463383963
+39393332633465333864346437383533633465343632323061623833393739646361383766656131
+31393066633364646637333765303266363438373330326334323666653262396137363036373734
+38663339326435383730366532363264386334376533653030343430633836376261346237646638
+37663864633832623237623335313938616562356366303638386531323035613534323130333439
+31613931636238373864303166623636643735383837343464323663633263313963626632633031
+64626535333231623461656661323264636333613430366637333631306663343438353763393537
+37663034336338376538333632623939643165613836656334323638653133333362393632653131
+38663937633065313531376134333238346163636162386665306332363635653934363761396339
+3366

host_vars/mailman3.archlinux.org/misc

+---
+filesystem: btrfs
+ipv4_address: 65.21.106.94
+wireguard_address: 10.0.0.37
+wireguard_public_key: obBFreFGNDLB17+PaJspE4qNeVX4o7ZPcJj3ZmJhahg=

host_vars/mailman3.archlinux.org/vault_wireguard.yml

+$ANSIBLE_VAULT;1.1;AES256
+66656238613063383133353765323966656638396230336461366564306566323232356262336362
+3565373965363634623939626139336638383034373338390a316566383963643935663536326436
+61366538663931313037386363666234633031376663303330346564346439323139666566346238
+6236336638373364350a316463336433386363363430626437326463323935316463666166353561
+63623363353763633936326131613639336637646363326137653638303061666636343237353164
+30666537613964643234333165353066636363343666316664323631303335623861303233633837
+62336665653432393937623839323334336261376564343130386665333335386131366633373130
+62313639626130383636

hosts

@@ -45,6 +45,7 @@ security.archlinux.org
 md.archlinux.org
 lists.archlinux.org
 gluebuddy.archlinux.org
+mailman3.archlinux.org
 
 [public_html]
 homedir.archlinux.org
@@ -127,6 +128,7 @@ gluebuddy.archlinux.org
 homedir.archlinux.org
 lists.archlinux.org
 mail.archlinux.org
+mailman3.archlinux.org
 man.archlinux.org
 matrix.archlinux.org
 md.archlinux.org

playbooks/mailman3.archlinux.org.yml

+- name: setup mailman3 server
+  hosts: mailman3.archlinux.org
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: firewalld }
+    - { role: wireguard }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: hardening }
+    - { role: borg_client, tags: ["borg"] }
+    - { role: prometheus_exporters }
+    - { role: promtail }
+    - { role: nginx, nginx_firewall_zone: wireguard }
+    - { role: uwsgi }
+    - { role: postgres }
+    - { role: mailman3 }

roles/mailman/handlers/main.yml

@@ -7,3 +7,9 @@
 
 - name: reload postfix
   service: name=postfix state=reloaded
+
+- name: run postmap
+  command: postmap /etc/postfix/{{ item }}
+  loop:
+    - aliases
+    - transport

roles/mailman/tasks/main.yml

@@ -21,10 +21,19 @@
   loop:
     - aliases
     - milter_header_checks
-  notify: reload postfix
+  notify: run postmap
+
+- name: install postfix templated maps
+  template: src={{ item }}.j2 dest=/etc/postfix/{{ item }} owner=root group=root mode=0644
+  loop:
+    - transport
+  notify: run postmap
 
 - name: open firewall holes for postfix
-  ansible.posix.firewalld: service=smtp permanent=true state=enabled immediate=yes
+  ansible.posix.firewalld: service=smtp zone={{ item }} permanent=true state=enabled immediate=yes
+  loop:
+    -
+    - wireguard
   when: configure_firewall
   tags:
     - firewall

roles/mailman/templates/main.cf.j2

@@ -18,6 +18,11 @@ smtp_tls_security_level = may
 mydomain = {{ lists_domain }}
 myorigin = {{ lists_domain }}
 mydestination = {{ lists_domain }}
+mynetworks =
+    127.0.0.0/8
+    [::1]/128
+    [fe80::]/64
+    {{ hostvars['mailman3.archlinux.org']['wireguard_address'] }}
 
 # fatal: configuration error: mailbox_size_limit is smaller than message_size_limit
 message_size_limit = 104857600
@@ -41,9 +46,10 @@ smtpd_reject_footer = For assistance contact <postmaster@archlinux.org>. Please
 smtpd_milters = inet:localhost:11332
 non_smtpd_milters = $smtpd_milters
 
-alias_maps = texthash:/etc/postfix/aliases hash:/var/lib/mailman/data/aliases
-local_recipient_maps = $alias_maps
+alias_maps = hash:/etc/postfix/aliases hash:/var/lib/mailman/data/aliases
+local_recipient_maps = hash:/etc/postfix/transport $alias_maps
 alias_database = $alias_maps
+transport_maps = hash:/etc/postfix/transport
 
 milter_header_checks = pcre:/etc/postfix/milter_header_checks
 

roles/mailman/templates/nginx.d.conf.j2

@@ -51,4 +51,10 @@ server {
         uwsgi_pass      unix:/run/uwsgi/mailman.sock;
     }
 
+    location ~ ^/(static|mailman3|archives|user-profile|accounts|admin3)($|/) {
+        proxy_pass http://{{ hostvars['mailman3.archlinux.org']['wireguard_address'] }};
+        proxy_set_header Host {{ lists_domain }};
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
 }

roles/mailman/templates/transport.j2

+# AUTOMATICALLY GENERATED BY MAILMAN ON 2021-07-03 22:52:50
+#
+# This file is generated by Mailman, and is kept in sync with the binary hash
+# file.  YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're
+# doing, and can keep the two files properly in sync.  If you screw it up,
+# you're on your own.
+    
+# Aliases which are visible only in the @lists.archlinux.org domain.

roles/mailman3/defaults/main.yml

+lists_domain: lists.archlinux.org

roles/mailman3/files/postfix.cfg

+[postfix]
+# Additional configuration variables for the postfix MTA.
+
+# This variable describe the program to use for regenerating the transport map
+# db file, from the associated plain text files.  The file being updated will
+# be appended to this string (with a separating space), so it must be
+# appropriate for os.system().
+postmap_command: /usr/bin/true
+
+# This variable describes the type of transport maps that will be generated by
+# mailman to be used with postfix for LMTP transport. By default, it is set to
+# hash, but mailman also supports `regex` tables.
+transport_file_type: hash

roles/mailman3/handlers/main.yml

+---
+- name: reload mailman
+  service: name=mailman3 state=reloaded
+
+- name: restart mailman-web
+  service: name=uwsgi@mailman\\x2dweb.service state=restarted

roles/mailman3/tasks/main.yml

+---
+- name: install mailman3, mailman3-hyperkitty, python-psycopg2, mailman-web and uwsgi-plugin-python
+  pacman: name=mailman3,mailman3-hyperkitty,python-psycopg2,mailman-web,uwsgi-plugin-python state=present
+  register: install
+
+- name: install {mailman,mailman-web} configuration
+  template: src={{ item.src }} dest={{ item.dest }} owner=root group={{ item.group }} mode=0640
+  loop:
+    - {src: mailman.cfg.j2, dest: /etc/mailman.cfg, group: mailman}
+    - {src: mailman-hyperkitty.cfg.j2, dest: /etc/mailman-hyperkitty.cfg, group: mailman}
+    - {src: settings.py.j2, dest: /etc/mailman3/settings.py, group: mailman-web}
+    - {src: urls.py.j2, dest: /etc/mailman3/urls.py, group: mailman-web}
+  notify:
+    - reload mailman
+    - restart mailman-web
+
+- name: install mailman postfix.cfg configuration
+  copy: src=postfix.cfg dest=/etc/postfix.cfg owner=root group=root mode=0644
+  notify: reload mailman
+
+- name: make nginx log dir
+  file: path=/var/log/nginx/{{ lists_domain }} state=directory owner=root group=root mode=0755
+
+- name: set up nginx
+  template: src=nginx.d.conf.j2 dest="/etc/nginx/nginx.d/mailman.conf" owner=root group=root mode=644
+  notify: reload nginx
+  tags: ['nginx']
+
+- name: create postgres {mailman,mailman-web} user
+  postgresql_user: name={{ item.username }} password={{ item.password }}
+  loop:
+    - {username: "{{ vault_mailman_db_user }}", password: "{{ vault_mailman_db_password }}"}
+    - {username: "{{ vault_mailman_web_db_user }}", password: "{{ vault_mailman_web_db_password }}"}
+  become: true
+  become_user: postgres
+  become_method: su
+  no_log: true
+
+- name: create {mailman,mailman-web} db
+  postgresql_db: name={{ item.db }} owner={{ item.owner }}
+  loop:
+    - {db: mailman, owner: "{{ vault_mailman_db_user }}"}
+    - {db: mailman-web, owner: "{{ vault_mailman_web_db_user }}"}
+  become: true
+  become_user: postgres
+  become_method: su
+
+- name: run Django management tasks
+  command: django-admin {{ item }} --pythonpath /etc/mailman3 --settings settings
+  loop:
+    - migrate
+    - loaddata
+    - collectstatic
+    - compress
+  become: true
+  become_user: mailman-web
+  when: install.changed
+
+- name: open LMTP ipv4 port for lists.archlinux.org
+  ansible.posix.firewalld: zone=wireguard state=enabled permanent=true immediate=yes
+    rich_rule="rule family=ipv4 source address={{ hostvars['lists.archlinux.org']['wireguard_address'] }} port protocol=tcp port=8024 accept"
+  tags:
+    - firewall
+
+- name: start and enable mailman{.service,-*.timer}
+  systemd: name={{ item }} enabled=yes daemon_reload=yes state=started
+  loop:
+    - mailman3.service
+    - mailman3-digests.timer
+    - mailman3-gatenews.timer
+    - mailman3-notify.timer
+    - uwsgi@mailman\x2dweb.service

roles/mailman3/templates/mailman.cfg.j2

+[mailman]
+layout: fhs
+
+[database]
+class: mailman.database.postgresql.PostgreSQLDatabase
+url: postgres://{{ vault_mailman_db_user }}:{{ vault_mailman_db_password }}@/mailman
+
+[webservice]
+admin_user: {{ vault_mailman_admin_user }}
+admin_pass: {{ vault_mailman_admin_pass }}
+
+[mta]
+configuration: /etc/postfix.cfg
+lmtp_host: {{ hostvars['mailman3.archlinux.org']['wireguard_address'] }}
+lmtp_port: 8024
+smtp_host: {{ hostvars['lists.archlinux.org']['wireguard_address'] }}
+smtp_port: 25
+
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
+enable: yes
+configuration: /etc/mailman-hyperkitty.cfg

roles/mailman3/templates/nginx.d.conf.j2

+server {
+    listen       80;
+    listen       [::]:80;
+    server_name  {{ lists_domain }} localhost;
+
+    set_real_ip_from {{ hostvars['lists.archlinux.org']['wireguard_address'] }}/32;
+    real_ip_header   X-Forwarded-For;
+
+    access_log   /var/log/nginx/{{ lists_domain }}/access.log main;
+    access_log   /var/log/nginx/{{ lists_domain }}/access.log.json json_main;
+    error_log    /var/log/nginx/{{ lists_domain }}/error.log;
+
+    location /static {
+      alias /var/lib/mailman-web/static;
+    }
+
+    # include uwsgi_params
+    location / {
+      include /etc/nginx/uwsgi_params;
+      uwsgi_pass unix:/run/mailman-web/mailman-web.sock;
+    }
+}

roles/mailman3/templates/settings.py.j2

+# mailman-web config
+
+from mailman_web.settings.base import *
+from mailman_web.settings.mailman import *
+
+
+#: Default list of admins who receive the emails from error logging.
+ADMINS = (
+    ('Mailman Suite Admin', 'root@{{ lists_domain }}'),
+)
+
+# Postgresql datbase setup.
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'mailman-web',
+        'USER': '{{ vault_mailman_web_db_user }}',
+        'PASSWORD': '{{ vault_mailman_web_db_password }}',
+    }
+}
+
+#: See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [
+    "localhost",  # Archiving API from Mailman, keep it.
+    "{{ lists_domain }}",
+]
+
+#: Current Django Site being served. This is used to customize the web host
+#: being used to serve the current website. For more details about Django
+#: site, see: https://docs.djangoproject.com/en/dev/ref/contrib/sites/
+SITE_ID = 1
+
+SECRET_KEY = '{{ vault_mailman_web_secret_key }}'
+
+MAILMAN_REST_API_USER = '{{ vault_mailman_admin_user }}'
+MAILMAN_REST_API_PASS = '{{ vault_mailman_admin_pass }}'
+MAILMAN_ARCHIVER_KEY = '{{ vault_mailman_archiver_key }}'
+
+# https://docs.djangoproject.com/en/3.2/topics/email/#smtp-backend
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST = '{{ hostvars['lists.archlinux.org']['wireguard_address'] }}'
+EMAIL_PORT = 25
+
+#: Sender in Emails sent out by Postorius.
+DEFAULT_FROM_EMAIL = 'postorius@{{ lists_domain }}'
+SERVER_EMAIL = 'root@{{ lists_domain }}'
+
+POSTORIUS_TEMPLATE_BASE_URL = 'http://localhost'
+HYPERKITTY_ALLOW_WEB_POSTING = False
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': '/var/lib/mailman-web/fulltext_index'
+    }
+}

roles/mailman3/templates/urls.py.j2

+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Postorius.
+#
+# Postorius is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Postorius is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Postorius.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from django.conf.urls import include, url
+from django.contrib import admin
+from django.urls import reverse_lazy
+from django.views.generic import RedirectView
+
+urlpatterns = [
+    url(r'^$', RedirectView.as_view(
+        url=reverse_lazy('list_index'),
+        permanent=True)),
+    url(r'^mailman3/', include('postorius.urls')),
+    url(r'^archives/', include('hyperkitty.urls')),
+    url(r'', include('django_mailman3.urls')),
+    url(r'^accounts/', include('allauth.urls')),
+    # Django admin
+    url(r'^admin3/', admin.site.urls),
+]

roles/nginx/defaults/main.yml

 ---
 letsencrypt_validation_dir: "/var/lib/letsencrypt"
+nginx_firewall_zone: