README.md

@@ -159,7 +159,6 @@ The following steps should be used to update our managed servers:
 
 #### Services
 
-  - mailman
   - projects (projects.archlinux.org)
 
 ### apollo
@@ -213,6 +212,7 @@ The following steps should be used to update our managed servers:
 #### Services
   - ~/user/ webhost
 
+<<<<<<< HEAD
 ### accounts.archlinux.org
 
 This server is /special/. It runs keycloak and is central to our unified Arch Linux account management world.
@@ -258,6 +258,11 @@ Medium-fast-ish packet.net box with Debian on it. Is currently maintained manual
 #### Services
   - GitLab runner
 
+## mailman.archlinux.org
+
+#### Services:
+  - mailman
+
 ## Ansible repo workflows
 
 ### Replace vault password and change vaulted passwords

group_vars/all/vault_mailman.yml

+$ANSIBLE_VAULT;1.1;AES256
+65633134383537376232316263656530316464323366363137633262363930663739633337333632
+3732633133633466666566353637366234343165623731320a653663646338393037366166623534
+39643163623739623634393130333162666330656638343264336163366263643832653966613964
+3163386530343835370a396665373839613936313237396139376433616237373535646131616532
+62323239643938653435356263626631306666663538373731636236353362333630346165623237
+35613238373834633766363730333032653536643834616532356639666133373036386465373066
+64633935366435633130396535646362363131396631363739373632626565393563653231636431
+63393161666337343836653336303831393133383032653637633138346434646566623935393864
+65633837623561353330393161343863666636316362363239333139353066656662343761646537
+3737306366306236343565306662623038316138663863626666

group_vars/all/vault_postgres.yml

 $ANSIBLE_VAULT;1.1;AES256
-63366165396562363135333830643834663532353865653138636334343664343138313365336436
-6436383535623062656466646461303365373533363430610a373930366237326137613362336164
-34633732376464646437356137343631353434396432623633353036663738343538303966353464
-6535383735323763330a633436646331623131633564393130376139363061663139626366666634
-66643763376463386231663832303664633632613530633266313431646333316534326237373137
-36663233303561313965633333313738643331396465666263663034336163303339383437353332
-64626462393336623130316535303531623634656235313939636232653930303432636364386330
-61613736356239613935323430396233323335363862353039343936653631656562656231323237
-65336663666166326630663565353032303461613431343662326535363761333665336137316161
-62366561383736326338346362333939386332356137653866383334333262663839313438363631
-31663062373366383133343063313931366637346131626338656538613166656664393930373733
-62336639356361663962373039366362343966616363653838313538623039666665633565323765
-62346337636336663333613766396436313238346565633133383030633931613965396261333766
-31326337646438623631616639383764636332336336353830616633396336333536623861356637
-333839656139326135636238643561356366
+37613035623633363139396563653537663135636238653733333835326662333963653232363137
+3031356138613966666563373637363866346335336335390a663062356366306661346231383139
+39323734613732643839396133663265363936336565343466373631623763303165383366363764
+3832353836373733330a333162393364383433643865666361326633373537373663653439376135
+66306633393933666562346338396165303631373936656161303937613638336530656363313164
+32343130386661373862653030623662636534376563633133323066376633643037373262353361
+37373263653836303163376537303639636136363334343430646563666337646530313636346435
+63613537393966636430306462313663343432353462343861306430643162326237386239613831
+65663066653639613432316438326633366364643539316362306132633033643636623732646135
+63303533666666306564316435613264343335353561613930663133346462393130356164646163
+31336139643763366339323036373430623466616330393532306466656665303238363834396331
+33636437666165303163343636363733626235323435653234333930373934386565616335343034
+31396366336635613261666533633033626238323664653762316439303538656533356339616336
+38323766396161643036613565333162646561326630646363373430303532353465336532366335
+63313966616534336566636538356164336365396638663165613331323036343837396264373630
+39363065393332353961633235663466343134656534346330653662636461313934383436663639
+32366265656630333266323935353164643962646364643734346162313362313031653939313965
+64316237303338356138313138323734653432376337333062343837316132326332623436346139
+626338366338626432343064333939613861

host_vars/mailman3.archlinux.org

+---
+filesystem: btrfs

playbooks/mailman.archlinux.org.yml

+---
+
+- name: setup mailman.archlinux.org
+  hosts: mailman3.archlinux.org
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: tools }
+    - { role: firewalld }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: certbot }
+    - { role: nginx }
+    - { role: sudo }
+    - { role: borg-client }
+    - { role: postfix, postfix_relayhost: "orion.archlinux.org" }
+    - { role: fail2ban }
+    - role: postgres
+      postgres_max_connections: 100
+      postgres_ssl: 'off'
+      postgres_shared_buffers: 512MB
+      postgres_effective_cache_size: 1GB
+    - { role: mailman }

roles/mailman/defaults/main.yml

+# Mailman
+mailman_domain: mailman3.archlinux.org # lists.archlinux.org
+mailman_db_user: mailman
+mailman_nginx_conf: /etc/nginx/nginx.d/mailman.conf
+
+# Hyperkitty
+hyperkitty_dir: /usr/share/webapps/hyperkitty
+hyperkitty_db_user: hyperkitty
+hyperkitty_admin_user: hyperkitty
+hyperkitty_admin_email: hyperkitty@archlinux.org

roles/mailman/files/uwsgi-secure@.service

+[Unit]
+Description=uWSGI service unit
+After=syslog.target
+
+[Service]
+ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/%I.ini
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStop=/bin/kill -INT $MAINPID
+Type=notify
+SuccessExitStatus=15 17 29 30
+StandardError=syslog
+NotifyAccess=all
+KillSignal=SIGQUIT
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectSystem=full
+ReadWriteDirectories=/etc/webapps /var/lib/
+ProtectHome=yes
+NoNewPrivileges=yes
+
+[Install]
+WantedBy=multi-user.target

roles/mailman/files/uwsgi-secure@.socket

+[Unit]
+Description=Socket for uWSGI %I
+
+[Socket]
+ListenStream=/run/%I/%I.sock
+SocketGroup=http
+SocketMode=0660
+
+[Install]
+WantedBy=sockets.target

roles/mailman/tasks/main.yml

 ---
 
+- name: install mailman
+  pacman: name=mailman3,mailman3-hyperkitty,postorius,python-psycopg2,uwsgi,uwsgi-plugin-python,hyperkitty state=present
+
+- name: add mailman postgres db
+  postgresql_db: db=mailman
+  become: yes
+  become_user: postgres
+  become_method: su
+
+- name: add mailman postgres user
+  postgresql_user: db=mailman name=mailman password={{ vault_postgres_users.mailman }} encrypted=true
+  become: yes
+  become_user: postgres
+  become_method: su
+
+- template: src="mailman.cfg.j2" dest="/etc/mailman.cfg" owner=mailman group=root mode=0644
+
+- name: add hyperkitty postgres db
+  postgresql_db: db=hyperkitty
+  become: yes
+  become_user: postgres
+  become_method: su
+
+- name: add hyperkitty postgres user
+  postgresql_user: db=hyperkitty name={{ hyperkitty_db_user }} password={{ vault_postgres_users.hyperkitty }} encrypted=true
+  become: yes
+  become_user: postgres
+  become_method: su
+
+- file: src=/etc/webapps/hyperkitty/settings_local.py dest=/usr/share/webapps/hyperkitty/settings_local.py owner=root group=hyperkitty state=link
+
+- template: src="hyperkitty.py.j2" dest="/etc/webapps/hyperkitty/settings_local.py" owner=root group=hyperkitty mode=0644
+
+# TODO: only run when required, ie. hyperkitty package updated
+- name: generate a hyperkitty database
+  command: django-admin migrate --pythonpath {{ hyperkitty_dir }} --settings settings
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+
+- name: run collectstatic for hyperkitty
+  command: django-admin collectstatic --pythonpath {{ hyperkitty_dir }} --settings settings --noinput
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+
+- name: run compress for hyperkitty
+  command: django-admin compress --pythonpath {{ hyperkitty_dir }} --settings settings
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+
+# TODO: run only once
+- name: populate the hyperkitty database
+  command: django-admin loaddata --pythonpath {{ hyperkitty_dir }} --settings settings first_start
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+
+# TODO: run only once
+- name: populate the hyperkitty database
+  command: django-admin loaddata --pythonpath {{ hyperkitty_dir }} --settings settings first_start
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+
+- name: check if hyperkitty admin user exists
+  command: echo "from django.contrib.auth import get_user_model; User = get_user_model(); print(User.objects.filter(username='{{hyperkitty_admin_user}}').count()>0)" | /usr/bin/python ./manage.py shell
+  args:
+    chdir: "{{ hyperkitty_dir }}"
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+  register: hyperkitty_superuser_existed
+
+- name: create hyperkitty admin user
+  command: django-admin createsuperuser --pythonpath {{ hyperkitty_dir }} --settings settings --noinput --username={{ hyperkitty_admin_user }} --email={{ hyperkitty_admin_email }}
+  environment:
+    DJANGO_SUPERUSER_PASSWORD: "{{ vault_mailman_hyperkitty_admin_password }}"
+  become: yes
+  become_user: hyperkitty
+  become_method: sudo
+  when: not hyperkitty_superuser_existed
+
+- name: copy uwsgi-secure@.socket service
+  copy: src=uwsgi-secure@.service dest=/etc/systemd/system/uwsgi-secure@.service
+  notify:
+    - daemon reload
+
+- name: copy uwsgi-secure@.socket service
+  copy: src=uwsgi-secure@.socket dest=/etc/systemd/system/uwsgi-secure@.socket
+  notify:
+    - daemon reload
+
+- name: create ssl cert
+  command: certbot certonly --email webmaster@archlinux.org --agree-tos --rsa-key-size 4096 --renew-by-default --webroot -w {{letsencrypt_validation_dir}} -d '{{ mailman_domain }}' creates='/etc/letsencrypt/live/{{ mailman_domain }}/fullchain.pem'
+
+- name: make nginx log dir
+  file: path=/var/log/nginx/{{ mailman_domain }} state=directory owner=root group=root mode=0755
+
 - name: set up nginx
-  template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/mailman.conf owner=root group=root mode=644
+  template: src=nginx.d.conf.j2 dest="{{ mailman_nginx_conf }}" owner=root group=root mode=644
   notify:
     - reload nginx
-  tags:
-    - nginx
+  tags: ['nginx']
 
 - name: make nginx log dir
   file: path=/var/log/nginx/{{ mailman_domain }} state=directory owner=root group=root mode=0755
-  when: archweb_site
-  tags:
-    - nginx
 
+- name: enable hyperkitty socket
+  service: name="uwsgi-secure@hyperkitty.socket" enabled=yes state=started
+
+- name: enable hyperkitty asynchronous operations service
+  service: name="hyperkitty-qcluster.service" enabled=yes state=started
+
+- name: start and enable mailman core service
+  service: name="mailman3.service" enabled=yes state=started
+
+- name: start and enable mailman digests timer
+  service: name="mailman3-digests.timer" enabled=yes state=started
+
+- name: start and enable mailman notify timer
+  service: name="mailman3-notify.timer" enabled=yes state=started

roles/mailman/templates/hyperkitty.py.j2

+SECRET_KEY = '{{ vault_mailman_hyperkitty_secret_key }}'
+
+DEBUG = False
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+     ('HyperKitty Admin', 'admin@archlinux.org'),
+)
+
+ALLOWED_HOSTS = [
+	"localhost",
+	"127.0.0,1",
+	"{{ mailman_domain }}",
+]
+
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST = 'localhost'
+EMAIL_PORT = 25
+EMAIL_HOST_USER = ""
+EMAIL_HOST_PASSWORD = ""
+
+DATABASES = {
+    'default': {
+          'ENGINE'  : 'django.db.backends.postgresql_psycopg2',
+          'NAME'    : 'hyperkitty',
+          'USER'    : '{{ hyperkitty_db_user }}',
+          'PASSWORD': '{{ vault_postgres_users.hyperkitty }}',
+          'HOST'    : 'localhost',
+          'PORT'    : '',
+    }
+}

roles/mailman/templates/mailman.cfg.j2

+[mailman]
+layout: fhs
+
+[database]
+class: mailman.database.postgresql.PostgreSQLDatabase
+url: postgres://{{ mailman_db_user}}:{{ vault_postgres_users.mailman }}@localhost/mailman
+
+#[archiver.hyperkitty]
+#class: mailman_hyperkitty.Archiver
+#enable: yes
+#configuration: /etc/mailman-hyperkitty.cfg

roles/mailman/templates/nginx.d.conf.j2

 server {
     listen       80;
     listen       [::]:80;
-    server_name  mailman.archlinux.org;
+    server_name  mailman3.archlinux.org;
 
     access_log   /var/log/nginx/{{ mailman_domain }}/access.log reduced;
     error_log    /var/log/nginx/{{ mailman_domain }}/error.log;
@@ -17,17 +17,21 @@ server {
 server {
     listen       443 ssl http2;
     listen       [::]:443 ssl http2;
-    server_name  mailman.archlinux.org;
+    server_name  mailman3.archlinux.org;
 
     access_log   /var/log/nginx/{{ mailman_domain }}/access.log reduced;
     error_log    /var/log/nginx/{{ mailman_domain }}/error.log;
 
-    ssl_certificate      /etc/letsencrypt/live/mailman.archlinux.org/fullchain.pem;
-    ssl_certificate_key  /etc/letsencrypt/live/mailman.archlinux.org/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/mailman.archlinux.org/chain.pem;
+    ssl_certificate      /etc/letsencrypt/live/mailman3.archlinux.org/fullchain.pem;
+    ssl_certificate_key  /etc/letsencrypt/live/mailman3.archlinux.org/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/mailman3.archlinux.org/chain.pem;
 
-    location / {
-        access_log off;
-        return 301 https://{{ mailman_domain }}$request_uri;
+    charset utf-8;
+    client_max_body_size 75M;
+    root /usr/share/webapps/hyperkitty;
+
+    location ~^/(accounts|admin|hyperkitty)/(.*)$ {
+         include /etc/nginx/uwsgi_params;
+         uwsgi_pass unix:/run/hyperkitty/hyperkitty.sock;
     }
 }