Merge branch 'aurweb-v6.0.0' into 'master'

aurweb: update rollout for >= v6.0.0

See merge request !525

host_vars/aur-dev.archlinux.org/vault_aurweb.yml

 $ANSIBLE_VAULT;1.1;AES256
-34333262646637333030666438633639653163316562626232383765383733323632353635626534
-6136663761633231326438323466366436626635363862370a633832383065626666346436633362
-31346430383730353234313166363665326663316233383561623765393834356661363134663138
-3233653039633830610a356631313330626533313239316662336138306664343436336630653362
-62653538663665336339636162316564323638303864636533393632633337396236653735663236
-65643965636166646165386335636462383866346139393934626335313033636330646239373265
-363331656536343431613936636331646233
+38373334353261373639366164343736343838333930326664386665376531346434373436656464
+6266633062343062633637643635333235646461313430330a353635313234613035396266303038
+37383336366265383437363265666135663937343234333734656231643935396432326332346665
+6536343363326463660a343162376130623364363434326636383762663364653666313865633831
+34323032313562326439363739616535363065356239326336326237663366653936326563613865
+64373163653132386539303161356231353234316239623836636636666338663638353161316230
+61356137663864373064663439356438393536366131323562626637373461366134363331373664
+35613538366164323630353262323332393033363831303761306365663561376631303061393264
+64356335393736646361626438656539633132373233303461393838356430323935343632356363
+62356532626430303964643632613039336564626332643438373939646236396331613136323635
+39643665643533306230333430626631356365326435646139396666653164353463633134366665
+36373761396135323133346538316632333232316331663632346134653138363736373634636536
+6233

host_vars/aur.archlinux.org/vault_aurweb.yml

 $ANSIBLE_VAULT;1.1;AES256
-39666139613363323634346538326233393165646338393231386136623839613135623232663665
-6636316361636366623031323331663138313635616431360a373633356434666434303063653564
-61636662616633393039343633376333343266373465646235386437336135346132303162373431
-6232383034626363340a653662373932396435356433616431303861313863363263656162663964
-63653661616438313031323639346236373339656139626561623166373664346438306639343862
-65613735663135653764363935366637313864616563373665393536316438393930306637313261
-633062343032303033303039646165613961
+33633635313563306332363963333565623762366239633461613334383232363735363031333163
+3132313638313433353063303130653863343132646366650a326631643836373364376332613765
+37303363326232383433376162616531316434646264666435633638653535386138616138626463
+6565323133343136640a643134653330626430373536303665316663363563613263343931393735
+35626233376138356132633265313464656437643035633131643931313262393863333662353234
+63663865373935353965343439663239343063313235653533306265396236343566393232633738
+66623165303265663538303632616633646136306231383763643532666336623832326566646134
+31636363356337353338646233653030353763363363313932373836356264396361636533376132
+31613539306361383864666135363063393330643237393366636434353561393033353130323062
+38383939353334643161616135313466653637376634316361663330666234383338353933346232
+38623562613735613631623033323963386139613836343535386133643961313162373863363236
+32663836356539643936366335663334353836373237323963613964326165656431323739326237
+3763

hosts

@@ -98,7 +98,6 @@ repro1.pkgbuild.com
 repro2.pkgbuild.com
 
 [memcached]
-aur.archlinux.org
 wiki.archlinux.org
 patchwork.archlinux.org
 

playbooks/aur-dev.archlinux.org.yml

@@ -11,8 +11,7 @@
     - { role: nginx }
     - { role: mariadb }
     - { role: sudo }
-    - { role: php_fpm, php_extensions: ['iconv', 'memcached', 'mysqli', 'pdo_mysql'], zend_extensions: ['opcache'] }
-    - { role: memcached }
+    - { role: redis }
     - { role: uwsgi }
     - { role: borg_client, tags: ["borg"] }
     - { role: postfix_null }

playbooks/aur.archlinux.org.yml

@@ -13,8 +13,7 @@
     - { role: nginx }
     - { role: mariadb, mariadb_innodb_buffer_pool_size: '1G' }
     - { role: sudo }
-    - { role: php_fpm, php_extensions: ['iconv', 'memcached', 'mysqli', 'pdo_mysql'], zend_extensions: ['opcache'] }
-    - { role: memcached }
+    - { role: redis }
     - { role: uwsgi }
     - { role: borg_client, tags: ["borg"] }
     - { role: postfix_null }

roles/aurweb/defaults/main.yml

 ---
+aurweb_asgi_bind: '127.0.0.1:8000'
 
 aurweb_domain: 'aur.archlinux.org'
 aurweb_repository: 'https://gitlab.archlinux.org/archlinux/aurweb.git'
@@ -8,6 +9,7 @@ aurweb_git_dir: "{{ aurweb_dir }}/aur.git"
 aurweb_git_hook: '/usr/local/bin/aurweb-git-update'
 aurweb_nginx_conf: '/etc/nginx/nginx.d/aurweb.conf'
 aurweb_version: 'live'
+aurweb_pgp_keys: ['0F985B6F99B6686854C44EC3F7E46DED420788F3']
 
 aurweb_db: 'aur'
 aurweb_db_host: 'localhost'
@@ -18,9 +20,11 @@ aurweb_socket: '/run/php-fpm/{{aurweb_user}}.socket'
 cgit_socket: '/run/uwsgi/cgit.sock'
 smartgit_socket: '/run/uwsgi/smartgit.sock'
 
-aurweb_cache: 'memcache'
+aurweb_cache: 'redis'
 aurweb_cache_pkginfo_ttl: '86400'
 aurweb_request_limt: '4000'
 aurweb_window_length: '86400'
 aurweb_memcached_socket: '/run/memcached/aurweb.sock'
 aurweb_memcached_memory: 2048
+
+aurweb_workers: 4

roles/aurweb/tasks/main.yml

@@ -6,20 +6,11 @@
       - asciidoc
       - highlight
       - make
-      - php-memcached
-      - pyalpm
-      - python-alembic
-      - python-bleach
-      - python-markdown
-      - python-mysql-connector
-      - python-pygit2
-      - python-srcinfo
-      - python-fastapi
-      - python-jinja
-      - python-email-validator
-      - python-orjson
       - sudo
       - uwsgi-plugin-cgi
+      - python-poetry
+      - gcc
+      - pkg-config
 
 - name: install the cgit package
   pacman:
@@ -41,11 +32,21 @@
 - name: Create directory
   file: path={{ aurweb_dir }} state=directory owner={{ aurweb_user }} group=http mode=0775
 
+- name: receive valid signing keys
+  command: /usr/bin/gpg --keyserver keys.openpgp.org --recv {{ item }}
+  loop: '{{ aurweb_pgp_keys }}'
+  become: true
+  become_user: "{{ aurweb_user }}"
+  register: gpg
+  changed_when: "gpg.rc == 0"
+
 - name: clone aurweb repo
   git: >
     repo={{ aurweb_repository }}
     dest="{{ aurweb_dir }}"
     version={{ aurweb_version }}
+    verify_commit: true
+    gpg_whitelist: '{{ aurweb_pgp_keys }}'
   become: true
   become_user: "{{ aurweb_user }}"
   register: release
@@ -79,7 +80,7 @@
   no_log: true
 
 - name: initialize the database
-  command: python -m aurweb.initdb
+  command: poetry run python -m aurweb.initdb
   args:
     chdir: "{{ aurweb_dir }}"
   become: true
@@ -87,7 +88,7 @@
   when: db_created.changed
 
 - name: run migrations
-  command: alembic upgrade head
+  command: poetry run alembic upgrade head
   args:
     chdir: "{{ aurweb_dir }}"
   environment:
@@ -97,18 +98,43 @@
   when: release.changed or db_created.changed
 
 - name: Check python module availability
-  command: "python3 -c 'import aurweb'"
+  command: poetry run python3 -c 'import aurweb'
+  args:
+    chdir: "{{ aurweb_dir }}"
+  become: true
+  become_user: "{{ aurweb_user }}"
   ignore_errors: true
   register: aurweb_installed
   tags:
     - skip_ansible_lint
 
 - name: Install python module
-  command: "python3 setup.py install --install-scripts=/usr/local/bin"
+  command: poetry install
   args:
     chdir: "{{ aurweb_dir }}"
+  become: true
+  become_user: "{{ aurweb_user }}"
   when: release.changed or aurweb_installed.rc != 0
 
+- name: install custom aurweb-git-auth wrapper script
+  template: src=aurweb-git-auth.sh.j2 dest=/usr/local/bin/aurweb-git-auth.sh owner=root group=root mode=0755
+  when: release.changed
+
+- name: install custom aurweb-git-serve wrapper script
+  template: src=aurweb-git-serve.sh.j2 dest=/usr/local/bin/aurweb-git-serve.sh owner=root group=root mode=0755
+  when: release.changed
+
+- name: install custom aurweb-git-update wrapper script
+  template: src=aurweb-git-update.sh.j2 dest=/usr/local/bin/aurweb-git-update.sh owner=root group=root mode=0755
+  when: release.changed
+
+- name: link custom aurweb-git-update wrapper to hooks/update
+  file:
+    src: /usr/local/bin/aurweb-git-update.sh
+    dest: "{{ aurweb_dir }}/aur.git/hooks/update"
+    state: link
+  when: release.changed
+
 - name: Generate HTML documentation
   make:
     chdir: "{{ aurweb_dir }}/doc"
@@ -136,16 +162,6 @@
 - name: make nginx log dir
   file: path=/var/log/nginx/{{ aurweb_domain }} state=directory owner=root group=root mode=0755
 
-- name: configure php-fpm
-  template:
-    src=php-fpm.conf.j2 dest="/etc/php/php-fpm.d/{{ aurweb_user }}.conf"
-    owner=root group=root mode=0644
-  notify:
-    - restart php-fpm@{{ aurweb_user }}
-
-- name: start and enable systemd socket
-  service: name=php-fpm@{{ aurweb_user }}.socket state=started enabled=true
-
 - name: install cgit configuration
   template: src=cgitrc.j2 dest="{{ aurweb_conf_dir }}/cgitrc" owner=root group=root mode=0644
 
@@ -223,15 +239,6 @@
   tags:
     - skip_ansible_lint
 
-- name: create symlink for git hook
-  file:
-    src: "{{ aurweb_git_hook }}"
-    dest: "{{ aurweb_git_dir }}/hooks/update"
-    owner: root
-    group: root
-    mode: 0755
-    state: link
-
 - name: install AUR systemd service and timers
   template: src={{ item }}.j2 dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
   with_items:
@@ -239,7 +246,6 @@
     - aurweb-git.timer
     - aurweb-aurblup.service
     - aurweb-aurblup.timer
-    - aurweb-memcached.service
     - aurweb-mkpkglists.service
     - aurweb-mkpkglists.timer
     - aurweb-pkgmaint.service
@@ -250,20 +256,22 @@
     - aurweb-tuvotereminder.timer
     - aurweb-usermaint.service
     - aurweb-usermaint.timer
+    - aurweb.service
+
+- name: configure sshd
+  template: src=aurweb_config.j2 dest={{ sshd_includes_dir }}/aurweb_config owner=root group=root mode=0600 validate='/usr/sbin/sshd -t -f %s'
+  notify:
+    - restart sshd
 
 - name: start and enable AUR systemd services and timers
-  service: name={{ item }} enabled=yes state=started
+  service: name={{ item }} enabled=yes state=restarted daemon_reload=yes
   with_items:
     - aurweb-git.timer
     - aurweb-aurblup.timer
-    - aurweb-memcached.service
     - aurweb-mkpkglists.timer
     - aurweb-pkgmaint.timer
     - aurweb-popupdate.timer
     - aurweb-tuvotereminder.timer
     - aurweb-usermaint.timer
-
-- name: configure sshd
-  template: src=aurweb_config.j2 dest={{ sshd_includes_dir }}/aurweb_config owner=root group=root mode=0600 validate='/usr/sbin/sshd -t -f %s'
-  notify:
-    - restart sshd
+    - aurweb.service
+  when: release.changed

roles/aurweb/templates/aurweb-aurblup.service.j2

@@ -6,7 +6,8 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-aurblup
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run aurweb-aurblup
 
 ReadWritePaths={{ aurweb_dir }}
 NoNewPrivileges=true

roles/aurweb/templates/aurweb-git-auth.sh.j2

+#!/bin/bash
+cd "{{ aurweb_dir }}"
+exec poetry run aurweb-git-auth "$@"

roles/aurweb/templates/aurweb-git-serve.sh.j2

+#!/bin/bash
+cd "{{ aurweb_dir }}"
+exec poetry run aurweb-git-serve "$@"

roles/aurweb/templates/aurweb-git-update.sh.j2

+#!/bin/bash
+cd "{{ aurweb_dir }}"
+exec poetry run aurweb-git-update "$@"

roles/aurweb/templates/aurweb-mkpkglists.service.j2

@@ -6,7 +6,8 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-mkpkglists --extended
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run aurweb-mkpkglists --extended
 
 NoNewPrivileges=true
 LockPersonality=true

roles/aurweb/templates/aurweb-pkgmaint.service.j2

@@ -6,7 +6,8 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-pkgmaint
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run aurweb-pkgmaint
 
 NoNewPrivileges=true
 LockPersonality=true
@@ -15,7 +16,7 @@ CapabilityBoundingSet=
 PrivateDevices=true
 PrivateTmp=true
 ProtectSystem=strict
-ProtectHome=true
+ProtectHome=read-only
 
 MemoryDenyWriteExecute=true
 RemoveIPC=true

roles/aurweb/templates/aurweb-popupdate.service.j2

@@ -6,7 +6,8 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-popupdate
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run aurweb-popupdate
 
 NoNewPrivileges=true
 LockPersonality=true
@@ -15,7 +16,7 @@ CapabilityBoundingSet=
 PrivateDevices=true
 PrivateTmp=true
 ProtectSystem=strict
-ProtectHome=true
+ProtectHome=read-only
 
 MemoryDenyWriteExecute=true
 RemoveIPC=true

roles/aurweb/templates/aurweb-tuvotereminder.service.j2

@@ -6,7 +6,8 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-tuvotereminder
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run aurweb-tuvotereminder
 
 NoNewPrivileges=true
 LockPersonality=true

roles/aurweb/templates/aurweb-usermaint.service.j2

@@ -6,7 +6,8 @@ After=mysqld.service
 [Service]
 Type=oneshot
 User={{ aurweb_user }}
-ExecStart=/usr/local/bin/aurweb-usermaint
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run aurweb-usermaint
 
 NoNewPrivileges=true
 LockPersonality=true
@@ -15,7 +16,7 @@ CapabilityBoundingSet=
 PrivateDevices=true
 PrivateTmp=true
 ProtectSystem=strict
-ProtectHome=true
+ProtectHome=read-only
 
 MemoryDenyWriteExecute=true
 RemoveIPC=true

roles/aurweb/templates/aurweb.service.j2

+[Unit]
+Description=aurweb asgi server
+
+[Service]
+User={{ aurweb_user }}
+WorkingDirectory={{ aurweb_dir }}
+ExecStart=/usr/bin/poetry run gunicorn \
+    --log-config {{ aurweb_dir }}/logging.conf \
+    --bind {{ aurweb_asgi_bind }} \
+    --workers {{ aurweb_workers }} \
+    -k uvicorn.workers.UvicornWorker \
+    aurweb.asgi:app
+
+[Install]
+WantedBy=multi-user.target

roles/aurweb/templates/aurweb_config.j2

 Match User {{ aurweb_user }}
         PasswordAuthentication no
-        AuthorizedKeysCommand /usr/local/bin/aurweb-git-auth "%t" "%k"
+        AuthorizedKeysCommand /usr/local/bin/aurweb-git-auth.sh "%t" "%k"
         AuthorizedKeysCommandUser {{ aurweb_user }}
         AcceptEnv AUR_OVERWRITE

roles/aurweb/templates/cgitrc.j2

 virtual-root=/cgit/
 clone-prefix=https://{{ aurweb_domain }}
 noheader=0
-favicon=/images/favicon.ico
+favicon=/static/images/favicon.ico
 logo=
-css=/css/cgit.css
+css=/static/css/cgit.css
 snapshots=tar.gz
 readme=:README.md
 readme=:README

roles/aurweb/templates/config.j2

@@ -4,6 +4,7 @@ user = {{ aurweb_db_user }}
 password = {{ vault_aurweb_db_password }}
 
 [options]
+aurwebdir = {{ aurweb_dir }}
 {% if maintenance is defined and maintenance %}
 enable_maintenance = 1
 maintenance-exceptions = {{ maintenance_remote_machine }}
@@ -16,7 +17,7 @@ cache_pkginfo_ttl = {{ aurweb_cache_pkginfo_ttl }}
 aur_location = https://{{ aurweb_domain }}
 git_clone_uri_anon = https://{{ aurweb_domain }}/%s.git
 git_clone_uri_priv = ssh://{{ aurweb_user }}@{{ aurweb_domain }}/%s.git
-memcache_servers = {{ aurweb_memcached_socket }}:0
+redis_address = redis://localhost
 
 [ratelimit]
 request_limit = {{ aurweb_request_limt }}
@@ -27,9 +28,13 @@ Ed25519 = SHA256:RFzBCUItH9LZS0cKB5UE6ceAYhBD5C8GeOBip8Z11+4
 ECDSA = SHA256:uTa/0PndEgPZTf76e1DFqXKJEXKsn7m9ivhLQtzGOCI
 RSA =  SHA256:5s5cIyReIfNNVGRFdDbe3hdYiI5OelHGpw2rOUud3Q8
 
+[auth]
+git-serve-cmd = /usr/local/bin/aurweb-git-serve.sh
+
 [serve]
 repo-path = {{ aurweb_git_dir }}
 git-shell-cmd = /usr/bin/sh
+git-update-cmd = /usr/local/bin/aurweb-git-update.sh
 ssh-cmdline = ssh {{ aurweb_user }}@{{ aurweb_domain }}
 
 [update]
@@ -45,3 +50,11 @@ packagesmetafile = {{ aurweb_dir }}/web/html/packages-meta-v1.json.gz
 packagesmetaextfile = {{ aurweb_dir }}/web/html/packages-meta-ext-v1.json.gz
 pkgbasefile = {{ aurweb_dir }}/web/html/pkgbase.gz
 userfile = {{ aurweb_dir }}/web/html/users.gz
+
+[notifications]
+notify-cmd = aurweb-notify
+{# An email used for server error notifications. #}
+postmaster = {{ vault_aurweb_postmaster }}
+
+[fastapi]
+session_secret = {{ vault_aurweb_secret }}