.coveragerc

@@ -3,7 +3,7 @@ disable_warnings = already-imported
 
 [report]
 include = aurweb/*
-fail_under = 85
+fail_under = 95
 exclude_lines =
     if __name__ == .__main__.:
     pragma: no cover

.dockerignore

-*/*.mo
+# Config files
 conf/config
 conf/config.sqlite
 conf/config.sqlite.defaults
 conf/docker
 conf/docker.defaults
+
+# Compiled translation files
+**/*.mo
+
+# Typical virtualenv directories
+env/
+venv/
+.venv/
+
+# Test output
+htmlcov/
+test-emails/
+test/__pycache__
+test/test-results
+test/trash_directory*
+.coverage
+.pytest_cache

.editorconfig

 # EditorConfig configuration for aurweb
-# https://EditorConfig.org
+# https://editorconfig.org
 
 # Top-most EditorConfig file
 root = true
@@ -8,6 +8,3 @@ root = true
 end_of_line = lf
 insert_final_newline = true
 charset = utf-8
-
-[*.{php,t}]
-indent_style = tab

.env

 FASTAPI_BACKEND="uvicorn"
 FASTAPI_WORKERS=2
 MARIADB_SOCKET_DIR="/var/run/mysqld/"
-AURWEB_PHP_PREFIX=https://localhost:8443
 AURWEB_FASTAPI_PREFIX=https://localhost:8444
 AURWEB_SSHD_PREFIX=ssh://aur@localhost:2222
 GIT_DATA_DIR="./aur.git/"

.gitignore

@@ -24,7 +24,6 @@ conf/docker
 conf/docker.defaults
 data.sql
 dummy-data.sql*
-env/
 fastapi_aw/
 htmlcov/
 po/*.mo
@@ -32,7 +31,7 @@ po/*.po~
 po/POTFILES
 schema/aur-schema-sqlite.sql
 test/test-results/
-test/trash directory*
+test/trash_directory*
 web/locale/*/
 web/html/*.gz
 
@@ -44,3 +43,21 @@ doc/rpc.html
 
 # Ignore .python-version file from Pyenv
 .python-version
+
+# Ignore coverage report
+coverage.xml
+
+# Ignore pytest report
+report.xml
+
+# Ignore test emails
+test-emails/
+
+# Ignore typical virtualenv directories
+env/
+venv/
+.venv/
+
+# Ignore some terraform files
+/ci/tf/.terraform
+/ci/tf/terraform.tfstate*

.gitlab-ci.yml

@@ -13,24 +13,22 @@ variables:
   TEST_RECURSION_LIMIT: 10000
   CURRENT_DIR: "$(pwd)"
   LOG_CONFIG: logging.test.conf
+  DEV_FQDN: aurweb-$CI_COMMIT_REF_SLUG.sandbox.archlinux.page
+  INFRASTRUCTURE_REPO: https://gitlab.archlinux.org/archlinux/infrastructure.git
 
 lint:
   stage: .pre
   before_script:
-    - pacman -Sy --noconfirm --noprogressbar --cachedir .pkg-cache
+    - pacman -Sy --noconfirm --noprogressbar
       archlinux-keyring
-    - pacman -Syu --noconfirm --noprogressbar --cachedir .pkg-cache
+    - pacman -Syu --noconfirm --noprogressbar
       git python python-pre-commit
   script:
-    # https://github.com/pre-commit/pre-commit/issues/2178#issuecomment-1002163763
-    - export SETUPTOOLS_USE_DISTUTILS=stdlib
     - export XDG_CACHE_HOME=.pre-commit
     - pre-commit run -a
 
 test:
   stage: test
-  tags:
-    - fast-single-thread
   before_script:
     - export PATH="$HOME/.poetry/bin:${PATH}"
     - ./docker/scripts/install-deps.sh
@@ -51,44 +49,113 @@ test:
     # Run sharness.
     - make -C test sh
     # Run pytest.
-    - pytest
+    - pytest --junitxml="pytest-report.xml"
     - make -C test coverage # Produce coverage reports.
-  coverage: '/TOTAL.*\s+(\d+\%)/'
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
     reports:
+      junit: pytest-report.xml
       coverage_report:
         coverage_format: cobertura
         path: coverage.xml
 
-deploy:
+.init_tf: &init_tf
+  - pacman -Syu --needed --noconfirm terraform
+  - export TF_VAR_name="aurweb-${CI_COMMIT_REF_SLUG}"
+  - TF_ADDRESS="${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"
+  - cd ci/tf
+  - >
+    terraform init \
+      -backend-config="address=${TF_ADDRESS}" \
+      -backend-config="lock_address=${TF_ADDRESS}/lock" \
+      -backend-config="unlock_address=${TF_ADDRESS}/lock" \
+      -backend-config="username=x-access-token" \
+      -backend-config="password=${TF_STATE_GITLAB_ACCESS_TOKEN}" \
+      -backend-config="lock_method=POST" \
+      -backend-config="unlock_method=DELETE" \
+      -backend-config="retry_wait_min=5"
+
+deploy_review:
   stage: deploy
-  tags:
-    - secure
+  script:
+    - *init_tf
+    - terraform apply -auto-approve
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    url: https://$DEV_FQDN
+    on_stop: stop_review
+    auto_stop_in: 1 week
   rules:
-    - if: $CI_COMMIT_BRANCH == "pu"
+    - if: $CI_COMMIT_REF_NAME =~ /^renovate\//
+      when: never
+    - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb"
       when: manual
-  variables:
-    FASTAPI_BACKEND: gunicorn
-    FASTAPI_WORKERS: 5
-    AURWEB_PHP_PREFIX: https://aur-dev.archlinux.org
-    AURWEB_FASTAPI_PREFIX: https://aur-dev.archlinux.org
-    AURWEB_SSHD_PREFIX: ssh://aur@aur-dev.archlinux.org:2222
-    COMMIT_HASH: $CI_COMMIT_SHA
-    GIT_DATA_DIR: git_data
+
+provision_review:
+  stage: deploy
+  needs:
+    - deploy_review
   script:
-    - pacman -Syu --noconfirm docker docker-compose socat openssh
-    - chmod 600 ${SSH_KEY}
-    - socat "UNIX-LISTEN:/tmp/docker.sock,reuseaddr,fork" EXEC:"ssh -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -Ti ${SSH_KEY} ${SSH_USER}@${SSH_HOST}" &
-    - export DOCKER_HOST="unix:///tmp/docker.sock"
-    # Set secure login config for aurweb.
-    - sed -ri "s/^(disable_http_login).*$/\1 = 1/" conf/config.dev
-    - docker-compose build
-    - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml down --remove-orphans
-    - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml up -d
-    - docker image prune -f
-    - docker container prune -f
-    - docker volume prune -f
+    - *init_tf
+    - pacman -Syu --noconfirm --needed ansible git openssh jq
+      # Get ssh key from terraform state file
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - terraform show -json |
+      jq -r '.values.root_module.resources[] |
+      select(.address == "tls_private_key.this") |
+      .values.private_key_openssh' > ~/.ssh/id_ed25519
+    - chmod 400 ~/.ssh/id_ed25519
+      # Clone infra repo
+    - git clone $INFRASTRUCTURE_REPO
+    - cd infrastructure
+      # Remove vault files
+    - rm $(git grep -l 'ANSIBLE_VAULT;1.1;AES256$')
+      # Remove vault config
+    - sed -i '/^vault/d' ansible.cfg
+      # Add host config
+    - mkdir -p host_vars/$DEV_FQDN
+    - 'echo "filesystem: btrfs" > host_vars/$DEV_FQDN/misc'
+      # Add host
+    - echo "$DEV_FQDN" > hosts
+      # Add our pubkey and hostkeys
+    - ssh-keyscan $DEV_FQDN >> ~/.ssh/known_hosts
+    - ssh-keygen -f ~/.ssh/id_ed25519 -y > pubkeys/aurweb-dev.pub
+      # Run our ansible playbook
+    - >
+      ansible-playbook playbooks/aur-dev.archlinux.org.yml \
+        -e "aurdev_fqdn=$DEV_FQDN" \
+        -e "aurweb_repository=$CI_REPOSITORY_URL" \
+        -e "aurweb_version=$CI_COMMIT_SHA" \
+        -e "{\"vault_mariadb_users\":{\"root\":\"aur\"}}" \
+        -e "vault_aurweb_db_password=aur" \
+        -e "vault_aurweb_gitlab_instance=https://does.not.exist" \
+        -e "vault_aurweb_error_project=set-me" \
+        -e "vault_aurweb_error_token=set-me" \
+        -e "vault_aurweb_secret=aur" \
+        -e "vault_goaurrpc_metrics_token=aur" \
+        -e '{"root_additional_keys": ["moson.pub", "aurweb-dev.pub"]}'
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    action: access
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^renovate\//
+      when: never
+    - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb"
 
+stop_review:
+  stage: deploy
+  needs:
+    - deploy_review
+  script:
+    - *init_tf
+    - terraform destroy -auto-approve
+    - 'curl --silent --show-error --fail --header "Private-Token: ${TF_STATE_GITLAB_ACCESS_TOKEN}" --request DELETE "${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"'
   environment:
-    name: development
-    url: https://aur-dev.archlinux.org
+    name: review/$CI_COMMIT_REF_NAME
+    action: stop
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^renovate\//
+      when: never
+    - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb"
+      when: manual

.gitlab/issue_templates/Account Request.md

-## Checklist
-
-- [ ] I have set a Username in the Details section
-- [ ] I have set an Email in the Details section
-- [ ] I have set a valid Account Type in the Details section
-
-## Details
-
-- Instance: aur-dev.archlinux.org
-- Username: the_username_you_want
-- Email: valid@email.org
-- Account Type: (User|Trusted User)
-
-/label account-request

.gitlab/issue_templates/Bug.md

+<!--
+This template is used to report potential bugs with the AURweb website.
+
+NOTE: All comment sections with a MODIFY note need to be edited. All checkboxes
+in the "Checklist" section need to be checked by the owner of the issue.
+-->
+/label ~bug ~unconfirmed
+/title [BUG] <!-- MODIFY: add subject -->
+<!--
+Please do not remove the above quick actions, which automatically label the
+issue and assign relevant users.
+-->
+
 ### Checklist
 
-This bug template is meant to provide bug issues for code existing in
-the aurweb repository. This bug template is **not meant** to handle
-bugs with user-uploaded packages.
+**NOTE:** This bug template is meant to provide bug issues for code existing in
+the aurweb repository.
 
-To work out a bug you have found in a user-uploaded package, contact
-the package's maintainer first. If you receive no response, file the
-relevant package request against it so TUs can deal with cleanup.
+**This bug template is not meant to handle bugs with user-uploaded packages.**
+To report issues you might have found in a user-uploaded package, contact
+the package's maintainer in comments.
 
 - [ ] I confirm that this is an issue with aurweb's code and not a
       user-uploaded package.
@@ -29,7 +41,7 @@ this bug.
 
 ### Logs
 
-If you have any logs relevent to the bug, include them here in
+If you have any logs relevant to the bug, include them here in
 quoted or code blocks.
 
 ### Version(s)

.gitlab/issue_templates/Feature.md

+<!--
+This template is used to feature request for AURweb website.
+
+NOTE: All comment sections with a MODIFY note need to be edited. All checkboxes
+in the "Checklist" section need to be checked by the owner of the issue.
+-->
+/label ~feature ~unconfirmed
+/title [FEATURE] <!-- MODIFY: add subject -->
+<!--
+Please do not remove the above quick actions, which automatically label the
+issue and assign relevant users.
+-->
+
+### Checklist
+
+**NOTE:** This bug template is meant to provide bug issues for code existing in
+the aurweb repository.
+
+**This bug template is not meant to handle bugs with user-uploaded packages.**
+To report issues you might have found in a user-uploaded package, contact
+the package's maintainer in comments.
+
 - [ ] I have summed up the feature in concise words in the [Summary](#summary) section.
 - [ ] I have completely described the feature in the [Description](#description) section.
 - [ ] I have completed the [Blockers](#blockers) section.
@@ -28,5 +50,3 @@ Example:
 
 - [Feature] Do not allow users to be Tyrants
     - \<(issue|merge_request)_link\>
-
-/label feature unconsidered

.gitlab/issue_templates/Feedback.md

-**NOTE:** This issue template is only applicable to FastAPI implementations
-in the code-base, which only exists within the `pu` branch. If you wish to
-file an issue for the current PHP implementation of aurweb, please file a
-standard issue prefixed with `[Bug]` or `[Feature]`.
-
-
-**Checklist**
-
-- [ ] I have prefixed the issue title with `[Feedback]` along with a message
-      pointing to the route or feature tested.
-    - Example: `[Feedback] /packages/{name}`
-- [ ] I have completed the [Changes](#changes) section.
-- [ ] I have completed the [Bugs](#bugs) section.
-- [ ] I have completed the [Improvements](#improvements) section.
-- [ ] I have completed the [Summary](#summary) section.
-
-### Changes
-
-Please describe changes in user experience when compared to the PHP
-implementation. This section can actually hold a lot of info if you
-are up for it -- changes in routes, HTML rendering, back-end behavior,
-etc.
-
-If you cannot see any changes from your standpoint, include a short
-statement about that fact.
-
-### Bugs
-
-Please describe any bugs you've experienced while testing the route
-pertaining to this issue. A "perfect" bug report would include your
-specific experience, what you expected to occur, and what happened
-otherwise. If you can, please include output of `docker-compose logs fastapi`
-with your report; especially if any unintended exceptions occurred.
-
-### Improvements
-
-If you've experienced improvements in the route when compared to PHP,
-please do include those here. We'd like to know if users are noticing
-these improvements and how they feel about them.
-
-There are multiple routes with no improvements. For these, just include
-a short sentence about the fact that you've experienced none.
-
-### Summary
-
-First: If you've gotten here and completed the [Changes](#changes),
-[Bugs](#bugs), and [Improvements](#improvements) sections, we'd like
-to thank you very much for your contribution and willingness to test.
-We are not a company, and we are not a large team; any bit of assistance
-here helps the project astronomically and moves us closer toward a
-new release.
-
-That being said: please include an overall summary of your experience
-and how you felt about the current implementation which you're testing
-in comparison with PHP (current aur.archlinux.org, or https://localhost:8443
-through docker).
-
-/label feedback

.pre-commit-config.yaml

 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -8,10 +8,11 @@ repos:
       - id: check-toml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+        exclude: ^po/
       - id: debug-statements
 
   - repo: https://github.com/myint/autoflake
-    rev: v1.7.7
+    rev: v2.3.1
     hooks:
       - id: autoflake
         args:
@@ -20,16 +21,16 @@ repos:
           - --ignore-init-module-imports
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.13.2
     hooks:
       - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 24.4.1
     hooks:
       - id: black
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 7.0.0
     hooks:
       - id: flake8

.tx/config

 [main]
-host = https://www.transifex.com
+host = https://app.transifex.com
 
 [o:lfleischer:p:aurweb:r:aurwebpot]
 file_filter = po/<lang>.po

CONTRIBUTING.md

@@ -91,13 +91,14 @@ browser if desired.
 Accessible services (on the host):
 
 - https://localhost:8444 (python via nginx)
-- https://localhost:8443 (php via nginx)
 - localhost:13306 (mariadb)
 - localhost:16379 (redis)
 
 Docker services, by default, are setup to be hot reloaded when source code
 is changed.
 
+For detailed setup instructions have a look at [TESTING](TESTING)
+
 #### Using INSTALL
 
 The [INSTALL](INSTALL) file describes steps to install the application on

INSTALL

@@ -14,8 +14,7 @@ read the instructions below.
     $ cd aurweb
     $ poetry install
 
-2) Setup a web server with PHP and MySQL. Configure the web server to redirect
-   all URLs to /index.php/foo/bar/. The following block can be used with nginx:
+2) Setup a web server with MySQL. The following block can be used with nginx:
 
     server {
         # https is preferred and can be done easily with LetsEncrypt
@@ -31,14 +30,6 @@ read the instructions below.
         ssl_certificate /etc/ssl/certs/aur.cert.pem;
         ssl_certificate_key /etc/ssl/private/aur.key.pem;
 
-        # Asset root. This is used to match against gzip archives.
-        root   /srv/http/aurweb/web/html;
-
-        # TU Bylaws redirect.
-        location = /trusted-user/TUbylaws.html {
-            return 301 https://tu-bylaws.aur.archlinux.org;
-        }
-
         # smartgit location.
         location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" {
             include      uwsgi_params;
@@ -63,6 +54,9 @@ read the instructions below.
 
         # Static archive assets.
         location ~ \.gz$ {
+            # Asset root. This is used to match against gzip archives.
+            root   /srv/http/aurweb/archives;
+
             types { application/gzip text/plain }
             default_type text/plain;
             add_header Content-Encoding gzip;
@@ -126,7 +120,7 @@ interval:
     */2 * * * * bash -c 'poetry run aurweb-pkgmaint'
     */2 * * * * bash -c 'poetry run aurweb-usermaint'
     */2 * * * * bash -c 'poetry run aurweb-popupdate'
-    */12 * * * * bash -c 'poetry run aurweb-tuvotereminder'
+    */12 * * * * bash -c 'poetry run aurweb-votereminder'
 
 7) Create a new database and a user and import the aurweb SQL schema:
 

README.md

@@ -11,8 +11,8 @@ The aurweb project includes
 * A web interface to search for packaging scripts and display package details.
 * An SSH/Git interface to submit and update packages and package meta data.
 * Community features such as comments, votes, package flagging and requests.
-* Editing/deletion of packages and accounts by Trusted Users and Developers.
-* Area for Trusted Users to post AUR-related proposals and vote on them.
+* Editing/deletion of packages and accounts by Package Maintainers and Developers.
+* Area for Package Maintainers to post AUR-related proposals and vote on them.
 
 Directory Layout
 ----------------
@@ -26,7 +26,6 @@ Directory Layout
 * `schema`: schema for the SQL database
 * `test`: test suite and test cases
 * `upgrading`: instructions for upgrading setups from one release to another
-* `web`: PHP-based web interface for the AUR
 
 Documentation
 -------------
@@ -51,13 +50,13 @@ Links
 
 * Questions, comments, and patches related to aurweb can be sent to the AUR
   development mailing list: aur-dev@archlinux.org -- mailing list archives:
-  https://mailman.archlinux.org/mailman/listinfo/aur-dev
+  https://lists.archlinux.org/archives/list/aur-dev@lists.archlinux.org/
 
 Translations
 ------------
 
 Translations are welcome via our Transifex project at
-https://www.transifex.com/lfleischer/aurweb; see `doc/i18n.txt` for details.
+https://www.transifex.com/lfleischer/aurweb; see [doc/i18n.md](./doc/i18n.md) for details.
 
 ![Transifex](https://www.transifex.com/projects/p/aurweb/chart/image_png)
 

TESTING

 Setup Testing Environment
 =========================
 
-Note that this setup is only to test the web interface. If you need to have a
-full aurweb instance with cgit, ssh interface, etc, follow the directions in
-INSTALL.
+The quickest way to get you hacking on aurweb is to utilize docker.
+In case you prefer to run it bare-metal see instructions further below.
 
-docker-compose
---------------
+Containerized environment
+-------------------------
 
 1) Clone the aurweb project:
 
     $ git clone https://gitlab.archlinux.org/archlinux/aurweb.git
+    $ cd aurweb
 
 2) Install the necessary packages:
 
-    # pacman -S docker-compose
+    # pacman -S --needed docker docker-compose
 
-2) Build the aurweb:latest image:
+3) Build the aurweb:latest image:
 
-    $ cd /path/to/aurweb/
-    $ docker-compose build
+    # systemctl start docker
+    # docker compose build
 
-3) Run local Docker development instance:
+4) Run local Docker development instance:
 
-    $ cd /path/to/aurweb/
-    $ docker-compose up -d nginx
+    # docker compose up -d
 
-4) Browse to local aurweb development server.
+5) Browse to local aurweb development server.
 
-   Python: https://localhost:8444/
-   PHP: https://localhost:8443/
+   https://localhost:8444/
 
-5) [Optionally] populate the database with dummy data:
+6) [Optionally] populate the database with dummy data:
 
-    $ docker-compose up mariadb
-    $ docker-compose exec mariadb  /bin/sh
+    # docker compose exec mariadb /bin/bash
     # pacman -S --noconfirm words fortune-mod
     # poetry run schema/gendummydata.py dummy_data.sql
-    # mysql -uaur -paur aurweb < dummy_data.sql
+    # mariadb -uaur -paur aurweb < dummy_data.sql
+    # exit
+
+    Inspect `dummy_data.sql` for test credentials.
+    Passwords match usernames.
+
+We now have fully set up environment which we can start and stop with:
+
+    # docker compose start
+    # docker compose stop
+
+Proceed with topic "Setup for running tests"
 
-Inspect `dummy_data.sql` for test credentials. Passwords match usernames.
 
-Bare Metal
-----------
+Bare Metal installation
+-----------------------
+
+Note that this setup is only to test the web interface. If you need to have a
+full aurweb instance with cgit, ssh interface, etc, follow the directions in
+INSTALL.
 
 1) Clone the aurweb project:
 
     $ git clone git://git.archlinux.org/aurweb.git
+    $ cd aurweb
 
 2) Install the necessary packages:
 
-    # pacman -S python-poetry
+    # pacman -S --needed python-poetry mariadb words fortune-mod nginx
 
-4) Install the package/dependencies via `poetry`:
+3) Install the package/dependencies via `poetry`:
 
-    $ cd /path/to/aurweb/
     $ poetry install
 
-3) Copy conf/config.dev to conf/config and replace YOUR_AUR_ROOT by the absolute
+4) Copy conf/config.dev to conf/config and replace YOUR_AUR_ROOT by the absolute
    path to the root of your aurweb clone. sed can do both tasks for you:
 
     $ sed -e "s;YOUR_AUR_ROOT;$PWD;g" conf/config.dev > conf/config
@@ -65,23 +76,110 @@ Bare Metal
    Note that when the upstream config.dev is updated, you should compare it to
    your conf/config, or regenerate your configuration with the command above.
 
-4) Prepare a database:
+5) Set up mariadb:
 
-    $ cd /path/to/aurweb/
+    # mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
+    # systemctl start mariadb
+    # mariadb -u root
+    > CREATE USER 'aur'@'localhost' IDENTIFIED BY 'aur';
+    > GRANT ALL ON *.* TO 'aur'@'localhost' WITH GRANT OPTION;
+    > CREATE DATABASE aurweb;
+    > exit
 
-    $ AUR_CONFIG=conf/config poetry run python -m aurweb.initdb
+6) Prepare a database and insert dummy data:
 
+    $ AUR_CONFIG=conf/config poetry run python -m aurweb.initdb
     $ poetry run schema/gendummydata.py dummy_data.sql
-    $ mysql -uaur -paur aurweb < dummy_data.sql
+    $ mariadb -uaur -paur aurweb < dummy_data.sql
 
-5) Run the test server:
+7) Run the test server:
 
    ## set AUR_CONFIG to our locally created config
-   $ export AUR_CONFIG=conf/config
+    $ export AUR_CONFIG=conf/config
 
    ## with aurweb.spawn
-   $ poetry run python -m aurweb.spawn
+    $ poetry run python -m aurweb.spawn
 
    ## with systemd service
-   $ sudo install -m644 examples/aurweb.service /etc/systemd/system/
-   $ systemctl enable --now aurweb.service
+    $ sudo install -m644 examples/aurweb.service /etc/systemd/system/
+    # systemctl enable --now aurweb.service
+
+
+Setup for running tests
+-----------------------
+
+If you've set up a docker environment, you can run the full test-suite with:
+    # docker compose run test
+
+You can collect code-coverage data with:
+    $ ./util/fix-coverage data/.coverage
+
+See information further below on how to visualize the data.
+
+For running individual tests, we need to perform a couple of additional steps.
+In case you did the bare-metal install, steps 2, 3, 4 and 5 should be skipped.
+
+1) Install the necessary packages:
+
+    # pacman -S --needed python-poetry mariadb-libs asciidoc openssh
+
+2) Install the package/dependencies via `poetry`:
+
+    $ poetry install
+
+3) Copy conf/config.dev to conf/config and replace YOUR_AUR_ROOT by the absolute
+   path to the root of your aurweb clone. sed can do both tasks for you:
+
+    $ sed -e "s;YOUR_AUR_ROOT;$PWD;g" conf/config.dev > conf/config
+
+   Note that when the upstream config.dev is updated, you should compare it to
+   your conf/config, or regenerate your configuration with the command above.
+
+4) Edit the config file conf/config and change the mysql/mariadb portion
+
+    We can make use of our mariadb docker container instead of having to install
+    mariadb. Change the config as follows:
+
+    ---------------------------------------------------------------------
+    ; MySQL database information. User defaults to root for containerized
+    ; testing with mysqldb. This should be set to a non-root user.
+    user = root
+    password = aur
+    host = 127.0.0.1
+    port = 13306
+    ;socket = /var/run/mysqld/mysqld.sock
+    ---------------------------------------------------------------------
+
+5) Start our mariadb docker container
+
+    # docker compose start mariadb
+
+6) Set environment variables
+
+    $ export AUR_CONFIG=conf/config
+    $ export LOG_CONFIG=logging.test.conf
+
+7) Compile translation & doc files
+
+    $ make -C po install
+    $ make -C doc
+
+Now we can run our python test-suite or individual tests with:
+
+    $ poetry run pytest test/
+    $ poetry run pytest test/test_whatever.py
+
+To run Sharness tests:
+
+    $ poetry run make -C test sh
+
+The e-Mails that have been generated can be found at test-emails/
+
+After test runs, code-coverage reports can be created with:
+    ## CLI report
+    $ coverage report
+
+    ## HTML version stored at htmlcov/
+    $ coverage html
+
+More information about tests can be found at test/README.md

aurweb/archives/spec/pkgbases.py

@@ -15,13 +15,7 @@ class Spec(SpecBase):
         self.pkgbases_repo = GitInfo(config.get("git-archive", "pkgbases-repo"))
 
     def generate(self) -> Iterable[SpecOutput]:
-        filt = PackageBase.PackagerUID.isnot(None)
-        query = (
-            db.query(PackageBase.Name)
-            .filter(filt)
-            .order_by(PackageBase.Name.asc())
-            .all()
-        )
+        query = db.query(PackageBase.Name).order_by(PackageBase.Name.asc()).all()
         pkgbases = [pkgbase.Name for pkgbase in query]
 
         self.add_output(

aurweb/archives/spec/pkgnames.py

@@ -15,11 +15,9 @@ class Spec(SpecBase):
         self.pkgnames_repo = GitInfo(config.get("git-archive", "pkgnames-repo"))
 
     def generate(self) -> Iterable[SpecOutput]:
-        filt = PackageBase.PackagerUID.isnot(None)
         query = (
             db.query(Package.Name)
             .join(PackageBase, PackageBase.ID == Package.PackageBaseID)
-            .filter(filt)
             .order_by(Package.Name.asc())
             .all()
         )

aurweb/asgi.py

@@ -6,6 +6,7 @@ import re
 import sys
 import traceback
 import typing
+from contextlib import asynccontextmanager
 from urllib.parse import quote_plus
 
 import requests
@@ -13,8 +14,7 @@ from fastapi import FastAPI, HTTPException, Request, Response
 from fastapi.responses import RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from jinja2 import TemplateNotFound
-from prometheus_client import multiprocess
-from sqlalchemy import and_, or_
+from sqlalchemy import and_
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.middleware.authentication import AuthenticationMiddleware
 from starlette.middleware.sessions import SessionMiddleware
@@ -22,7 +22,6 @@ from starlette.middleware.sessions import SessionMiddleware
 import aurweb.captcha  # noqa: F401
 import aurweb.config
 import aurweb.filters  # noqa: F401
-import aurweb.pkgbase.util as pkgbaseutil
 from aurweb import aur_logging, prometheus, util
 from aurweb.aur_redis import redis_connection
 from aurweb.auth import BasicAuthBackend
@@ -34,9 +33,17 @@ from aurweb.routers import APP_ROUTES
 from aurweb.templates import make_context, render_template
 
 logger = aur_logging.get_logger(__name__)
+session_secret = aurweb.config.get("fastapi", "session_secret")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    await app_startup()
+    yield
+
 
 # Setup the FastAPI app.
-app = FastAPI()
+app = FastAPI(lifespan=lifespan)
 
 # Instrument routes with the prometheus-fastapi-instrumentator
 # library with custom collectors and expose /metrics.
@@ -45,7 +52,25 @@ instrumentator().add(prometheus.http_requests_total())
 instrumentator().instrument(app)
 
 
-@app.on_event("startup")
+if aurweb.config.get("tracing", "otlp_endpoint"):
+    from opentelemetry import trace
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+    # Instrument FastAPI for tracing
+    FastAPIInstrumentor.instrument_app(app)
+
+    resource = Resource(attributes={"service.name": "aurweb"})
+    otlp_endpoint = aurweb.config.get("tracing", "otlp_endpoint")
+    otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
+    span_processor = BatchSpanProcessor(otlp_exporter)
+    trace.set_tracer_provider(TracerProvider(resource=resource))
+    trace.get_tracer_provider().add_span_processor(span_processor)
+
+
 async def app_startup():
     # https://stackoverflow.com/questions/67054759/about-the-maximum-recursion-error-in-fastapi
     # Test failures have been observed by internal starlette code when
@@ -68,7 +93,6 @@ async def app_startup():
             f"Supported backends: {str(aurweb.db.DRIVERS.keys())}"
         )
 
-    session_secret = aurweb.config.get("fastapi", "session_secret")
     if not session_secret:
         raise Exception("[fastapi] session_secret must not be empty")
 
@@ -78,15 +102,7 @@ async def app_startup():
             "endpoint is disabled."
         )
 
-    app.mount("/static/css", StaticFiles(directory="web/html/css"), name="static_css")
-    app.mount("/static/js", StaticFiles(directory="web/html/js"), name="static_js")
-    app.mount(
-        "/static/images", StaticFiles(directory="web/html/images"), name="static_images"
-    )
-
-    # Add application middlewares.
-    app.add_middleware(AuthenticationMiddleware, backend=BasicAuthBackend())
-    app.add_middleware(SessionMiddleware, secret_key=session_secret)
+    app.mount("/static", StaticFiles(directory="static"), name="static_files")
 
     # Add application routes.
     def add_router(module):
@@ -98,12 +114,6 @@ async def app_startup():
     get_engine()
 
 
-def child_exit(server, worker):  # pragma: no cover
-    """This function is required for gunicorn customization
-    of prometheus multiprocessing."""
-    multiprocess.mark_process_dead(worker.pid)
-
-
 async def internal_server_error(request: Request, exc: Exception) -> Response:
     """
     Catch all uncaught Exceptions thrown in a route.
@@ -219,10 +229,16 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> Respon
     if exc.status_code == http.HTTPStatus.NOT_FOUND:
         tokens = request.url.path.split("/")
         matches = re.match("^([a-z0-9][a-z0-9.+_-]*?)(\\.git)?$", tokens[1])
-        if matches:
+        if matches and len(tokens) == 2:
             try:
                 pkgbase = get_pkg_or_base(matches.group(1))
-                context = pkgbaseutil.make_context(request, pkgbase)
+                context["pkgbase"] = pkgbase
+                context["git_clone_uri_anon"] = aurweb.config.get(
+                    "options", "git_clone_uri_anon"
+                )
+                context["git_clone_uri_priv"] = aurweb.config.get(
+                    "options", "git_clone_uri_priv"
+                )
             except HTTPException:
                 pass
 
@@ -284,21 +300,18 @@ async def check_terms_of_service(request: Request, call_next: typing.Callable):
     """This middleware function redirects authenticated users if they
     have any outstanding Terms to agree to."""
     if request.user.is_authenticated() and request.url.path != "/tos":
-        unaccepted = (
+        accepted = (
             query(Term)
             .join(AcceptedTerm)
             .filter(
-                or_(
-                    AcceptedTerm.UsersID != request.user.ID,
-                    and_(
-                        AcceptedTerm.UsersID == request.user.ID,
-                        AcceptedTerm.TermsID == Term.ID,
-                        AcceptedTerm.Revision < Term.Revision,
-                    ),
-                )
+                and_(
+                    AcceptedTerm.UsersID == request.user.ID,
+                    AcceptedTerm.TermsID == Term.ID,
+                    AcceptedTerm.Revision >= Term.Revision,
+                ),
             )
         )
-        if query(Term).count() > unaccepted.count():
+        if query(Term).count() - accepted.count() > 0:
             return RedirectResponse("/tos", status_code=int(http.HTTPStatus.SEE_OTHER))
 
     return await util.error_or_result(call_next, request)
@@ -320,3 +333,8 @@ async def id_redirect_middleware(request: Request, call_next: typing.Callable):
         return RedirectResponse(f"{path}/{id}{qs}")
 
     return await util.error_or_result(call_next, request)
+
+
+# Add application middlewares.
+app.add_middleware(AuthenticationMiddleware, backend=BasicAuthBackend())
+app.add_middleware(SessionMiddleware, secret_key=session_secret)

aurweb/aur_redis.py

@@ -7,6 +7,11 @@ from aurweb import aur_logging
 logger = aur_logging.get_logger(__name__)
 pool = None
 
+if aurweb.config.get("tracing", "otlp_endpoint"):
+    from opentelemetry.instrumentation.redis import RedisInstrumentor
+
+    RedisInstrumentor().instrument()
+
 
 class FakeConnectionPool:
     """A fake ConnectionPool class which holds an internal reference