Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • aliu/aurweb
  • morganamilo/aurweb
  • tex/aurweb
  • abitrolly/aurweb
  • muflone/aurweb
  • anthraxx/aurweb
  • jafari/aurweb
  • levitating/aurweb
  • freso/aurweb
  • okabe/aurweb
  • rafaelff/aurweb
  • zoorat/aurweb
  • auerhuhn/aurweb
  • nils/aurweb
  • antiz/aurweb
  • henry-zhr/aurweb
  • segaja/aurweb
  • som015/aurweb
  • gromit/aurweb
  • belongingtome47/aurweb
  • moson/aurweb
  • steppaa23/aurweb
  • bittin/aurweb
  • jkhsjdhjs/aurweb
  • whynothugo/aurweb
  • matt/aurweb
  • fosskers/aurweb
  • awalgarg/aurweb
  • robertoszek/aurweb
  • ainola/aurweb
  • fluix/aurweb
  • hwittenborn/aurweb
  • jocke-l/aurweb
  • eschwartz/aurweb
  • mackilanu/aurweb
  • artafinde/aurweb
  • klausenbusk/aurweb
  • felixonmars/aurweb
  • kevr/aurweb
  • hashworks/aurweb
  • freswa/aurweb
  • lahwaacz/aurweb
  • jelle/aurweb
  • ffy00/aurweb
  • archlinux/aurweb
45 results
Show changes
Commits on Source (1337)
Showing with 989 additions and 177 deletions
[run]
disable_warnings = already-imported
[report]
include = aurweb/*
fail_under = 95
exclude_lines =
if __name__ == .__main__.:
pragma: no cover
# Config files
conf/config
conf/config.sqlite
conf/config.sqlite.defaults
conf/docker
conf/docker.defaults
# Compiled translation files
**/*.mo
# Typical virtualenv directories
env/
venv/
.venv/
# Test output
htmlcov/
test-emails/
test/__pycache__
test/test-results
test/trash_directory*
.coverage
.pytest_cache
# EditorConfig configuration for aurweb
# https://EditorConfig.org
# https://editorconfig.org
# Top-most EditorConfig file
root = true
......@@ -8,6 +8,3 @@ root = true
end_of_line = lf
insert_final_newline = true
charset = utf-8
[*.{php,t}]
indent_style = tab
FASTAPI_BACKEND="uvicorn"
FASTAPI_WORKERS=2
MARIADB_SOCKET_DIR="/var/run/mysqld/"
AURWEB_FASTAPI_PREFIX=https://localhost:8444
AURWEB_SSHD_PREFIX=ssh://aur@localhost:2222
GIT_DATA_DIR="./aur.git/"
TEST_RECURSION_LIMIT=10000
COMMIT_HASH=
# style: Run pre-commit
9c6c13b78a30cb9d800043410799e29631f803d2
/data/
__pycache__/
*.py[cod]
.vim/
.pylintrc
.coverage
.idea
/cache/*
/logs/*
/build/
/dist/
/aurweb.egg-info/
/personal/
/notes/
/vendor/
/pyrightconfig.json
/taskell.md
aur.git/
aurweb.sqlite3
conf/config
conf/config.sqlite
conf/config.sqlite.defaults
conf/docker
conf/docker.defaults
data.sql
dummy-data.sql*
fastapi_aw/
htmlcov/
po/*.mo
po/*.po~
po/POTFILES
web/locale/*/
aur.git/
__pycache__/
*.py[cod]
test/test-results/
test/trash directory*
schema/aur-schema-sqlite.sql
test/test-results/
test/trash_directory*
web/locale/*/
web/html/*.gz
# Do not stage compiled asciidoc: make -C doc
doc/rpc.html
# Ignore any user-configured .envrc files at the root.
/.envrc
# Ignore .python-version file from Pyenv
.python-version
# Ignore coverage report
coverage.xml
# Ignore pytest report
report.xml
# Ignore test emails
test-emails/
# Ignore typical virtualenv directories
env/
venv/
.venv/
# Ignore some terraform files
/ci/tf/.terraform
/ci/tf/terraform.tfstate*
image: archlinux
image: archlinux:base-devel
cache:
key: system-v1
paths:
# For some reason Gitlab CI only supports storing cache/artifacts in a path relative to the build directory
- .pkg-cache
- .venv
- .pre-commit
variables:
AUR_CONFIG: conf/config # Default MySQL config setup in before_script.
DB_HOST: localhost
TEST_RECURSION_LIMIT: 10000
CURRENT_DIR: "$(pwd)"
LOG_CONFIG: logging.test.conf
DEV_FQDN: aurweb-$CI_COMMIT_REF_SLUG.sandbox.archlinux.page
INFRASTRUCTURE_REPO: https://gitlab.archlinux.org/archlinux/infrastructure.git
before_script:
- pacman -Syu --noconfirm --noprogressbar --needed --cachedir .pkg-cache
base-devel git gpgme protobuf pyalpm python-mysql-connector
python-pygit2 python-srcinfo python-bleach python-markdown
python-sqlalchemy python-alembic python-pytest python-werkzeug
python-pytest-tap python-fastapi hypercorn nginx python-authlib
python-itsdangerous python-httpx python-orjson
lint:
stage: .pre
before_script:
- pacman -Sy --noconfirm --noprogressbar
archlinux-keyring
- pacman -Syu --noconfirm --noprogressbar
git python python-pre-commit
script:
- export XDG_CACHE_HOME=.pre-commit
- pre-commit run -a
test:
stage: test
before_script:
- export PATH="$HOME/.poetry/bin:${PATH}"
- ./docker/scripts/install-deps.sh
- virtualenv -p python3 .venv
- source .venv/bin/activate # Enable our virtualenv cache
- ./docker/scripts/install-python-deps.sh
- useradd -U -d /aurweb -c 'AUR User' aur
- ./docker/mariadb-entrypoint.sh
- (cd '/usr' && /usr/bin/mysqld_safe --datadir='/var/lib/mysql') &
- 'until : > /dev/tcp/127.0.0.1/3306; do sleep 1s; done'
- cp -v conf/config.dev conf/config
- sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config
- ./docker/test-mysql-entrypoint.sh # Create mysql AUR_CONFIG.
- make -C po all install # Compile translations.
- make -C doc # Compile asciidoc.
- make -C test clean # Cleanup coverage.
script:
# Run sharness.
- make -C test sh
# Run pytest.
- pytest --junitxml="pytest-report.xml"
- make -C test coverage # Produce coverage reports.
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
artifacts:
reports:
junit: pytest-report.xml
coverage_report:
coverage_format: cobertura
path: coverage.xml
.init_tf: &init_tf
- pacman -Syu --needed --noconfirm terraform
- export TF_VAR_name="aurweb-${CI_COMMIT_REF_SLUG}"
- TF_ADDRESS="${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"
- cd ci/tf
- >
terraform init \
-backend-config="address=${TF_ADDRESS}" \
-backend-config="lock_address=${TF_ADDRESS}/lock" \
-backend-config="unlock_address=${TF_ADDRESS}/lock" \
-backend-config="username=x-access-token" \
-backend-config="password=${TF_STATE_GITLAB_ACCESS_TOKEN}" \
-backend-config="lock_method=POST" \
-backend-config="unlock_method=DELETE" \
-backend-config="retry_wait_min=5"
deploy_review:
stage: deploy
script:
- *init_tf
- terraform apply -auto-approve
environment:
name: review/$CI_COMMIT_REF_NAME
url: https://$DEV_FQDN
on_stop: stop_review
auto_stop_in: 1 week
rules:
- if: $CI_COMMIT_REF_NAME =~ /^renovate\//
when: never
- if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb"
when: manual
provision_review:
stage: deploy
needs:
- deploy_review
script:
- *init_tf
- pacman -Syu --noconfirm --needed ansible git openssh jq
# Get ssh key from terraform state file
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- terraform show -json |
jq -r '.values.root_module.resources[] |
select(.address == "tls_private_key.this") |
.values.private_key_openssh' > ~/.ssh/id_ed25519
- chmod 400 ~/.ssh/id_ed25519
# Clone infra repo
- git clone $INFRASTRUCTURE_REPO
- cd infrastructure
# Remove vault files
- rm $(git grep -l 'ANSIBLE_VAULT;1.1;AES256$')
# Remove vault config
- sed -i '/^vault/d' ansible.cfg
# Add host config
- mkdir -p host_vars/$DEV_FQDN
- 'echo "filesystem: btrfs" > host_vars/$DEV_FQDN/misc'
# Add host
- echo "$DEV_FQDN" > hosts
# Add our pubkey and hostkeys
- ssh-keyscan $DEV_FQDN >> ~/.ssh/known_hosts
- ssh-keygen -f ~/.ssh/id_ed25519 -y > pubkeys/aurweb-dev.pub
# Run our ansible playbook
- >
ansible-playbook playbooks/aur-dev.archlinux.org.yml \
-e "aurdev_fqdn=$DEV_FQDN" \
-e "aurweb_repository=$CI_REPOSITORY_URL" \
-e "aurweb_version=$CI_COMMIT_SHA" \
-e "{\"vault_mariadb_users\":{\"root\":\"aur\"}}" \
-e "vault_aurweb_db_password=aur" \
-e "vault_aurweb_gitlab_instance=https://does.not.exist" \
-e "vault_aurweb_error_project=set-me" \
-e "vault_aurweb_error_token=set-me" \
-e "vault_aurweb_secret=aur" \
-e "vault_goaurrpc_metrics_token=aur" \
-e '{"root_additional_keys": ["moson.pub", "aurweb-dev.pub"]}'
environment:
name: review/$CI_COMMIT_REF_NAME
action: access
rules:
- if: $CI_COMMIT_REF_NAME =~ /^renovate\//
when: never
- if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb"
stop_review:
stage: deploy
needs:
- deploy_review
script:
- make -C test
- *init_tf
- terraform destroy -auto-approve
- 'curl --silent --show-error --fail --header "Private-Token: ${TF_STATE_GITLAB_ACCESS_TOKEN}" --request DELETE "${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"'
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
rules:
- if: $CI_COMMIT_REF_NAME =~ /^renovate\//
when: never
- if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb"
when: manual
## Checklist
- [ ] I have set a Username in the Details section
- [ ] I have set an Email in the Details section
- [ ] I have set a valid Account Type in the Details section
## Details
- Instance: aur-dev.archlinux.org
- Username: the_username_you_want
- Email: valid@email.org
- Account Type: (User|Trusted User)
<!--
This template is used to report potential bugs with the AURweb website.
NOTE: All comment sections with a MODIFY note need to be edited. All checkboxes
in the "Checklist" section need to be checked by the owner of the issue.
-->
/label ~bug ~unconfirmed
/title [BUG] <!-- MODIFY: add subject -->
<!--
Please do not remove the above quick actions, which automatically label the
issue and assign relevant users.
-->
### Checklist
**NOTE:** This bug template is meant to provide bug issues for code existing in
the aurweb repository.
**This bug template is not meant to handle bugs with user-uploaded packages.**
To report issues you might have found in a user-uploaded package, contact
the package's maintainer in comments.
- [ ] I confirm that this is an issue with aurweb's code and not a
user-uploaded package.
- [ ] I have described the bug in complete detail in the
[Description](#description) section.
- [ ] I have specified steps in the [Reproduction](#reproduction) section.
......@@ -17,7 +41,7 @@ this bug.
### Logs
If you have any logs relevent to the bug, include them here in
If you have any logs relevant to the bug, include them here in
quoted or code blocks.
### Version(s)
......@@ -32,3 +56,5 @@ aurweb's HTML render output. If you're testing locally, use the
commit on which you are experiencing the bug. If you have found
a bug which exists on live aur.archlinux.org, include the version
located at the bottom of the webpage.
/label bug unconfirmed
<!--
This template is used to feature request for AURweb website.
NOTE: All comment sections with a MODIFY note need to be edited. All checkboxes
in the "Checklist" section need to be checked by the owner of the issue.
-->
/label ~feature ~unconfirmed
/title [FEATURE] <!-- MODIFY: add subject -->
<!--
Please do not remove the above quick actions, which automatically label the
issue and assign relevant users.
-->
### Checklist
**NOTE:** This bug template is meant to provide bug issues for code existing in
the aurweb repository.
**This bug template is not meant to handle bugs with user-uploaded packages.**
To report issues you might have found in a user-uploaded package, contact
the package's maintainer in comments.
- [ ] I have summed up the feature in concise words in the [Summary](#summary) section.
- [ ] I have completely described the feature in the [Description](#description) section.
- [ ] I have completed the [Blockers](#blockers) section.
......
**NOTE:** This issue template is only applicable to FastAPI implementations
in the code-base, which only exists within the `pu` branch. If you wish to
file an issue for the current PHP implementation of aurweb, please file a
standard issue prefixed with `[Bug]` or `[Feature]`.
**Checklist**
- [ ] I have prefixed the issue title with `[Feedback]` along with a message
pointing to the route or feature tested.
- Example: `[Feedback] /packages/{name}`
- [ ] I have completed the [Changes](#changes) section.
- [ ] I have completed the [Bugs](#bugs) section.
- [ ] I have completed the [Improvements](#improvements) section.
- [ ] I have completed the [Summary](#summary) section.
### Changes
Please describe changes in user experience when compared to the PHP
implementation. This section can actually hold a lot of info if you
are up for it -- changes in routes, HTML rendering, back-end behavior,
etc.
If you cannot see any changes from your standpoint, include a short
statement about that fact.
### Bugs
Please describe any bugs you've experienced while testing the route
pertaining to this issue. A "perfect" bug report would include your
specific experience, what you expected to occur, and what happened
otherwise. If you can, please include output of `docker-compose logs fastapi`
with your report; especially if any unintended exceptions occurred.
### Improvements
If you've experienced improvements in the route when compared to PHP,
please do include those here. We'd like to know if users are noticing
these improvements and how they feel about them.
There are multiple routes with no improvements. For these, just include
a short sentence about the fact that you've experienced none.
### Summary
First: If you've gotten here and completed the [Changes](#changes),
[Bugs](#bugs), and [Improvements](#improvements) sections, we'd like
to thank you very much for your contribution and willingness to test.
We are not a company, and we are not a large team; any bit of assistance
here helps the project astronomically and moves us closer toward a
new release.
That being said: please include an overall summary of your experience
and how you felt about the current implementation which you're testing
in comparison with PHP (current aur.archlinux.org, or https://localhost:8443
through docker).
hooks:
- &base
language: python
types: [python]
require_serial: true
exclude: ^migrations/versions
- &flake8
id: flake8
name: flake8
entry: flake8
<<: *base
- &isort
id: isort
name: isort
entry: isort
<<: *base
repos:
- repo: local
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: check-added-large-files
- id: check-case-conflict
- id: check-merge-conflict
- id: check-toml
- id: end-of-file-fixer
- id: trailing-whitespace
exclude: ^po/
- id: debug-statements
- repo: https://github.com/myint/autoflake
rev: v2.3.1
hooks:
- id: autoflake
args:
- --in-place
- --remove-all-unused-imports
- --ignore-init-module-imports
- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
- id: isort
- repo: https://github.com/psf/black
rev: 24.4.1
hooks:
- <<: *flake8
- <<: *isort
args: ['--check-only', '--diff']
- id: black
- repo: https://github.com/PyCQA/flake8
rev: 7.0.0
hooks:
- id: flake8
[main]
host = https://www.transifex.com
host = https://app.transifex.com
[aurweb.aurwebpot]
[o:lfleischer:p:aurweb:r:aurwebpot]
file_filter = po/<lang>.po
source_file = po/aurweb.pot
source_lang = en
# Contributing
Patches should be sent to the [aur-dev@lists.archlinux.org][1] mailing list.
Patches should be sent to the [aur-dev@lists.archlinux.org][1] mailing list
or included in a merge request on the [aurweb repository][2].
Before sending patches, you are recommended to run `flake8` and `isort`.
You can add a git hook to do this by installing `python-pre-commit` and running
`pre-commit install`.
[1] https://lists.archlinux.org/listinfo/aur-dev
[1]: https://lists.archlinux.org/mailman3/lists/aur-dev.lists.archlinux.org/
[2]: https://gitlab.archlinux.org/archlinux/aurweb
### Coding Guidelines
DISCLAIMER: We realise the code doesn't necessarily follow all the rules.
This is an attempt to establish a standard coding style for future
development.
1. All source modified or added within a patchset **must** maintain equivalent
or increased coverage by providing tests that use the functionality
2. Please keep your source within an 80 column width
3. Use four space indentation
4. Use [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/)
5. DRY: Don't Repeat Yourself
6. All code should be tested for good _and_ bad cases (see [test/README.md][3])
[3]: https://gitlab.archlinux.org/archlinux/aurweb/-/blob/master/test/README.md
Test patches that increase coverage in the codebase are always welcome.
### Coding Style
We use `autoflake`, `isort`, `black` and `flake8` to enforce coding style in a
PEP-8 compliant way. These tools run in GitLab CI using `pre-commit` to verify
that any pushed code changes comply with this.
To enable the `pre-commit` git hook, install the `pre-commit` package either
with `pacman` or `pip` and then run `pre-commit install --install-hooks`. This
will ensure formatting is done before any code is commited to the git
repository.
There are plugins for editors or IDEs which automate this process. Some
example plugins:
- [tenfyzhong/autoflake.vim](https://github.com/tenfyzhong/autoflake.vim)
- [fisadev/vim-isort](https://github.com/fisadev/vim-isort)
- [psf/black](https://github.com/psf/black)
- [nvie/vim-flake8](https://github.com/nvie/vim-flake8)
- [prabirshrestha/vim-lsp](https://github.com/prabirshrestha/vim-lsp)
- [dense-analysis/ale](https://github.com/dense-analysis/ale)
See `setup.cfg`, `pyproject.toml` and `.pre-commit-config.yaml` for tool
specific configurations.
### Development Environment
To get started with local development, an instance of aurweb must be
brought up. This can be done using the following sections:
- [Using Docker](#using-docker)
- [Using INSTALL](#using-install)
There are a number of services aurweb employs to run the application
in its entirety:
- ssh
- cron jobs
- starlette/fastapi asgi server
Project structure:
- `./aurweb`: `aurweb` Python package
- `./templates`: Jinja2 templates
- `./docker`: Docker scripts and configuration files
#### Using Docker
Using Docker, we can run the entire infrastructure in two steps:
# Build the aurweb:latest image
$ docker-compose build
# Start all services in the background
$ docker-compose up -d nginx
`docker-compose` services will generate a locally signed root certificate
at `./data/root_ca.crt`. Users can import this into ca-certificates or their
browser if desired.
Accessible services (on the host):
- https://localhost:8444 (python via nginx)
- localhost:13306 (mariadb)
- localhost:16379 (redis)
Docker services, by default, are setup to be hot reloaded when source code
is changed.
For detailed setup instructions have a look at [TESTING](TESTING)
#### Using INSTALL
The [INSTALL](INSTALL) file describes steps to install the application on
bare-metal systems.
FROM archlinux:base-devel
VOLUME /root/.cache/pypoetry/cache
VOLUME /root/.cache/pypoetry/artifacts
VOLUME /root/.cache/pre-commit
ENV PATH="/root/.poetry/bin:${PATH}"
ENV PYTHONPATH=/aurweb
ENV AUR_CONFIG=conf/config
ENV COMPOSE=1
# Install system-wide dependencies.
COPY ./docker/scripts/install-deps.sh /install-deps.sh
RUN /install-deps.sh
# Copy Docker scripts
COPY ./docker /docker
COPY ./docker/scripts/* /usr/local/bin/
# Copy over all aurweb files.
COPY . /aurweb
# Working directory is aurweb root @ /aurweb.
WORKDIR /aurweb
# Copy initial config to conf/config.
RUN cp -vf conf/config.dev conf/config
RUN sed -i "s;YOUR_AUR_ROOT;/aurweb;g" conf/config
# Install Python dependencies.
RUN /docker/scripts/install-python-deps.sh compose
# Compile asciidocs.
RUN make -C doc
# Add our aur user.
RUN useradd -U -d /aurweb -c 'AUR User' aur
# Setup some default system stuff.
RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime
# Install translations.
RUN make -C po all install
# Install pre-commit repositories and run lint check.
RUN pre-commit run -a
......@@ -4,64 +4,129 @@ Setup on Arch Linux
For testing aurweb patches before submission, you can use the instructions in
TESTING for testing the web interface only.
Note that you can only do limited testing using the PHP built-in web server.
In particular, the cgit interface will be unusable as well as the ssh+git
interface. For a detailed description on how to setup a full aurweb server,
For a detailed description on how to setup a full aurweb server,
read the instructions below.
1) Clone the aurweb project:
1) Clone the aurweb project and install it (via `python-poetry`):
$ cd /srv/http/
$ git clone https://gitlab.archlinux.org/archlinux/aurweb.git
$ cd /srv/http/
$ git clone git://git.archlinux.org/aurweb.git
$ cd aurweb
$ poetry install
2) Setup a web server with PHP and MySQL. Configure the web server to redirect
all URLs to /index.php/foo/bar/. The following block can be used with nginx:
2) Setup a web server with MySQL. The following block can be used with nginx:
server {
listen 80;
# https is preferred and can be done easily with LetsEncrypt
# or self-CA signing. Users can still listen over 80 for plain
# http, for which the [options] disable_http_login used to toggle
# the authentication feature.
listen 443 ssl http2;
server_name aur.local aur;
root /srv/http/aurweb/web/html;
index index.php;
# To enable SSL proxy properly, make sure gunicorn and friends
# are supporting forwarded headers over 127.0.0.1 or any if
# the asgi server is contacted by non-localhost hosts.
ssl_certificate /etc/ssl/certs/aur.cert.pem;
ssl_certificate_key /etc/ssl/private/aur.key.pem;
# smartgit location.
location ~ "^/([a-z0-9][a-z0-9.+_-]*?)(\.git)?/(git-(receive|upload)-pack|HEAD|info/refs|objects/(info/(http-)?alternates|packs)|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\.(pack|idx))$" {
include uwsgi_params;
uwsgi_pass smartgit;
uwsgi_modifier1 9;
uwsgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend;
uwsgi_param PATH_INFO /aur.git/$3;
uwsgi_param GIT_HTTP_EXPORT_ALL "";
uwsgi_param GIT_NAMESPACE $1;
uwsgi_param GIT_PROJECT_ROOT /srv/http/aurweb;
}
location ~ ^/[^/]+\.php($|/) {
fastcgi_pass unix:/var/run/php-fpm/php-fpm.sock;
fastcgi_index index.php;
fastcgi_split_path_info ^(/[^/]+\.php)(/.*)$;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
include fastcgi_params;
# cgitrc.proto should be configured and located somewhere
# of your choosing.
location ~ ^/cgit {
include uwsgi_params;
rewrite ^/cgit/([^?/]+/[^?]*)?(?:\?(.*))?$ /cgit.cgi?url=$1&$2 last;
uwsgi_modifier1 9;
uwsgi_param CGIT_CONFIG /srv/http/aurweb/conf/cgitrc.proto;
uwsgi_pass cgit;
}
location ~ .* {
rewrite ^/(.*)$ /index.php/$1 last;
# Static archive assets.
location ~ \.gz$ {
# Asset root. This is used to match against gzip archives.
root /srv/http/aurweb/archives;
types { application/gzip text/plain }
default_type text/plain;
add_header Content-Encoding gzip;
expires 5m;
}
}
Ensure to enable the pdo_mysql extension in php.ini.
# For everything else, proxy the http request to (guni|uvi|hyper)corn.
# The ASGI server application should allow this request's IP to be
# forwarded via the headers used below.
# https://docs.gunicorn.org/en/stable/settings.html#forwarded-allow-ips
location / {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Protocol ssl;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Ssl on;
}
}
3) Optionally copy conf/config.defaults to /etc/aurweb/. Create or copy
/etc/aurweb/config (this is expected to contain all configuration settings
if the defaults file does not exist) and adjust the configuration (pay
attention to disable_http_login, enable_maintenance and aur_location).
4) Install Python modules and dependencies:
4) Install system-wide dependencies:
# pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \
python-bleach python-markdown python-alembic python-jinja \
python-itsdangerous python-authlib python-httpx hypercorn \
python-orjson
# python3 setup.py install
# pacman -S git gpgme cgit curl openssh uwsgi uwsgi-plugin-cgi \
python-poetry
5) Create a new MySQL database and a user and import the aurweb SQL schema:
5) Create a new user:
$ python -m aurweb.initdb
# useradd -U -d /srv/http/aurweb -c 'AUR user' aur
# su - aur
6) Create a new user:
6a) Install Python dependencies via poetry:
# useradd -U -d /srv/http/aurweb -c 'AUR user' aur
# Install the package and scripts as the aur user.
$ poetry install
6b) Setup Services
aurweb utilizes the following systemd services:
- mariadb
- redis (optional, requires [options] cache 'redis')
- `examples/aurweb.service`
6c) Setup Cron
Using [cronie](https://archlinux.org/packages/core/x86_64/cronie/):
7) Initialize the Git repository:
# su - aur
$ crontab -e
The following crontab file uses every script meant to be run on an
interval:
AUR_CONFIG='/etc/aurweb/config'
*/5 * * * * bash -c 'poetry run aurweb-mkpkglists --extended'
*/2 * * * * bash -c 'poetry run aurweb-aurblup'
*/2 * * * * bash -c 'poetry run aurweb-pkgmaint'
*/2 * * * * bash -c 'poetry run aurweb-usermaint'
*/2 * * * * bash -c 'poetry run aurweb-popupdate'
*/12 * * * * bash -c 'poetry run aurweb-votereminder'
7) Create a new database and a user and import the aurweb SQL schema:
$ poetry run python -m aurweb.initdb
8) Initialize the Git repository:
# mkdir /srv/http/aurweb/aur.git/
# cd /srv/http/aurweb/aur.git/
......@@ -69,19 +134,26 @@ read the instructions below.
# git config --local transfer.hideRefs '^refs/'
# git config --local --add transfer.hideRefs '!refs/'
# git config --local --add transfer.hideRefs '!HEAD'
# ln -s /usr/local/bin/aurweb-git-update hooks/update
# chown -R aur .
Link to `aurweb-git-update` poetry wrapper provided at
`examples/aurweb-git-update.sh` which should be installed
somewhere as executable.
# ln -s /path/to/aurweb-git-update.sh hooks/update
It is recommended to read doc/git-interface.txt for more information on the
administration of the package Git repository.
8) Configure sshd(8) for the AUR. Add the following lines at the end of your
sshd_config(5) and restart the sshd. Note that OpenSSH 6.9 or newer is
needed!
9) Configure sshd(8) for the AUR. Add the following lines at the end of your
sshd_config(5) and restart the sshd.
If using a virtualenv, copy `examples/aurweb-git-auth.sh` to a location
and call it below:
Match User aur
PasswordAuthentication no
AuthorizedKeysCommand /usr/local/bin/aurweb-git-auth "%t" "%k"
AuthorizedKeysCommand /path/to/aurweb-git-auth.sh "%t" "%k"
AuthorizedKeysCommandUser aur
AcceptEnv AUR_OVERWRITE
......@@ -100,8 +172,17 @@ read the instructions below.
Sample systemd unit files for fcgiwrap can be found under conf/.
10) If you want memcache to cache MySQL data.
10) If you want Redis to cache data.
# pacman -S redis
# systemctl enable --now redis
And edit the configuration file to enabled redis caching
(`[options] cache = redis`).
# pacman -S php-memcached
11) Start `aurweb.service`.
And edit the configuration file to enabled memcache caching.
An example systemd unit has been included at `examples/aurweb.service`.
This unit can be used to manage the aurweb asgi backend. By default,
it is configured to use `poetry` as the `aur` user; this should be
configured as needed.
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
......@@ -11,38 +11,56 @@ The aurweb project includes
* A web interface to search for packaging scripts and display package details.
* An SSH/Git interface to submit and update packages and package meta data.
* Community features such as comments, votes, package flagging and requests.
* Editing/deletion of packages and accounts by Trusted Users and Developers.
* Area for Trusted Users to post AUR-related proposals and vote on them.
* Editing/deletion of packages and accounts by Package Maintainers and Developers.
* Area for Package Maintainers to post AUR-related proposals and vote on them.
Directory Layout
----------------
* `aurweb`: aurweb Python modules, Git interface and maintenance scripts
* `conf`: configuration and configuration templates
* `static`: static resource files
* `templates`: jinja2 template collection
* `doc`: project documentation
* `po`: translation files for strings in the aurweb interface
* `schema`: schema for the SQL database
* `test`: test suite and test cases
* `upgrading`: instructions for upgrading setups from one release to another
* `web`: web interface for the AUR
Documentation
-------------
| What | Link |
|--------------|--------------------------------------------------|
| Installation | [INSTALL](./INSTALL) |
| Testing | [test/README.md](./test/README.md) |
| Git | [doc/git-interface.txt](./doc/git-interface.txt) |
| Maintenance | [doc/maintenance.txt](./doc/maintenance.txt) |
| RPC | [doc/rpc.txt](./doc/rpc.txt) |
| Docker | [doc/docker.md](./doc/docker.md) |
Links
-----
* The repository is hosted at https://gitlab.archlinux.org/archlinux/aurweb -- see
doc/CodingGuidelines for information on the patch submission process.
* The repository is hosted at https://gitlab.archlinux.org/archlinux/aurweb
-- see [CONTRIBUTING.md](./CONTRIBUTING.md) for information on the patch submission process.
* Bugs can (and should) be submitted to the aurweb bug tracker:
https://bugs.archlinux.org/index.php?project=2
https://gitlab.archlinux.org/archlinux/aurweb/-/issues/new?issuable_template=Bug
* Questions, comments, and patches related to aurweb can be sent to the AUR
development mailing list: aur-dev@archlinux.org -- mailing list archives:
https://mailman.archlinux.org/mailman/listinfo/aur-dev
https://lists.archlinux.org/archives/list/aur-dev@lists.archlinux.org/
Translations
------------
Translations are welcome via our Transifex project at
https://www.transifex.com/lfleischer/aurweb; see `doc/i18n.txt` for details.
https://www.transifex.com/lfleischer/aurweb; see [doc/i18n.md](./doc/i18n.md) for details.
![Transifex](https://www.transifex.com/projects/p/aurweb/chart/image_png)
Testing
-------
See [test/README.md](test/README.md) for details on dependencies and testing.
Setup Testing Environment
=========================
The quickest way to get you hacking on aurweb is to utilize docker.
In case you prefer to run it bare-metal see instructions further below.
Containerized environment
-------------------------
1) Clone the aurweb project:
$ git clone https://gitlab.archlinux.org/archlinux/aurweb.git
$ cd aurweb
2) Install the necessary packages:
# pacman -S --needed docker docker-compose
3) Build the aurweb:latest image:
# systemctl start docker
# docker compose build
4) Run local Docker development instance:
# docker compose up -d
5) Browse to local aurweb development server.
https://localhost:8444/
6) [Optionally] populate the database with dummy data:
# docker compose exec mariadb /bin/bash
# pacman -S --noconfirm words fortune-mod
# poetry run schema/gendummydata.py dummy_data.sql
# mariadb -uaur -paur aurweb < dummy_data.sql
# exit
Inspect `dummy_data.sql` for test credentials.
Passwords match usernames.
We now have fully set up environment which we can start and stop with:
# docker compose start
# docker compose stop
Proceed with topic "Setup for running tests"
Bare Metal installation
-----------------------
Note that this setup is only to test the web interface. If you need to have a
full aurweb instance with cgit, ssh interface, etc, follow the directions in
INSTALL.
1) Clone the aurweb project:
$ git clone https://gitlab.archlinux.org/archlinux/aurweb.git
$ git clone git://git.archlinux.org/aurweb.git
$ cd aurweb
2) Install the necessary packages:
# pacman -S --needed php php-sqlite sqlite words fortune-mod \
python python-sqlalchemy python-alembic \
python-fastapi uvicorn nginx \
python-authlib python-itsdangerous python-httpx \
words fortune-mod
# pacman -S --needed python-poetry mariadb words fortune-mod nginx
Ensure to enable the pdo_sqlite extension in php.ini.
3) Install the package/dependencies via `poetry`:
$ poetry install
4) Copy conf/config.dev to conf/config and replace YOUR_AUR_ROOT by the absolute
path to the root of your aurweb clone. sed can do both tasks for you:
$ sed -e "s;YOUR_AUR_ROOT;$PWD;g" conf/config.dev > conf/config
Note that when the upstream config.dev is updated, you should compare it to
your conf/config, or regenerate your configuration with the command above.
5) Set up mariadb:
# mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
# systemctl start mariadb
# mariadb -u root
> CREATE USER 'aur'@'localhost' IDENTIFIED BY 'aur';
> GRANT ALL ON *.* TO 'aur'@'localhost' WITH GRANT OPTION;
> CREATE DATABASE aurweb;
> exit
6) Prepare a database and insert dummy data:
$ AUR_CONFIG=conf/config poetry run python -m aurweb.initdb
$ poetry run schema/gendummydata.py dummy_data.sql
$ mariadb -uaur -paur aurweb < dummy_data.sql
7) Run the test server:
## set AUR_CONFIG to our locally created config
$ export AUR_CONFIG=conf/config
## with aurweb.spawn
$ poetry run python -m aurweb.spawn
## with systemd service
$ sudo install -m644 examples/aurweb.service /etc/systemd/system/
# systemctl enable --now aurweb.service
Setup for running tests
-----------------------
If you've set up a docker environment, you can run the full test-suite with:
# docker compose run test
You can collect code-coverage data with:
$ ./util/fix-coverage data/.coverage
See information further below on how to visualize the data.
For running individual tests, we need to perform a couple of additional steps.
In case you did the bare-metal install, steps 2, 3, 4 and 5 should be skipped.
1) Install the necessary packages:
# pacman -S --needed python-poetry mariadb-libs asciidoc openssh
2) Install the package/dependencies via `poetry`:
$ poetry install
3) Copy conf/config.dev to conf/config and replace YOUR_AUR_ROOT by the absolute
path to the root of your aurweb clone. sed can do both tasks for you:
......@@ -27,15 +135,51 @@ INSTALL.
Note that when the upstream config.dev is updated, you should compare it to
your conf/config, or regenerate your configuration with the command above.
4) Prepare the testing database:
4) Edit the config file conf/config and change the mysql/mariadb portion
We can make use of our mariadb docker container instead of having to install
mariadb. Change the config as follows:
---------------------------------------------------------------------
; MySQL database information. User defaults to root for containerized
; testing with mysqldb. This should be set to a non-root user.
user = root
password = aur
host = 127.0.0.1
port = 13306
;socket = /var/run/mysqld/mysqld.sock
---------------------------------------------------------------------
5) Start our mariadb docker container
# docker compose start mariadb
6) Set environment variables
$ export AUR_CONFIG=conf/config
$ export LOG_CONFIG=logging.test.conf
7) Compile translation & doc files
$ make -C po install
$ make -C doc
Now we can run our python test-suite or individual tests with:
$ poetry run pytest test/
$ poetry run pytest test/test_whatever.py
To run Sharness tests:
$ cd /path/to/aurweb/
$ poetry run make -C test sh
$ AUR_CONFIG=conf/config python -m aurweb.initdb
The e-Mails that have been generated can be found at test-emails/
$ schema/gendummydata.py data.sql
$ sqlite3 aurweb.sqlite3 < data.sql
After test runs, code-coverage reports can be created with:
## CLI report
$ coverage report
5) Run the test server:
## HTML version stored at htmlcov/
$ coverage html
$ AUR_CONFIG=conf/config python -m aurweb.spawn
More information about tests can be found at test/README.md
# aurweb.archives