Commit 5bd46d18 authored by Kevin Morris's avatar Kevin Morris
Browse files

Improve Docker ecosystem

Instead of using Dockerfile for everything, we've introduced
a docker-compose.yml file and kept the Dockerfile to producing
a pure base image for the services defined.

docker-compose services:

- `mariadb` - Setup mariadb
- `sharness` - Run sharness suites
- `pytest-mysql` - Run pytest suites with MariaDB
- `pytest-sqlite` - Run pytest suites with SQLite
- `test` - Run all tests and produce a collective coverage report
    - This target mounts a cache volume and copies any successful
      coverage report back to `./cache/.coverage`. Users can run
      `./util/fix-coverage ./cache/.coverage` to rewrite source
      code paths and move coverage into place to view reports
      on your local system.

== Get Started ==

Build `aurweb:latest`.

    $ docker build -t aurweb:latest .

Run all tests via `docker-compose`.

    $ docker-compose up test

You can also purely run `pytest` in SQLite or MariaDB modes.

    $ docker-compose up pytest-sqlite
    $ docker-compose up pytest-mysql...
parent 3b8e3f3e
......@@ -18,3 +18,4 @@ fastapi_aw/
.pylintrc
.coverage
.idea
/cache/*
FROM archlinux
COPY . /aurweb
WORKDIR /aurweb
FROM archlinux:base-devel
# Setup some default system stuff.
RUN bash -c 'echo "127.0.0.1 localhost" >> /etc/hosts'
RUN bash -c 'echo "::1 localhost" >> /etc/hosts'
RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime
RUN mkdir -p .pkg-cache
# Install dependencies.
RUN pacman -Syu --noconfirm base-devel git gpgme protobuf pyalpm \
python-mysql-connector python-pygit2 python-srcinfo python-bleach \
RUN pacman -Syu --noconfirm --noprogressbar \
--cachedir .pkg-cache git gpgme protobuf pyalpm \
python-mysqlclient python-pygit2 python-srcinfo python-bleach \
python-markdown python-sqlalchemy python-alembic python-pytest \
python-werkzeug python-pytest-tap python-fastapi nginx python-authlib \
python-itsdangerous python-httpx python-jinja python-pytest-cov \
python-requests python-aiofiles python-python-multipart \
python-pytest-asyncio python-coverage hypercorn python-bcrypt \
python-email-validator openssh python-lxml
python-email-validator openssh python-lxml mariadb mariadb-libs \
python-isort flake8
# Remove aurweb.sqlite3 if it was copied over via COPY.
RUN rm -fv aurweb.sqlite3
RUN useradd -U -d /aurweb -c 'AUR User' aur
# Setup our test config.
RUN sed -r "s;YOUR_AUR_ROOT;/aurweb;g" conf/config.dev > conf/config
COPY docker /docker
# Install translations.
RUN AUR_CONFIG=conf/config make -C po all install
# Initialize the database.
RUN AUR_CONFIG=conf/config python -m aurweb.initdb
WORKDIR /aurweb
COPY . .
# Test everything!
RUN make -C test
ENV PYTHONPATH=/aurweb
ENV AUR_CONFIG=conf/config
# Produce a coverage report.
RUN coverage report --include='aurweb/*'
RUN make -C po all install
RUN python setup.py install
#
# Docker service definitions for the aurweb project.
#
# Notable services:
# - `sharness` - Run sharness test suites
# - `pytest-mysql` - Run pytest suites with MariaDB
# - `pytest-sqlite` - Run pytest suites with SQLite
# - `test` - Run sharness, pytest-mysql and pytest-sqlite
# - `mariadb` - `port 13306` - MariaDB server for docker
#
# Copyright (C) 2021 aurweb Development
# All Rights Reserved.
version: "3.8"
services:
mariadb:
image: aurweb:latest
init: true
entrypoint: /docker/mariadb-entrypoint.sh
command: /docker/scripts/run-mariadb.sh mysqld_safe --datadir=/var/lib/mysql
ports:
# This will expose mariadbd on 127.0.0.1:13306 in the host.
# Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb`
- "13306:3306"
volumes:
- mariadb_run:/var/run/mysqld # Bind socket in this volume.
- mariadb_data:/var/lib/mysql
healthcheck:
test: "bash /docker/health/mariadb.sh"
interval: 2s
timeout: 60s
sharness:
image: aurweb:latest
init: true
environment:
- AUR_CONFIG=conf/config.sqlite
entrypoint: /docker/test-sqlite-entrypoint.sh
command: /docker/scripts/run-sharness.sh
volumes:
- ./cache:/cache
pytest-mysql:
image: aurweb:latest
init: true
environment:
- AUR_CONFIG=conf/config
entrypoint: /docker/test-mysql-entrypoint.sh
command: /docker/scripts/run-pytests.sh clean
depends_on:
mariadb:
condition: service_healthy
links:
- mariadb
volumes:
- mariadb_run:/var/run/mysqld
- ./cache:/cache
pytest-sqlite:
image: aurweb:latest
init: true
environment:
- AUR_CONFIG=conf/config.sqlite
entrypoint: /docker/test-sqlite-entrypoint.sh
command: /docker/scripts/run-pytests.sh clean
volumes:
- mariadb_run:/var/run/mysqld
- ./cache:/cache
test:
image: aurweb:latest
init: true
environment:
- AUR_CONFIG=conf/config
entrypoint: /docker/tests-entrypoint.sh
command: /docker/scripts/run-tests.sh
depends_on:
mariadb:
condition: service_healthy
links:
- mariadb
volumes:
- mariadb_run:/var/run/mysqld
- ./cache:/cache
volumes:
mariadb_run: {} # Share /var/run/mysqld/mysqld.sock
mariadb_data: {} # Share /var/lib/mysql
#!/bin/bash
exec mysqladmin ping --silent
#!/bin/bash
set -eou pipefail
mariadb-install-db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
exec "$@"
#!/bin/bash
mysqld_safe --datadir=/var/lib/mysql --skip-networking &
until mysqladmin ping --silent; do
sleep 1s
done
# Create test database.
mysql -u root -e "CREATE USER 'aur'@'%' IDENTIFIED BY 'aur'" \
2>/dev/null || /bin/true
mysql -u root -e "DROP DATABASE aurweb_test" 2>/dev/null || /bin/true
mysql -u root -e "CREATE DATABASE aurweb_test"
mysql -u root -e "GRANT ALL PRIVILEGES ON aurweb_test.* TO 'aur'@'%'"
mysql -u root -e "FLUSH PRIVILEGES"
# Shutdown mariadb.
mysqladmin -uroot shutdown
exec "$@"
#!/bin/bash
set -eou pipefail
COVERAGE=1
PARAMS=()
while [ $# -ne 0 ]; do
key="$1"
case "$key" in
--no-coverage)
COVERAGE=0
shift
;;
-*)
echo "usage: $0 [--no-coverage] targets ..."
exit 1
;;
*)
PARAMS+=("$key")
shift
;;
esac
done
# Initialize the new database; ignore errors.
python -m aurweb.initdb 2>/dev/null || /bin/true
# Run pytest with optional targets in front of it.
make -C test "${PARAMS[@]}" pytest
# By default, report coverage and move it into cache.
if [ $COVERAGE -eq 1 ]; then
make -C test coverage
# /cache is mounted as a volume. Copy coverage into it.
# Users can then sanitize the coverage locally in their
# aurweb root directory: ./util/fix-coverage ./cache/.coverage
rm -f /cache/.coverage
cp -v .coverage /cache/.coverage
chmod 666 /cache/.coverage
fi
#!/bin/bash
set -eou pipefail
# Initialize the new database; ignore errors.
python -m aurweb.initdb 2>/dev/null || /bin/true
make -C test sh
#!/bin/bash
set -eou pipefail
dir=$(dirname $0)
# Clean up coverage and stuff.
make -C test clean
# Run sharness tests.
bash $dir/run-sharness.sh
# Run Python tests with MariaDB database.
# Pass --silence to avoid reporting coverage. We will do that below.
bash $dir/run-pytests.sh --no-coverage
# Export SQLite aurweb configuration.
export AUR_CONFIG=conf/config.sqlite
# Run Python tests.
bash $dir/run-pytests.sh --no-coverage
make -C test coverage
# /cache is mounted as a volume. Copy coverage into it.
# Users can then sanitize the coverage locally in their
# aurweb root directory: ./util/fix-coverage ./cache/.coverage
rm -f /cache/.coverage
cp -v .coverage /cache/.coverage
chmod 666 /cache/.coverage
# Run flake8 and isort checks.
for dir in aurweb test migrations; do
flake8 --count $dir
isort --check-only $dir
done
#!/bin/bash
set -eou pipefail
DB_NAME="aurweb_test"
DB_HOST="mariadb"
DB_USER="aur"
DB_PASS="aur"
# Setup a config for our mysql db.
cp -vf conf/config.dev conf/config
sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config
sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config
sed -ri "s/^(host) = .+/\1 = ${DB_HOST}/" conf/config
sed -ri "s/^(user) = .+/\1 = ${DB_USER}/" conf/config
sed -ri "s/^;?(password) = .+/\1 = ${DB_PASS}/" conf/config
# The port can be excluded from use if properly using
# volumes to share the mysql socket from the mariadb service.
# Example port sed:
# sed -i "s/^;?(port = .+)$/\1/" conf/config
# Continue onto the main command.
exec "$@"
#!/bin/bash
set -eou pipefail
DB_BACKEND="sqlite"
DB_NAME="aurweb.sqlite3"
# Create an SQLite config from the default dev config.
cp -vf conf/config.dev conf/config.sqlite
cp -vf conf/config.defaults conf/config.sqlite.defaults
# Modify it for SQLite.
sed -i "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.sqlite
sed -ri "s/^(backend) = .+/\1 = ${DB_BACKEND}/" conf/config.sqlite
sed -ri "s/^(name) = .+/\1 = ${DB_NAME}/" conf/config.sqlite
exec "$@"
#!/bin/bash
set -eou pipefail
dir="$(dirname $0)"
bash $dir/test-mysql-entrypoint.sh
bash $dir/test-sqlite-entrypoint.sh
exec "$@"
......@@ -17,10 +17,15 @@ else
sh: $(T)
endif
coverage:
cd .. && coverage report --include='aurweb/*'
cd .. && coverage xml --include='aurweb/*'
clean:
$(RM) -r test-results/
rm -f ../.coverage
$(T):
@echo "*** $@ ***"; $(SHELL) $@
.PHONY: check $(FOREIGN_TARGETS) clean $(T)
.PHONY: check coverage $(FOREIGN_TARGETS) clean $(T)
#!/usr/bin/env python3
""" A simple script which updates paths for .coverage executed
in another directory. This is particularly useful for fixing
.coverage files received via ./cache/.coverage after Docker runs.
Copyright (C) 2021 aurweb Development
All Rights Reserved.
"""
import os
import re
import sqlite3
import sys
import traceback
import aurweb.config
def eprint(*args, **kwargs):
print(*args, **kwargs, file=sys.stderru)
def main():
if len(sys.argv) == 1:
print(f"usage: {sys.argv[0]} .coverage")
return 1
coverage_db = sys.argv[1]
if not os.path.exists(coverage_db):
eprint("error: coverage file not found")
return 2
aurwebdir = aurweb.config.get("options", "aurwebdir")
new_coverage_db = os.path.join(aurwebdir, ".coverage")
with open(coverage_db, "rb") as i:
with open(new_coverage_db, "wb") as f:
f.write(i.read())
print(f"Copied coverage db to {new_coverage_db}.")
coverage_db = new_coverage_db
sqlite3.paramstyle = "format"
db = sqlite3.connect(coverage_db)
cursor = db.cursor()
results = cursor.execute("SELECT * FROM file")
files = dict()
for i, path in results:
files[i] = path
for _, i in enumerate(files.keys()):
new_path = re.sub(r'^/aurweb', aurwebdir, files[i])
cursor.execute("UPDATE file SET path = ? WHERE id = ?", (
new_path, i))
db.commit()
db.close()
return 0
if __name__ == "__main__":
e = 1
try:
e = main()
except Exception:
traceback.print_exc()
e = 1
sys.exit(e)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment