port over base HTML layout from PHP to FastAPI+Jinja2

+ Mounted static files (at web/html) to /static.
+ Added AURWEB_VERSION to aurweb.config (this is used around HTML
  to refer back to aurweb's release on git.archlinux.org), so we
  need it easily accessible in the Python codebase.
+ Implemented basic Jinja2 partials to put together whole aurweb
  pages. This may be missing some things currently and is a WIP
  until this set is ready to be merged.
+ Added config [options] aurwebdir = YOUR_AUR_ROOT; this configuration
  option should specify the root directory of the aurweb project.
  It is used by various parts of the FastAPI codebase to target
  project directories.

Added routes via aurweb.routers.html:
    * POST /language: Set your session language.
    * GET /favicon.ico: Redirect to /static/images/favicon.ico.
        * Some browsers always look for $ROOT/favicon.ico to get an icon
          for the page being loaded, regardless of a specified "shortcut
          icon" given in a <link> directive.
    * GET /: Home page; WIP.

* Updated aurweb.routers.html.language passes query parameters to
  its next redirection.

When calling aurweb.templates.render_template, the context passed should
be formed via the aurweb.templates.make_context. See
aurweb.routers.html.index for an example of this.

Signed-off-by: Kevin Morris <kevr@0cost.org>

.coveragerc

@@ -3,5 +3,6 @@ disable_warnings = already-imported
 
 [report]
 include = aurweb/*
+fail_under = 85
 exclude_lines =
     if __name__ == .__main__.:

INSTALL

@@ -49,7 +49,9 @@ read the instructions below.
 
     # pacman -S python-mysql-connector python-pygit2 python-srcinfo python-sqlalchemy \
                 python-bleach python-markdown python-alembic hypercorn \
-                python-itsdangerous python-authlib python-httpx
+                python-itsdangerous python-authlib python-httpx \
+                python-jinja python-aiofiles python-python-multipart \
+                python-requests
     # python3 setup.py install
 
 5) Create a new MySQL database and a user and import the aurweb SQL schema:

aurweb/asgi.py

@@ -2,13 +2,26 @@ import http
 
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
 from starlette.middleware.sessions import SessionMiddleware
 
 import aurweb.config
 
-from aurweb.routers import sso
+from aurweb.routers import html, sso
 
+routes = set()
+
+# Setup the FastAPI app.
 app = FastAPI()
+app.mount("/static/css",
+          StaticFiles(directory="web/html/css"),
+          name="static_css")
+app.mount("/static/js",
+          StaticFiles(directory="web/html/js"),
+          name="static_js")
+app.mount("/static/images",
+          StaticFiles(directory="web/html/images"),
+          name="static_images")
 
 session_secret = aurweb.config.get("fastapi", "session_secret")
 if not session_secret:
@@ -17,6 +30,14 @@ if not session_secret:
 app.add_middleware(SessionMiddleware, secret_key=session_secret)
 
 app.include_router(sso.router)
+app.include_router(html.router)
+
+# NOTE: Always keep this dictionary updated with all routes
+# that the application contains. We use this to check for
+# parameter value verification.
+routes = {route.path for route in app.routes}
+routes.update({route.path for route in sso.router.routes})
+routes.update({route.path for route in html.router.routes})
 
 
 @app.exception_handler(HTTPException)

aurweb/config.py

 import configparser
 import os
 
+# Publicly visible version of aurweb. This is used to display
+# aurweb versioning in the footer and must be maintained.
+# Todo: Make this dynamic/automated.
+AURWEB_VERSION = "v5.0.0"
+
 _parser = None
 
 

aurweb/routers/html.py

+""" AURWeb's primary routing module. Define all routes via @app.app.{get,post}
+decorators in some way; more complex routes should be defined in their
+own modules and imported here. """
+from http import HTTPStatus
+from urllib.parse import unquote
+
+from fastapi import APIRouter, Form, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+
+from aurweb.templates import make_context, render_template
+
+router = APIRouter()
+
+
+@router.get("/favicon.ico")
+async def favicon(request: Request):
+    """ Some browsers attempt to find a website's favicon via root uri at
+    /favicon.ico, so provide a redirection here to our static icon. """
+    return RedirectResponse("/static/images/favicon.ico")
+
+
+@router.post("/language", response_class=RedirectResponse)
+async def language(request: Request,
+                   set_lang: str = Form(...),
+                   next: str = Form(...),
+                   q: str = Form(default=None)):
+    """ A POST route used to set a session's language.
+
+    Return a 303 See Other redirect to {next}?next={next}. If we are
+    setting the language on any page, we want to preserve query
+    parameters across the redirect.
+    """
+    from aurweb.asgi import routes
+    if unquote(next) not in routes:
+        return HTMLResponse(
+            b"Invalid 'next' parameter.",
+            status_code=400)
+
+    query_string = "?" + q if q else str()
+    response = RedirectResponse(url=f"{next}{query_string}",
+                                status_code=int(HTTPStatus.SEE_OTHER))
+    response.set_cookie("AURLANG", set_lang)
+    return response
+
+
+@router.get("/", response_class=HTMLResponse)
+async def index(request: Request):
+    """ Homepage route. """
+    context = make_context(request, "Home")
+    return render_template("index.html", context)

aurweb/templates.py

+import copy
+import os
+
+from datetime import datetime
+from http import HTTPStatus
+
+import jinja2
+
+from fastapi import Request
+from fastapi.responses import HTMLResponse
+
+import aurweb.config
+
+from aurweb import l10n
+
+# Prepare jinja2 objects.
+loader = jinja2.FileSystemLoader(os.path.join(
+    aurweb.config.get("options", "aurwebdir"), "templates"))
+env = jinja2.Environment(loader=loader, autoescape=True,
+                         extensions=["jinja2.ext.i18n"])
+
+# Add tr translation filter.
+env.filters["tr"] = l10n.tr
+
+
+def make_context(request: Request, title: str, next: str = None):
+    """ Create a context for a jinja2 TemplateResponse. """
+
+    return {
+        "request": request,
+        "language": l10n.get_request_language(request),
+        "languages": l10n.SUPPORTED_LANGUAGES,
+        "title": title,
+        # The 'now' context variable will not show proper datetimes
+        # until we've implemented timezone support here.
+        "now": datetime.now(),
+        "config": aurweb.config,
+        "next": next if next else request.url.path
+    }
+
+
+def render_template(path: str, context: dict, status_code=int(HTTPStatus.OK)):
+    """ Render a Jinja2 multi-lingual template with some context. """
+
+    # Create a deep copy of our jinja2 environment. The environment in
+    # total by itself is 48 bytes large (according to sys.getsizeof).
+    # This is done so we can install gettext translations on the template
+    # environment being rendered without installing them into a global
+    # which is reused in this function.
+    templates = copy.copy(env)
+
+    translator = l10n.get_raw_translator_for_request(context.get("request"))
+    templates.install_gettext_translations(translator)
+
+    template = templates.get_template(path)
+    rendered = template.render(context)
+    return HTMLResponse(rendered, status_code=status_code)

conf/config.dev

@@ -16,6 +16,7 @@ name = YOUR_AUR_ROOT/aurweb.sqlite3
 ;password = aur
 
 [options]
+aurwebdir = YOUR_AUR_ROOT
 aur_location = http://127.0.0.1:8080
 disable_http_login = 0
 enable-maintenance = 0

templates/index.html

+{% extends 'partials/layout.html' %}
+
+{% block pageContent %}
+{% endblock %}

templates/partials/archdev-navbar.html

+<div id="archdev-navbar">
+    <ul>
+        <li><a href="/">AUR {% trans %}Home{% endtrans %}</a></li>
+        <li><a href="/packages/">{% trans %}Packages{% endtrans %}</a></li>
+        <li><a href="/register/">{% trans %}Register{% endtrans %}</a></li>
+        <li><a href="/login/">{% trans %}Login{% endtrans %}</a></li>
+    </ul>
+</div>

templates/partials/body.html

+<div id="content">
+    {% include 'partials/set_lang.html' %}
+    {% include 'partials/archdev-navbar.html' %}
+
+    {% block pageContent %}
+    <!-- Content block to be defined by extender. -->
+    {% endblock %}
+
+    {% include 'partials/footer.html' %}
+</div>

templates/partials/footer.html

+<div id="footer">
+    <p>aurweb <a href="https://git.archlinux.org/aurweb.git/log/?h={{ config.AURWEB_VERSION }}">{{ config.AURWEB_VERSION }}</a></p>
+    <p>Copyright &copy; 2004-{{ now.strftime("%Y") }} aurweb Development Team.</p>
+    <p>{% trans %}AUR packages are user produced content. Any use of the provided files is at your own risk.{% endtrans %}</p>
+</div>

templates/partials/head.html

+<head>
+    {% include 'partials/meta.html' %}
+
+    <!-- CSS -->
+    <link rel="stylesheet" href="/static/css/archweb.css">
+    <link rel="stylesheet" href="/static/css/aurweb.css">
+
+    <!-- Resources -->
+    <link rel="shortcut icon" href="/static/images/favicon.ico">
+
+    <!-- Alternate resources -->
+    <link rel="alternate" type="application/rss+xml"
+          title="Newest Packages RSS" href="/rss/">
+
+    <title>AUR ({{ language }}) - {{ title | tr }}</title>
+</head>

templates/partials/layout.html

+<!DOCTYPE html>
+<html lang="{{ language }}">
+    {% include 'partials/head.html' %}
+
+    <body>
+        {% include 'partials/navbar.html' %}
+        {% extends 'partials/body.html' %}
+        {% include 'partials/typeahead.html' %}
+    </body>
+</html>

templates/partials/meta.html

+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">

templates/partials/navbar.html

+<div id="archnavbar" class="anb-aur">
+    <div id="archnavbarlogo">
+        <h1>
+            <a href="/" title="Return to the main page">{% trans %}Arch Linux User Repository{% endtrans %}</a>
+        </h1>
+    </div>
+    <div id="archnavbarmenu">
+        <ul id="archnavbarlist">
+            <li id="anb-home"><a href="https://www.archlinux.org/" title="Arch news, packages, projects and more">{% trans %}Home{% endtrans %}</a></li>
+            <li id="anb-packages"><a href="https://www.archlinux.org/packages/" title="Arch Package Database">{% trans %}Packages{% endtrans %}</a></li>
+            <li id="anb-forums"><a href="https://bbs.archlinux.org/" title="Community forums">{% trans %}Forums{% endtrans %}</a></li>
+            <li id="anb-wiki"><a href="https://wiki.archlinux.org/" title="Community documentation">{% trans %}Wiki{% endtrans %}</a></li>
+            <li id="anb-bugs"><a href="https://bugs.archlinux.org/" title="Report and track bugs">{% trans %}Bugs{% endtrans %}</a></li>
+            <li id="anb-security"><a href="https://security.archlinux.org/" title="Arch Linux Security Tracker">{% trans %}Security{% endtrans %}</a></li>
+            <li id="anb-aur"><a href="/" title="Arch Linux User Repository">AUR</a></li>
+            <li id="anb-download"><a href="https://www.archlinux.org/download/" title="Get Arch Linux">{% trans %}Download{% endtrans %}</a></li>
+        </ul>
+    </div>
+</div>

templates/partials/set_lang.html

+<div id="lang_sub">
+    <form method="post" action="/language">
+        <fieldset>
+            <div>
+                <select id="id_setlang" name="set_lang">
+                    {% for domain, display in languages.items() %}
+                    <option
+                        value="{{ domain }}"
+                        {% if language == domain %}
+                        selected="selected"
+                        {% endif %}
+                    >
+                        {{ display }}
+                    </option>
+                    {% endfor %}
+                </select>
+
+                <!-- Pass our current url path as next. -->
+                <input type="hidden" name="next" value="{{ request.url.path }}">
+
+                <!-- Pass query_params over to /language via POST. -->
+                <input type="hidden" name="q" value="{{ request.query_params }}">
+
+                <input type="submit" value="Go">
+            </div>
+        </fieldset>
+    </form>
+</div>

templates/partials/typeahead.html

+<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
+<script type="text/javascript" src="/static/js/bootstrap-typeahead.min.js"></script>
+<script type="text/javascript">
+    $(document).ready(function() {
+        $('#pkgsearch-field').typeahead({
+            source: function(query, callback) {
+                $.getJSON('/rpc', {type: "suggest", arg: query}, function(data) {
+                        callback(data);
+                    });
+                },
+                matcher: function(item) { return true; },
+                sorter: function(items) { return items; },
+                menu: '<ul class="pkgsearch-typeahead"></ul>',
+                items: 20,
+                updater: function(item) {
+                document.location = '/packages/' + item;
+                return item;
+            }
+        }).attr('autocomplete', 'off');
+
+        $('#pkgsearch-field').keydown(function(e) {
+            if (e.keyCode == 13) {
+                var selectedItem = $('ul.pkgsearch-typeahead li.active');
+                if (selectedItem.length == 0) {
+                        $('#pkgsearch-form').submit();
+                }
+            }
+        });
+    });
+</script>

test/test_routes.py

+import urllib.parse
+
+from http import HTTPStatus
+
+import pytest
+
+from fastapi.testclient import TestClient
+
+from aurweb.asgi import app
+from aurweb.testing import setup_test_db
+
+client = TestClient(app)
+
+
+@pytest.fixture
+def setup():
+    setup_test_db("Users", "Session")
+
+
+def test_index():
+    """ Test the index route at '/'. """
+    # Use `with` to trigger FastAPI app events.
+    with client as req:
+        response = req.get("/")
+    assert response.status_code == int(HTTPStatus.OK)
+
+
+def test_favicon():
+    """ Test the favicon route at '/favicon.ico'. """
+    response1 = client.get("/static/images/favicon.ico")
+    response2 = client.get("/favicon.ico")
+    assert response1.status_code == int(HTTPStatus.OK)
+    assert response1.content == response2.content
+
+
+def test_language():
+    """ Test the language post route at '/language'. """
+    post_data = {
+        "set_lang": "de",
+        "next": "/"
+    }
+    with client as req:
+        response = req.post("/language", data=post_data)
+    assert response.status_code == int(HTTPStatus.SEE_OTHER)
+
+
+def test_language_invalid_next():
+    """ Test an invalid next route at '/language'. """
+    post_data = {
+        "set_lang": "de",
+        "next": "/BLAHBLAHFAKE"
+    }
+    with client as req:
+        response = req.post("/language", data=post_data)
+    assert response.status_code == int(HTTPStatus.BAD_REQUEST)
+
+def test_language_query_params():
+    """ Test the language post route with query params. """
+    next = urllib.parse.quote_plus("/")
+    post_data = {
+        "set_lang": "de",
+        "next": "/",
+        "q": f"next={next}"
+    }
+    q = post_data.get("q")
+    with client as req:
+        response = req.post("/language", data=post_data)
+        assert response.headers.get("location") == f"/?{q}"
+    assert response.status_code == int(HTTPStatus.SEE_OTHER)