Commit aecb6494 authored by Kevin Morris's avatar Kevin Morris
Browse files

use mysql backend in config.dev



First off: This commit changes the default development database
backend to mysql. sqlite, however, is still completely supported
with the caveat that a user must now modify config.dev to use
the sqlite backend.

While looking into this, it was discovered that our SQLAlchemy
backend for mysql (mysql-connector) completely broke model
attributes when we switched to utf8mb4_bin (binary) -- it does
not correct the correct conversion to and from binary utf8mb4.

The new, replacement dependency mysqlclient does. mysqlclient
is also recommended in SQLAlchemy documentation as the "best"
one available.

The mysqlclient backend uses a different exception flow then
sqlite, and so tests expecting IntegrityError has to be modified
to expect OperationalError from sqlalchemy.exc.

So, for each model that we define, check keys that can't be
NULL and raise sqlalchemy.exc.IntegrityError if we have to.
This way we keep our exceptions uniform.
Signed-off-by: Kevin Morris's avatarKevin Morris <kevr@0cost.org>
parent d7481b96
......@@ -98,9 +98,11 @@ def get_sqlalchemy_url():
param_query = None
else:
port = None
param_query = {'unix_socket': aurweb.config.get('database', 'socket')}
param_query = {
'unix_socket': aurweb.config.get('database', 'socket')
}
return constructor(
'mysql+mysqlconnector',
'mysql+mysqldb',
username=aurweb.config.get('database', 'user'),
password=aurweb.config.get('database', 'password'),
host=aurweb.config.get('database', 'host'),
......@@ -117,7 +119,7 @@ def get_sqlalchemy_url():
raise ValueError('unsupported database backend')
def get_engine():
def get_engine(echo: bool = False):
"""
Return the global SQLAlchemy engine.
......@@ -135,13 +137,24 @@ def get_engine():
# check_same_thread is for a SQLite technicality
# https://fastapi.tiangolo.com/tutorial/sql-databases/#note
connect_args["check_same_thread"] = False
engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args)
engine = create_engine(get_sqlalchemy_url(),
connect_args=connect_args,
echo=echo)
Session = sessionmaker(autocommit=False, autoflush=True, bind=engine)
session = Session()
return engine
def kill_engine():
global engine, Session, session
if engine:
session.close()
engine.dispose()
engine = Session = session = None
def connect():
"""
Return an SQLAlchemy connection. Connections are usually pooled. See
......@@ -160,8 +173,7 @@ class ConnectionExecutor:
def __init__(self, conn, backend=aurweb.config.get("database", "backend")):
self._conn = conn
if backend == "mysql":
import mysql.connector
self._paramstyle = mysql.connector.paramstyle
self._paramstyle = "format"
elif backend == "sqlite":
import sqlite3
self._paramstyle = sqlite3.paramstyle
......@@ -197,18 +209,17 @@ class Connection:
aur_db_backend = aurweb.config.get('database', 'backend')
if aur_db_backend == 'mysql':
import mysql.connector
import MySQLdb
aur_db_host = aurweb.config.get('database', 'host')
aur_db_name = aurweb.config.get('database', 'name')
aur_db_user = aurweb.config.get('database', 'user')
aur_db_pass = aurweb.config.get('database', 'password')
aur_db_socket = aurweb.config.get('database', 'socket')
self._conn = mysql.connector.connect(host=aur_db_host,
user=aur_db_user,
passwd=aur_db_pass,
db=aur_db_name,
unix_socket=aur_db_socket,
buffered=True)
self._conn = MySQLdb.connect(host=aur_db_host,
user=aur_db_user,
passwd=aur_db_pass,
db=aur_db_name,
unix_socket=aur_db_socket)
elif aur_db_backend == 'sqlite':
import sqlite3
aur_db_name = aurweb.config.get('database', 'name')
......@@ -217,7 +228,7 @@ class Connection:
else:
raise ValueError('unsupported database backend')
self._conn = ConnectionExecutor(self._conn)
self._conn = ConnectionExecutor(self._conn, aur_db_backend)
def execute(self, query, params=()):
return self._conn.execute(query, params)
......
......@@ -2,7 +2,6 @@ import argparse
import alembic.command
import alembic.config
import sqlalchemy
import aurweb.db
import aurweb.schema
......@@ -34,6 +33,8 @@ def feed_initial_data(conn):
def run(args):
aurweb.config.rehash()
# Ensure Alembic is fine before we do the real work, in order not to fail at
# the last step and leave the database in an inconsistent state. The
# configuration is loaded lazily, so we query it to force its loading.
......@@ -42,8 +43,7 @@ def run(args):
alembic_config.get_main_option('script_location')
alembic_config.attributes["configure_logger"] = False
engine = sqlalchemy.create_engine(aurweb.db.get_sqlalchemy_url(),
echo=(args.verbose >= 1))
engine = aurweb.db.get_engine(echo=(args.verbose >= 1))
aurweb.schema.metadata.create_all(engine)
feed_initial_data(engine.connect())
......
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
......@@ -11,7 +12,19 @@ class AcceptedTerm:
User: User = None, Term: Term = None,
Revision: int = None):
self.User = User
if not self.User:
raise IntegrityError(
statement="Foreign key UserID cannot be null.",
orig="AcceptedTerms.UserID",
params=("NULL"))
self.Term = Term
if not self.Term:
raise IntegrityError(
statement="Foreign key TermID cannot be null.",
orig="AcceptedTerms.TermID",
params=("NULL"))
self.Revision = Revision
......
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.schema import ApiRateLimit as _ApiRateLimit
......@@ -8,8 +9,20 @@ class ApiRateLimit:
Requests: int = None,
WindowStart: int = None):
self.IP = IP
self.Requests = Requests
if self.Requests is None:
raise IntegrityError(
statement="Column Requests cannot be null.",
orig="ApiRateLimit.Requests",
params=("NULL"))
self.WindowStart = WindowStart
if self.WindowStart is None:
raise IntegrityError(
statement="Column WindowStart cannot be null.",
orig="ApiRateLimit.WindowStart",
params=("NULL"))
mapper(ApiRateLimit, _ApiRateLimit, primary_key=[_ApiRateLimit.c.IP])
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.schema import Groups
......@@ -6,6 +7,11 @@ from aurweb.schema import Groups
class Group:
def __init__(self, Name: str = None):
self.Name = Name
if not self.Name:
raise IntegrityError(
statement="Column Name cannot be null.",
orig="Groups.Name",
params=("NULL"))
mapper(Group, Groups)
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.schema import Licenses
......@@ -6,6 +7,11 @@ from aurweb.schema import Licenses
class License:
def __init__(self, Name: str = None):
self.Name = Name
if not self.Name:
raise IntegrityError(
statement="Column Name cannot be null.",
orig="Licenses.Name",
params=("NULL"))
mapper(License, Licenses)
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
......@@ -11,7 +12,19 @@ class Package:
Name: str = None, Version: str = None,
Description: str = None, URL: str = None):
self.PackageBase = PackageBase
if not self.PackageBase:
raise IntegrityError(
statement="Foreign key UserID cannot be null.",
orig="Packages.PackageBaseID",
params=("NULL"))
self.Name = Name
if not self.Name:
raise IntegrityError(
statement="Column Name cannot be null.",
orig="Packages.Name",
params=("NULL"))
self.Version = Version
self.Description = Description
self.URL = URL
......
from datetime import datetime
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
......@@ -12,6 +13,12 @@ class PackageBase:
Maintainer: User = None, Submitter: User = None,
Packager: User = None, **kwargs):
self.Name = Name
if not self.Name:
raise IntegrityError(
statement="Column Name cannot be null.",
orig="PackageBases.Name",
params=("NULL"))
self.Flagger = Flagger
self.Maintainer = Maintainer
self.Submitter = Submitter
......
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
......@@ -12,8 +13,26 @@ class PackageDependency:
DepName: str = None, DepDesc: str = None,
DepCondition: str = None, DepArch: str = None):
self.Package = Package
if not self.Package:
raise IntegrityError(
statement="Foreign key PackageID cannot be null.",
orig="PackageDependencies.PackageID",
params=("NULL"))
self.DependencyType = DependencyType
self.DepName = DepName # nullable=False
if not self.DependencyType:
raise IntegrityError(
statement="Foreign key DepTypeID cannot be null.",
orig="PackageDependencies.DepTypeID",
params=("NULL"))
self.DepName = DepName
if not self.DepName:
raise IntegrityError(
statement="Column DepName cannot be null.",
orig="PackageDependencies.DepName",
params=("NULL"))
self.DepDesc = DepDesc
self.DepCondition = DepCondition
self.DepArch = DepArch
......
from sqlalchemy.orm import mapper
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
from aurweb.models.group import Group
......
from sqlalchemy.orm import mapper
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
from aurweb.models.package_base import PackageBase
......
from sqlalchemy.orm import mapper
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
from aurweb.models.license import License
......
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.db import make_relationship
......@@ -12,8 +13,26 @@ class PackageRelation:
RelName: str = None, RelCondition: str = None,
RelArch: str = None):
self.Package = Package
if not self.Package:
raise IntegrityError(
statement="Foreign key PackageID cannot be null.",
orig="PackageRelations.PackageID",
params=("NULL"))
self.RelationType = RelationType
if not self.RelationType:
raise IntegrityError(
statement="Foreign key RelTypeID cannot be null.",
orig="PackageRelations.RelTypeID",
params=("NULL"))
self.RelName = RelName # nullable=False
if not self.RelName:
raise IntegrityError(
statement="Column RelName cannot be null.",
orig="PackageRelations.RelName",
params=("NULL"))
self.RelCondition = RelCondition
self.RelArch = RelArch
......
from sqlalchemy import Column, Integer
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import backref, mapper, relationship
from aurweb.db import make_random_value
from aurweb.db import make_random_value, query
from aurweb.models.user import User
from aurweb.schema import Sessions
class Session:
UsersID = Column(Integer, nullable=True)
def __init__(self, **kwargs):
self.UsersID = kwargs.get("UsersID")
if not query(User, User.ID == self.UsersID).first():
raise IntegrityError(
statement="Foreign key UsersID cannot be null.",
orig="Sessions.UsersID",
params=("NULL"))
self.SessionID = kwargs.get("SessionID")
self.LastUpdateTS = kwargs.get("LastUpdateTS")
......
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import mapper
from aurweb.schema import Terms
......@@ -8,7 +9,19 @@ class Term:
Description: str = None, URL: str = None,
Revision: int = None):
self.Description = Description
if not self.Description:
raise IntegrityError(
statement="Column Description cannot be null.",
orig="Terms.Description",
params=("NULL"))
self.URL = URL
if not self.URL:
raise IntegrityError(
statement="Column URL cannot be null.",
orig="Terms.URL",
params=("NULL"))
self.Revision = Revision
......
......@@ -6,17 +6,19 @@
; development-specific options too.
[database]
backend = sqlite
name = YOUR_AUR_ROOT/aurweb.sqlite3
; Alternative MySQL configuration (Use either port of socket, if both defined port takes priority)
;backend = mysql
;name = aurweb
;user = aur
;password = aur
;host = localhost
; Options: mysql, sqlite.
backend = mysql
; If using sqlite, set name to the database file path.
name = aurweb
; MySQL database information. User defaults to root for containerized
; testing with mysqldb. This should be set to a non-root user.
user = root
;password = non-root-user-password
host = localhost
;port = 3306
;socket = /var/run/mysqld/mysqld.sock
socket = /var/run/mysqld/mysqld.sock
[options]
aurwebdir = YOUR_AUR_ROOT
......
......@@ -8,7 +8,7 @@ MAKEFLAGS = -j1
check: sh pytest
pytest:
cd .. && AUR_CONFIG=conf/config coverage run --append /usr/bin/pytest test
cd .. && coverage run --append /usr/bin/pytest test
ifdef PROVE
sh:
......
......@@ -802,18 +802,40 @@ def test_post_account_edit_ssh_pub_key():
assert response.status_code == int(HTTPStatus.OK)
# Now let's update what's already there to gain coverage over that path.
pk = str()
with tempfile.TemporaryDirectory() as tmpdir:
with open("/dev/null", "w") as null:
proc = Popen(["ssh-keygen", "-f", f"{tmpdir}/test.ssh", "-N", ""],
stdout=null, stderr=null)
proc.wait()
assert proc.returncode == 0
post_data["PK"] = make_ssh_pubkey()
# Read in the public key, then delete the temp dir we made.
pk = open(f"{tmpdir}/test.ssh.pub").read().rstrip()
with client as request:
response = request.post("/account/test/edit", cookies={
"AURSID": sid
}, data=post_data, allow_redirects=False)
post_data["PK"] = pk
assert response.status_code == int(HTTPStatus.OK)
def test_post_account_edit_missing_ssh_pubkey():
request = Request()
sid = user.login(request, "testPassword")
post_data = {
"U": user.Username,
"E": user.Email,
"PK": make_ssh_pubkey(),
"passwd": "testPassword"
}
with client as request:
response = request.post("/account/test/edit", cookies={
"AURSID": sid
}, data=post_data, allow_redirects=False)
assert response.status_code == int(HTTPStatus.OK)
post_data = {
"U": user.Username,
"E": user.Email,
"PK": str(), # Pass an empty string now to walk the delete path.
"passwd": "testPassword"
}
with client as request:
response = request.post("/account/test/edit", cookies={
......
......@@ -34,5 +34,5 @@ def test_api_rate_key_null_requests_raises_exception():
def test_api_rate_key_null_window_start_raises_exception():
from aurweb.db import session
with pytest.raises(IntegrityError):
create(ApiRateLimit, IP="127.0.0.1", WindowStart=1)
create(ApiRateLimit, IP="127.0.0.1", Requests=1)
session.rollback()
......@@ -4,6 +4,8 @@ import pytest
from starlette.authentication import AuthenticationError
import aurweb.config
from aurweb.auth import BasicAuthBackend, has_credential
from aurweb.db import create, query
from aurweb.models.account_type import AccountType
......@@ -53,13 +55,12 @@ async def test_auth_backend_invalid_sid():
async def test_auth_backend_invalid_user_id():
# Create a new session with a fake user id.
now_ts = datetime.utcnow().timestamp()
create(Session, UsersID=666, SessionID="realSession",
LastUpdateTS=now_ts + 5)
db_backend = aurweb.config.get("database", "backend")
with pytest.raises(IntegrityError):
create(Session, UsersID=666, SessionID="realSession",
LastUpdateTS=now_ts + 5)
# Here, we specify a real SID; but it's user is not there.
request.cookies["AURSID"] = "realSession"
with pytest.raises(AuthenticationError, match="Invalid User ID: 666"):
await backend.authenticate(request)
session.rollback()
@pytest.mark.asyncio
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment