From 91bc570e2d4e565a7a40e613ca2df31252d17336 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 18 Sep 2021 21:08:45 -0500 Subject: [PATCH 1/8] Began work on 'search' type for RPC --- aurweb/routers/rpc.py | 6 +++-- aurweb/rpc.py | 61 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index f12dfc2e..90b0f11f 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -50,7 +50,8 @@ async def rpc(request: Request, v: Optional[int] = Query(None), type: Optional[str] = Query(None), arg: Optional[str] = Query(None), - args: Optional[List[str]] = Query(None, alias="arg[]")): + args: Optional[List[str]] = Query(None, alias="arg[]"), + by: Optional[str] = Query(None)): # Defaults for returned data returned_data = {} @@ -96,6 +97,7 @@ async def rpc(request: Request, returned_data = RPC(v=v, type=type, argument_list=argument_list, - returned_data=returned_data) + returned_data=returned_data, + search_by=by) return returned_data diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 6160bef7..187d586f 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -21,6 +21,16 @@ DEP_TYPES = { OPTDEPENDS_ID: "OptDepends" } +# Add reversed keys for DEP_TYPES so we can find a dependency type from a type's +# name. +DEP_NAMES = {} +DEP_TYPE_VALUES = list(DEP_TYPES.values()) + +for i in DEP_TYPES.values(): + lowercase_value = i.lower() + current_value_position = DEP_TYPE_VALUES.index(i) + DEP_NAMES[lowercase_value] = list(DEP_TYPES.keys())[current_value_position] + # Define relationship types. REL_TYPES = { CONFLICTS_ID: "Conflicts", @@ -28,6 +38,15 @@ REL_TYPES = { REPLACES_ID: "Replaces" } +# Define search 'by' field types. +SEARCH_BY_TYPES = ["name", + "name-desc", + "maintainer", + "depends", + "makedepends", + "optdepends", + "checkdepends"] + # Define functions for request types. def add_deps(current_array, db_dep): @@ -82,7 +101,7 @@ def add_rels(current_array, db_rel): return current_array -def run_info(returned_data, package_name, snapshot_uri): +def run_info(returned_data, package_name, snapshot_uri, search_by): # Get package name. db_package = db.query(Package).filter(Package.Name == package_name) @@ -153,6 +172,27 @@ def run_info(returned_data, package_name, snapshot_uri): return returned_data +# Define search functions for searching the db for data. +def run_search(returned_data, package_name, snapshot_uri, search_by): + run_info_list = [] + + if search_by in DEP_NAMES.keys(): + dependency_id = DEP_NAMES[search_by] + db_dependency_packages = (db.query(PackageDependency) + .filter(PackageDependency.DepName == package_name) + .filter(PackageDependency.DepTypeID == dependency_id) + .all()) + + for i in db_dependency_packages: + upstream_package_name = db.query(PackageBase).filter(PackageBase.ID == i.PackageID).first().Name + run_info_list += [upstream_package_name] + + for i in run_info_list: + returned_data = run_info(returned_data, i, snapshot_uri, search_by) + + return returned_data + + def RPC(**function_args): # Get arguments. # @@ -161,6 +201,7 @@ def RPC(**function_args): type = function_args.get("type") args = function_args.get("argument_list") returned_data = function_args.get("returned_data") + search_by = function_args.get("search_by") # Get Snapshot URI snapshot_uri = config.get("options", "snapshot_uri") @@ -168,7 +209,8 @@ def RPC(**function_args): # Set request type to run. type_actions = { "info": run_info, - "multiinfo": run_info + "multiinfo": run_info, + "search": run_search } # This if statement should always be executed, as we checked if the @@ -176,6 +218,15 @@ def RPC(**function_args): if type in type_actions: run_request = type_actions.get(type) + # Validate 'by' query is valid when doing a search. + if type == "search": + if search_by is None: + search_by = "name-desc" + elif search_by not in SEARCH_BY_TYPES: + returned_data["type"] = "error" + returned_data["error"] = "Incorrect by field specified" + return returned_data + # If type is 'info', overwrite type to 'multiinfo' to match the # behavior of the PHP implementation. if type == "info": @@ -187,7 +238,11 @@ def RPC(**function_args): args = set(args) for i in args: - returned_data = run_request(returned_data, i, snapshot_uri) + returned_data = run_request(returned_data, i, snapshot_uri, search_by) + + # Return the error message if one was made. + if returned_data.get("type") == "error": + break elif type is None: returned_data["type"] = "error" -- GitLab From 5b7bd8cc724b990a836dc2c30405f4be4acf7ae6 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Tue, 21 Sep 2021 21:00:04 -0500 Subject: [PATCH 2/8] Added name, name-desc, and maintainer 'by' types --- aurweb/routers/rpc.py | 3 ++- aurweb/rpc.py | 43 +++++++++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/aurweb/routers/rpc.py b/aurweb/routers/rpc.py index 90b0f11f..4308d60b 100644 --- a/aurweb/routers/rpc.py +++ b/aurweb/routers/rpc.py @@ -98,6 +98,7 @@ async def rpc(request: Request, type=type, argument_list=argument_list, returned_data=returned_data, - search_by=by) + search_by=by, + request=request) return returned_data diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 187d586f..18d0e9a6 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -12,6 +12,7 @@ from aurweb.models.package_relation import PackageRelation from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID from aurweb.models.user import User +from aurweb.packages.search import PackageSearch # Define dependency types. DEP_TYPES = { @@ -101,7 +102,7 @@ def add_rels(current_array, db_rel): return current_array -def run_info(returned_data, package_name, snapshot_uri, search_by): +def run_info(returned_data, package_name, snapshot_uri, search_by, request): # Get package name. db_package = db.query(Package).filter(Package.Name == package_name) @@ -172,23 +173,40 @@ def run_info(returned_data, package_name, snapshot_uri, search_by): return returned_data +def run_search_dependencies(search_by, package_name, run_info_list): + dependency_id = DEP_NAMES[search_by] + db_dependency_packages = (db.query(PackageDependency) + .filter(PackageDependency.DepName == package_name) + .filter(PackageDependency.DepTypeID == dependency_id) + .all()) + + for i in db_dependency_packages: + upstream_package_name = db.query(PackageBase).filter(PackageBase.ID == i.PackageID).first().Name + run_info_list += [upstream_package_name] + # Define search functions for searching the db for data. -def run_search(returned_data, package_name, snapshot_uri, search_by): +def run_search(returned_data, package_name, snapshot_uri, search_by, request): run_info_list = [] if search_by in DEP_NAMES.keys(): - dependency_id = DEP_NAMES[search_by] - db_dependency_packages = (db.query(PackageDependency) - .filter(PackageDependency.DepName == package_name) - .filter(PackageDependency.DepTypeID == dependency_id) - .all()) + run_search_dependencies(search_by, package_name, run_info_list) + + else: + search = PackageSearch(request.user) + + search_types = { + "name": search._search_by_name, + "name-desc": search._search_by_namedesc, + "maintainer": search._search_by_maintainer + } + + search_runner = search_types.get(search_by) - for i in db_dependency_packages: - upstream_package_name = db.query(PackageBase).filter(PackageBase.ID == i.PackageID).first().Name - run_info_list += [upstream_package_name] + for i in search_runner(package_name).query.all(): + run_info_list += [i.Name] for i in run_info_list: - returned_data = run_info(returned_data, i, snapshot_uri, search_by) + returned_data = run_info(returned_data, i, snapshot_uri, search_by, request) return returned_data @@ -202,6 +220,7 @@ def RPC(**function_args): args = function_args.get("argument_list") returned_data = function_args.get("returned_data") search_by = function_args.get("search_by") + request = function_args.get("request") # Get Snapshot URI snapshot_uri = config.get("options", "snapshot_uri") @@ -238,7 +257,7 @@ def RPC(**function_args): args = set(args) for i in args: - returned_data = run_request(returned_data, i, snapshot_uri, search_by) + returned_data = run_request(returned_data, i, snapshot_uri, search_by, request) # Return the error message if one was made. if returned_data.get("type") == "error": -- GitLab From 95788f7ebd79fdaf0835124854f61308f478da94 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Tue, 21 Sep 2021 21:49:13 -0500 Subject: [PATCH 3/8] Simplified method for passing arguments to functions Added kwargs-styled arguments to all functions so we can more easily add arguments in the future --- aurweb/rpc.py | 62 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 18d0e9a6..03385f51 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -50,7 +50,10 @@ SEARCH_BY_TYPES = ["name", # Define functions for request types. -def add_deps(current_array, db_dep): +def add_deps(**args): + current_array = args["current_array"] + db_dep = args["db_dep"] + if db_dep.count() > 0: # Create lists for all dependency types. for i in DEP_TYPES.values(): @@ -76,7 +79,10 @@ def add_deps(current_array, db_dep): return current_array -def add_rels(current_array, db_rel): +def add_rels(**args): + current_array = args["current_array"] + db_rel = args["db_rel"] + if db_rel.count() > 0: # Create lists for all relationship types. for i in REL_TYPES.values(): @@ -102,7 +108,12 @@ def add_rels(current_array, db_rel): return current_array -def run_info(returned_data, package_name, snapshot_uri, search_by, request): +def run_info(**args): + returned_data = args["returned_data"] + package_name = args["package_name"] + snapshot_uri = args["snapshot_uri"] + search_by = args["search_by"] + # Get package name. db_package = db.query(Package).filter(Package.Name == package_name) @@ -150,11 +161,11 @@ def run_info(returned_data, package_name, snapshot_uri, search_by, request): # Generate dependency listing. db_dep = db.query(PackageDependency).filter(PackageDependency.PackageID == db_package.ID) - current_array = add_deps(current_array, db_dep) + current_array = add_deps(current_array=current_array, db_dep=db_dep) # Generate relationship listing. db_rel = db.query(PackageRelation).filter(PackageRelation.PackageID == db_package.ID) - current_array = add_rels(current_array, db_rel) + current_array = add_rels(current_array=current_array, db_rel=db_rel) # License table. current_array["License"] = [] @@ -173,7 +184,11 @@ def run_info(returned_data, package_name, snapshot_uri, search_by, request): return returned_data -def run_search_dependencies(search_by, package_name, run_info_list): +def run_search_dependencies(**args): + search_by = args["search_by"] + package_name = args["package_name"] + run_info_list = args["run_info_list"] + dependency_id = DEP_NAMES[search_by] db_dependency_packages = (db.query(PackageDependency) .filter(PackageDependency.DepName == package_name) @@ -185,12 +200,18 @@ def run_search_dependencies(search_by, package_name, run_info_list): run_info_list += [upstream_package_name] # Define search functions for searching the db for data. -def run_search(returned_data, package_name, snapshot_uri, search_by, request): +def run_search(**args): + returned_data = args["returned_data"] + package_name = args["package_name"] + snapshot_uri = args["snapshot_uri"] + search_by = args["search_by"] + request = args["request"] + run_info_list = [] if search_by in DEP_NAMES.keys(): run_search_dependencies(search_by, package_name, run_info_list) - + else: search = PackageSearch(request.user) @@ -206,21 +227,20 @@ def run_search(returned_data, package_name, snapshot_uri, search_by, request): run_info_list += [i.Name] for i in run_info_list: - returned_data = run_info(returned_data, i, snapshot_uri, search_by, request) + returned_data = run_info(returned_data=returned_data, + package_name=i, + snapshot_uri=snapshot_uri, + search_by=search_by) return returned_data def RPC(**function_args): - # Get arguments. - # - # We'll use 'v' in the future when we add v6. - # v = function_args.get("v") - type = function_args.get("type") - args = function_args.get("argument_list") - returned_data = function_args.get("returned_data") - search_by = function_args.get("search_by") - request = function_args.get("request") + type = function_args["type"] + args = function_args["argument_list"] + returned_data = function_args["returned_data"] + search_by = function_args["search_by"] + request = function_args["request"] # Get Snapshot URI snapshot_uri = config.get("options", "snapshot_uri") @@ -257,7 +277,11 @@ def RPC(**function_args): args = set(args) for i in args: - returned_data = run_request(returned_data, i, snapshot_uri, search_by, request) + returned_data = run_request(returned_data=returned_data, + package_name=i, + snapshot_uri=snapshot_uri, + search_by=search_by, + request=request) # Return the error message if one was made. if returned_data.get("type") == "error": -- GitLab From 2c56436cb5b37e6522752f85a5efc0b615d5adef Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Tue, 21 Sep 2021 21:58:35 -0500 Subject: [PATCH 4/8] Removed private function usage of packages.search.PackageSearch Moved over to the PackageSearch.search_by() function, which just required specific what we want to search by in addition to the package name. --- aurweb/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 03385f51..4badf19a 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -216,14 +216,14 @@ def run_search(**args): search = PackageSearch(request.user) search_types = { - "name": search._search_by_name, - "name-desc": search._search_by_namedesc, - "maintainer": search._search_by_maintainer + "name": "n", + "name-desc": "nd", + "maintainer": "m" } - search_runner = search_types.get(search_by) + search_id = search_types.get(search_by) - for i in search_runner(package_name).query.all(): + for i in search.search_by(search_id, package_name).query.all(): run_info_list += [i.Name] for i in run_info_list: -- GitLab From eed171744972d8259c2247b0b4f6eed9c4a160e6 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Tue, 21 Sep 2021 22:00:46 -0500 Subject: [PATCH 5/8] Fixed flake8 violations --- aurweb/rpc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 4badf19a..4e8b8a18 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -112,7 +112,6 @@ def run_info(**args): returned_data = args["returned_data"] package_name = args["package_name"] snapshot_uri = args["snapshot_uri"] - search_by = args["search_by"] # Get package name. db_package = db.query(Package).filter(Package.Name == package_name) @@ -199,6 +198,7 @@ def run_search_dependencies(**args): upstream_package_name = db.query(PackageBase).filter(PackageBase.ID == i.PackageID).first().Name run_info_list += [upstream_package_name] + # Define search functions for searching the db for data. def run_search(**args): returned_data = args["returned_data"] @@ -228,9 +228,9 @@ def run_search(**args): for i in run_info_list: returned_data = run_info(returned_data=returned_data, - package_name=i, - snapshot_uri=snapshot_uri, - search_by=search_by) + package_name=i, + snapshot_uri=snapshot_uri, + search_by=search_by) return returned_data -- GitLab From cdb7270943248923645b13c82b06cc332291af8a Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Tue, 21 Sep 2021 23:36:35 -0500 Subject: [PATCH 6/8] Added unit tests for searches Also fixed how arguments are passed to run_search_dependencies() We'll be squashing all of these commits so it shouldn't matter too much that I'm combining like this :) --- aurweb/rpc.py | 8 +- test/{test_rpc.py => test_rpc_info.py} | 0 test/test_rpc_search.py | 353 +++++++++++++++++++++++++ 3 files changed, 356 insertions(+), 5 deletions(-) rename test/{test_rpc.py => test_rpc_info.py} (100%) create mode 100644 test/test_rpc_search.py diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 4e8b8a18..06947891 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -210,7 +210,9 @@ def run_search(**args): run_info_list = [] if search_by in DEP_NAMES.keys(): - run_search_dependencies(search_by, package_name, run_info_list) + run_search_dependencies(search_by=search_by, + package_name=package_name, + run_info_list=run_info_list) else: search = PackageSearch(request.user) @@ -283,10 +285,6 @@ def RPC(**function_args): search_by=search_by, request=request) - # Return the error message if one was made. - if returned_data.get("type") == "error": - break - elif type is None: returned_data["type"] = "error" returned_data["error"] = "No request type/data specified." diff --git a/test/test_rpc.py b/test/test_rpc_info.py similarity index 100% rename from test/test_rpc.py rename to test/test_rpc_info.py diff --git a/test/test_rpc_search.py b/test/test_rpc_search.py new file mode 100644 index 00000000..c6d3da5a --- /dev/null +++ b/test/test_rpc_search.py @@ -0,0 +1,353 @@ +import orjson +import pytest + +from fastapi.testclient import TestClient + +from aurweb.asgi import app +from aurweb.db import begin, create, query +from aurweb.models.account_type import AccountType +from aurweb.models.dependency_type import DependencyType +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.package_dependency import PackageDependency +from aurweb.models.user import User +from aurweb.testing import setup_test_db + + +def make_request(path): + with TestClient(app) as request: + return request.get(path) + + +def remove_bad_keys(dict_data): + for i in dict_data["results"]: + i.pop("ID") + i.pop("PackageBaseID") + i.pop("FirstSubmitted") + i.pop("LastModified") + + return dict_data + + +@pytest.fixture(autouse=True) +def setup(): + setup_test_db("Users", "PackageBases", "Packages", "Licenses", + "PackageDepends", "PackageRelations", "PackageLicenses", + "PackageKeywords", "PackageVotes") + + with begin(): + account_type = query(AccountType, AccountType.AccountType == "User").first() + + dependency_depends = query(DependencyType, DependencyType.Name == "depends").first() + dependency_optdepends = query(DependencyType, DependencyType.Name == "optdepends").first() + dependency_makedepends = query(DependencyType, DependencyType.Name == "makedepends").first() + dependency_checkdepends = query(DependencyType, DependencyType.Name == "checkdepends").first() + + user1 = create(User, + Username="user1", + Email="user1@example.com", + RealName="Test User 1", + Passwd="testPassword", + AccountType=account_type) + + user2 = create(User, + Username="user2", + Email="user2@example.com", + RealName="Test User 2", + Passwd="testPassword", + AccountType=account_type) + + pkgbase1 = create(PackageBase, Name="big-chungus1", Maintainer=user1) + + pkgname1 = create(Package, + PackageBase=pkgbase1, + Name=pkgbase1.Name, + Description="Cookie Monster", + URL="https://example.com/") + + pkgbase2 = create(PackageBase, Name="big-chungus2", Maintainer=user1) + + pkgname2 = create(Package, + PackageBase=pkgbase2, + Name=pkgbase2.Name, + Description="Bunny bunny around bunny", + URL="https://example.com/") + + pkgbase3 = create(PackageBase, Name="small-chungus3", Maintainer=user2) + + pkgname3 = create(Package, + PackageBase=pkgbase3, + Name=pkgbase3.Name, + Description="Bunny bunny around bunny", + URL="https://example.com/") + + pkgbase4 = create(PackageBase, Name="small-chungus4", Maintainer=user1) + + pkgname4 = create(Package, + PackageBase=pkgbase4, + Name=pkgbase4.Name, + Description="Bunny bunny around bunny", + URL="https://example.com/") + + create(PackageDependency, + Package=pkgname1, + DependencyType=dependency_depends, + DepName="chungus-depends") + + create(PackageDependency, + Package=pkgname2, + DependencyType=dependency_optdepends, + DepName="chungus-optdepends") + + create(PackageDependency, + Package=pkgname3, + DependencyType=dependency_makedepends, + DepName="chungus-makedepends") + + create(PackageDependency, + Package=pkgname4, + DependencyType=dependency_checkdepends, + DepName="chungus-checkdepends") + + +def test_name(): + expected_data = { + 'version': 5, + 'results': [ + { + 'Name': 'big-chungus1', + 'Version': '', + 'Description': 'Cookie Monster', + 'URL': 'https://example.com/', + 'PackageBase': 'big-chungus1', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user1', + 'URLPath': '/cgit/aur.git/snapshot/big-chungus1.tar.gz', + 'Depends': ['chungus-depends'], + 'License': [], + 'Keywords': [] + }, + { + 'Name': 'big-chungus2', + 'Version': '', + 'Description': 'Bunny bunny around bunny', + 'URL': 'https://example.com/', + 'PackageBase': 'big-chungus2', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user1', + 'URLPath': '/cgit/aur.git/snapshot/big-chungus2.tar.gz', + 'OptDepends': ['chungus-optdepends'], + 'License': [], + 'Keywords': [] + } + ], + 'resultcount': 2, + 'type': 'search' + } + + returned_data = make_request("/rpc/?v=5&type=search&by=name&arg=big-chungus").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data + + +def test_name_desc(): + expected_data = { + 'version': 5, + 'results': [{ + 'Name': 'big-chungus1', + 'Version': '', + 'Description': 'Cookie Monster', + 'URL': 'https://example.com/', + 'PackageBase': 'big-chungus1', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user1', + 'URLPath': '/cgit/aur.git/snapshot/big-chungus1.tar.gz', + 'Depends': ['chungus-depends'], + 'License': [], + 'Keywords': [] + }], + 'resultcount': 1, + 'type': 'search' + } + + # Specifying no 'by' field default's the field to 'name-desc'. + returned_data1 = make_request("/rpc/?v=5&type=search&arg=cookie").content.decode() + returned_data2 = make_request("/rpc/?v=5&type=search&arg=cookie&by=name-desc").content.decode() + + returned_data1 = orjson.loads(returned_data1) + returned_data2 = orjson.loads(returned_data2) + + returned_data1 = remove_bad_keys(returned_data1) + returned_data2 = remove_bad_keys(returned_data2) + + for i in [returned_data1, returned_data2]: + assert i == expected_data + + +def test_maintainer(): + expected_data = { + 'version': 5, + 'results': [{ + 'Name': 'small-chungus3', + 'Version': '', + 'Description': 'Bunny bunny around bunny', + 'URL': 'https://example.com/', + 'PackageBase': 'small-chungus3', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user2', + 'URLPath': '/cgit/aur.git/snapshot/small-chungus3.tar.gz', + 'MakeDepends': ['chungus-makedepends'], + 'License': [], + 'Keywords': [] + }], + 'resultcount': 1, + 'type': 'search' + } + + returned_data = make_request("/rpc?v=5&type=search&by=maintainer&arg=user2").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data + + +def test_depends(): + expected_data = { + 'version': 5, + 'results': [{ + 'Name': 'big-chungus1', + 'Version': '', + 'Description': 'Cookie Monster', + 'URL': 'https://example.com/', + 'PackageBase': 'big-chungus1', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user1', + 'URLPath': '/cgit/aur.git/snapshot/big-chungus1.tar.gz', + 'Depends': ['chungus-depends'], + 'License': [], + 'Keywords': [] + }], + 'resultcount': 1, + 'type': 'search' + } + + returned_data = make_request("/rpc?v=5&type=search&by=depends&arg=chungus-depends").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data + + +def test_optdepends(): + expected_data = { + 'version': 5, + 'results': [{ + 'Name': 'big-chungus2', + 'Version': '', + 'Description': 'Bunny bunny around bunny', + 'URL': 'https://example.com/', + 'PackageBase': 'big-chungus2', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user1', + 'URLPath': '/cgit/aur.git/snapshot/big-chungus2.tar.gz', + 'OptDepends': ['chungus-optdepends'], + 'License': [], + 'Keywords': [] + }], + 'resultcount': 1, + 'type': 'search' + } + + returned_data = make_request("/rpc?v=5&type=search&by=optdepends&arg=chungus-optdepends").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data + + +def test_makedepends(): + expected_data = { + 'version': 5, + 'results': [{ + 'Name': 'small-chungus3', + 'Version': '', + 'Description': 'Bunny bunny around bunny', + 'URL': 'https://example.com/', + 'PackageBase': 'small-chungus3', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user2', + 'URLPath': '/cgit/aur.git/snapshot/small-chungus3.tar.gz', + 'MakeDepends': ['chungus-makedepends'], + 'License': [], + 'Keywords': [] + }], + 'resultcount': 1, + 'type': 'search' + } + + returned_data = make_request("/rpc?v=5&type=search&by=makedepends&arg=chungus-makedepends").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data + + +def test_checkdepends(): + expected_data = { + 'version': 5, + 'results': [{ + 'Name': 'small-chungus4', + 'Version': '', + 'Description': 'Bunny bunny around bunny', + 'URL': 'https://example.com/', + 'PackageBase': 'small-chungus4', + 'NumVotes': 0, + 'Popularity': 0.0, + 'OutOfDate': None, + 'Maintainer': 'user1', + 'URLPath': '/cgit/aur.git/snapshot/small-chungus4.tar.gz', + 'CheckDepends': ['chungus-checkdepends'], + 'License': [], + 'Keywords': [] + }], + 'resultcount': 1, + 'type': 'search' + } + + returned_data = make_request("/rpc?v=5&type=search&by=checkdepends&arg=chungus-checkdepends").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data + + +def test_bad_by_field(): + expected_data = { + 'version': 5, + 'results': [], + 'resultcount': 0, + 'type': 'error', + 'error': 'Incorrect by field specified' + } + + returned_data = make_request("/rpc?v=5&type=search&by=asdf&arg=chungus-checkdepends").content.decode() + returned_data = orjson.loads(returned_data) + returned_data = remove_bad_keys(returned_data) + + assert returned_data == expected_data -- GitLab From 87a9390acdcabe3cc69aef800a23df3e46bdfe50 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Wed, 22 Sep 2021 21:08:39 -0500 Subject: [PATCH 7/8] Fixed incorrect class used to check for RPC results when using dependency-based 'by' fields --- aurweb/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index 06947891..bc348cf8 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -195,7 +195,7 @@ def run_search_dependencies(**args): .all()) for i in db_dependency_packages: - upstream_package_name = db.query(PackageBase).filter(PackageBase.ID == i.PackageID).first().Name + upstream_package_name = db.query(Package).filter(Package.ID == i.PackageID).first().Name run_info_list += [upstream_package_name] -- GitLab From 7e12b8035d85bf751e3a97ad01d154194ed56828 Mon Sep 17 00:00:00 2001 From: Hunter Wittenborn Date: Sat, 25 Sep 2021 15:54:40 -0500 Subject: [PATCH 8/8] Removed unneeded database queries from RPC interface Also modified returned data for test_rpc_no_dependencies() in test/test_rpc_info.py, as there was a key that contained an incorrect value for some reason. --- aurweb/rpc.py | 67 +++++++++++++++++++------------------------ test/test_rpc_info.py | 2 +- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/aurweb/rpc.py b/aurweb/rpc.py index bc348cf8..015a9bbd 100644 --- a/aurweb/rpc.py +++ b/aurweb/rpc.py @@ -2,16 +2,9 @@ import aurweb.config as config from aurweb import db from aurweb.models.dependency_type import CHECKDEPENDS_ID, DEPENDS_ID, MAKEDEPENDS_ID, OPTDEPENDS_ID -from aurweb.models.license import License from aurweb.models.package import Package -from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency -from aurweb.models.package_keyword import PackageKeyword -from aurweb.models.package_license import PackageLicense -from aurweb.models.package_relation import PackageRelation -from aurweb.models.package_vote import PackageVote from aurweb.models.relation_type import CONFLICTS_ID, PROVIDES_ID, REPLACES_ID -from aurweb.models.user import User from aurweb.packages.search import PackageSearch # Define dependency types. @@ -109,28 +102,27 @@ def add_rels(**args): def run_info(**args): - returned_data = args["returned_data"] - package_name = args["package_name"] - snapshot_uri = args["snapshot_uri"] + returned_data = args.get("returned_data") + package_name = args.get("package_name") + snapshot_uri = args.get("snapshot_uri") + package_object = args.get("package_object") # Get package name. - db_package = db.query(Package).filter(Package.Name == package_name) - - if db_package.count() == 0: - return returned_data + if package_object is None: + db_package = db.query(Package).filter(Package.Name == package_name) - db_package = db_package.first() + if db_package.count() == 0: + return returned_data - # Get name of package under PackageBaseID. - db_package_baseid = db.query(PackageBase).filter(PackageBase.ID == db_package.PackageBaseID).first() + db_package = db_package.first() - # Get maintainer info. - db_package_maintainer = db.query(User).filter(User.ID == db_package_baseid.MaintainerUID).first() + else: + db_package = package_object + # Start building our array. current_array = {} returned_data["resultcount"] = returned_data["resultcount"] + 1 - # Data from the Packages table. current_array["ID"] = db_package.ID current_array["Name"] = db_package.Name current_array["PackageBaseID"] = db_package.PackageBaseID @@ -138,7 +130,7 @@ def run_info(**args): current_array["Description"] = db_package.Description current_array["URL"] = db_package.URL - # PackageBase table. + db_package_baseid = db_package.PackageBase current_array["PackageBase"] = db_package_baseid.Name current_array["NumVotes"] = db_package_baseid.NumVotes current_array["Popularity"] = db_package_baseid.Popularity @@ -146,36 +138,37 @@ def run_info(**args): current_array["FirstSubmitted"] = db_package_baseid.SubmittedTS current_array["LastModified"] = db_package_baseid.ModifiedTS - # User table. - try: + # Maintainer. + db_package_maintainer = db_package.PackageBase.Maintainer + + if db_package_maintainer is not None: current_array["Maintainer"] = db_package_maintainer.Username - except AttributeError: + else: current_array["Maintainer"] = None # Generate and add snapshot_uri. current_array["URLPath"] = snapshot_uri.replace("%s", package_name) # Add package votes. - current_array["NumVotes"] = db.query(PackageVote).count() + current_array["NumVotes"] = db_package.PackageBase.package_votes.count() # Generate dependency listing. - db_dep = db.query(PackageDependency).filter(PackageDependency.PackageID == db_package.ID) - current_array = add_deps(current_array=current_array, db_dep=db_dep) + current_array = add_deps(current_array=current_array, db_dep=db_package.package_dependencies) # Generate relationship listing. - db_rel = db.query(PackageRelation).filter(PackageRelation.PackageID == db_package.ID) - current_array = add_rels(current_array=current_array, db_rel=db_rel) + current_array = add_rels(current_array=current_array, db_rel=db_package.package_relations) # License table. current_array["License"] = [] + db_package_license = db_package.package_license - for i in db.query(PackageLicense).filter(PackageLicense.PackageID == db_package.ID): - current_array["License"] += [db.query(License).first().Name] + if db_package_license is not None: + current_array["License"] += [db_package_license.License.Name] # Keywords table. current_array["Keywords"] = [] - for i in db.query(PackageKeyword).filter(PackageKeyword.PackageBaseID == db_package_baseid.ID): + for i in db_package.PackageBase.keywords.all(): current_array["Keywords"] += [i.Keyword] # Add current array to returned results. @@ -195,8 +188,7 @@ def run_search_dependencies(**args): .all()) for i in db_dependency_packages: - upstream_package_name = db.query(Package).filter(Package.ID == i.PackageID).first().Name - run_info_list += [upstream_package_name] + run_info_list += [i.Package] # Define search functions for searching the db for data. @@ -226,13 +218,14 @@ def run_search(**args): search_id = search_types.get(search_by) for i in search.search_by(search_id, package_name).query.all(): - run_info_list += [i.Name] + run_info_list += [i] for i in run_info_list: returned_data = run_info(returned_data=returned_data, - package_name=i, + package_name=i.Name, snapshot_uri=snapshot_uri, - search_by=search_by) + search_by=search_by, + package_object=i) return returned_data diff --git a/test/test_rpc_info.py b/test/test_rpc_info.py index 21817b45..5eb3a854 100644 --- a/test/test_rpc_info.py +++ b/test/test_rpc_info.py @@ -278,7 +278,7 @@ def test_rpc_no_dependencies(): 'Description': 'Wubby wubby on wobba wuubu', 'URL': 'https://example.com/', 'PackageBase': 'chungy-chungus', - 'NumVotes': 3, + 'NumVotes': 0, 'Popularity': 0.0, 'OutOfDate': None, 'Maintainer': 'user1', -- GitLab