Commit bb6b7716 authored by David Runge's avatar David Runge
Browse files

Merge branch 'issues/15' into 'master'

Implement repo_management async

Closes #15

See merge request !8
parents bee1142b b51be116
Pipeline #9277 failed with stage
in 32 seconds
[[package]]
name = "aiofiles"
version = "0.6.0"
description = "File support for asyncio."
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "appdirs"
version = "1.4.4"
......@@ -308,6 +316,20 @@ toml = "*"
[package.extras]
testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
[[package]]
name = "pytest-asyncio"
version = "0.14.0"
description = "Pytest support for asyncio."
category = "dev"
optional = false
python-versions = ">= 3.5"
[package.dependencies]
pytest = ">=5.4.0"
[package.extras]
testing = ["async-generator (>=1.3)", "coverage", "hypothesis (>=5.7.1)"]
[[package]]
name = "regex"
version = "2020.11.13"
......@@ -403,9 +425,13 @@ testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)",
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "4b85406328a00d6d6cc7d9652a65481a106ee7e2048b9cae5f900b199003c638"
content-hash = "0b37a55b176431b27f12dbe7ca4428997b8c4cace23534cc677c07b980992282"
[metadata.files]
aiofiles = [
{file = "aiofiles-0.6.0-py3-none-any.whl", hash = "sha256:bd3019af67f83b739f8e4053c6c0512a7f545b9a8d91aaeab55e6e0f9d123c27"},
{file = "aiofiles-0.6.0.tar.gz", hash = "sha256:e0281b157d3d5d59d803e3f4557dcc9a3dff28a4dd4829a9ff478adae50ca092"},
]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
......@@ -662,6 +688,10 @@ pytest = [
{file = "pytest-6.2.2-py3-none-any.whl", hash = "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"},
{file = "pytest-6.2.2.tar.gz", hash = "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9"},
]
pytest-asyncio = [
{file = "pytest-asyncio-0.14.0.tar.gz", hash = "sha256:9882c0c6b24429449f5f969a5158b528f39bde47dc32e85b9f0403965017e700"},
{file = "pytest_asyncio-0.14.0-py3-none-any.whl", hash = "sha256:2eae1e34f6c68fc0a9dc12d4bea190483843ff4708d24277c41568d6b6044f1d"},
]
regex = [
{file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"},
......
......@@ -18,6 +18,7 @@ pydantic = "^1.8.1"
orjson = "^3.5.1"
Jinja2 = "^2.11.3"
subprocess-tee = "^0.2.0"
aiofiles = "^0.6.0"
[tool.poetry.dev-dependencies]
tox = "^3.23.0"
......@@ -28,6 +29,7 @@ flake8 = "^3.8.4"
black = "^20.8b1"
coverage = "^5.5"
mock = "^4.0.3"
pytest-asyncio = "^0.14.0"
[tool.poetry.scripts]
db2json = "repo_management.cli:db2json"
......
import asyncio
from argparse import ArgumentTypeError
from sys import exit
......@@ -13,9 +14,11 @@ def db2json() -> None:
try:
args = argparse.ArgParseFactory.db2json().parse_args()
operations.dump_db_to_json_files(
input_path=args.db_file,
output_path=args.output_dir,
asyncio.run(
operations.dump_db_to_json_files(
input_path=args.db_file,
output_path=args.output_dir,
)
)
except (errors.RepoManagementError, ArgumentTypeError) as e:
print(e)
......@@ -31,10 +34,12 @@ def json2db() -> None:
try:
args = argparse.ArgParseFactory.json2db().parse_args()
operations.create_db_from_json_files(
input_path=args.input_dir,
output_path=args.db_file,
db_type=defaults.RepoDbType.FILES if args.files else defaults.RepoDbType.DEFAULT,
asyncio.run(
operations.create_db_from_json_files(
input_path=args.input_dir,
output_path=args.db_file,
db_type=defaults.RepoDbType.FILES if args.files else defaults.RepoDbType.DEFAULT,
)
)
except (errors.RepoManagementError, ArgumentTypeError) as e:
print(e)
......
import io
from typing import Dict, List, Optional, Union
from typing import Dict, List, Union
from jinja2 import Environment, PackageLoader
from pydantic.error_wrappers import ValidationError
......@@ -7,7 +7,7 @@ from pydantic.error_wrappers import ValidationError
from repo_management import defaults, errors, models
def _files_data_to_model(data: io.StringIO) -> models.Files:
async def _files_data_to_model(data: io.StringIO) -> models.Files:
"""Read the contents of a 'files' file (represented as an instance of
io.StringIO) and convert it to a pydantic model
......@@ -46,7 +46,7 @@ def _files_data_to_model(data: io.StringIO) -> models.Files:
return models.Files(**output)
def _desc_data_line_to_dicts(
async def _desc_data_line_to_dicts(
current_header: str,
current_type: defaults.FieldType,
line: str,
......@@ -88,7 +88,7 @@ def _desc_data_line_to_dicts(
int_types[current_header] = int(line)
def _desc_data_to_model(data: io.StringIO) -> models.PackageDesc:
async def _desc_data_to_model(data: io.StringIO) -> models.PackageDesc:
"""Read the contents of a 'desc' file (represented as an instance of io.StringIO) and convert it to a pydantic model
Parameters
......@@ -126,7 +126,7 @@ def _desc_data_to_model(data: io.StringIO) -> models.PackageDesc:
if current_header:
try:
_desc_data_line_to_dicts(
await _desc_data_line_to_dicts(
current_header=current_header,
current_type=current_type,
line=line,
......@@ -148,37 +148,6 @@ def _desc_data_to_model(data: io.StringIO) -> models.PackageDesc:
)
def _transform_package_desc_to_output_package(
desc: models.PackageDesc, files: Optional[models.Files]
) -> models.OutputPackage:
"""Transform a PackageDesc model and an accompanying Files model to an OutputPackage model
Parameters
----------
desc: models.PackageDesc
A pydantic model, that has all required attributes (apart from the list of files) to create an OutputPackage
model
files: models.Files:
A pydantic model, that represents the list of files, that belong to the package described by desc
Returns
-------
models.OutputPackage
A pydantic model, that describes a package and its list of files
"""
desc_dict = desc.dict()
# remove attributes, that are represented on the pkgbase level
for name in ["base", "makedepends", "packager", "version"]:
if desc_dict.get(name):
del desc_dict[name]
if files:
return models.OutputPackage(**desc_dict, **files.dict())
else:
return models.OutputPackage(**desc_dict)
class RepoDbFile:
"""A class for handling templates for files used in repository database files (such as 'desc' or 'files')
......@@ -189,7 +158,7 @@ class RepoDbFile:
"""
def __init__(self, enable_async: bool = False) -> None:
def __init__(self, enable_async: bool = True) -> None:
"""Initialize an instance of RepDbFile
Parameters
......@@ -205,7 +174,7 @@ class RepoDbFile:
enable_async=enable_async,
)
def render_desc_template(self, model: models.PackageDesc, output: io.StringIO) -> None:
async def render_desc_template(self, model: models.PackageDesc, output: io.StringIO) -> None:
"""Use the 'desc' template to write a string to an output stream based on a model
Parameters
......@@ -217,9 +186,9 @@ class RepoDbFile:
"""
template = self.env.get_template("desc.j2")
output.write(template.render(model.dict()))
output.write(await template.render_async(model.dict()))
def render_files_template(self, model: models.Files, output: io.StringIO) -> None:
async def render_files_template(self, model: models.Files, output: io.StringIO) -> None:
"""Use the 'files' template to write a string to an output stream based on a model
Parameters
......@@ -231,4 +200,4 @@ class RepoDbFile:
"""
template = self.env.get_template("files.j2")
output.write(template.render(model.dict()))
output.write(await template.render_async(model.dict()))
......@@ -3,15 +3,16 @@ import re
import tarfile
import time
from pathlib import Path
from typing import Iterator
from typing import AsyncIterator
import aiofiles
import orjson
from pydantic.error_wrappers import ValidationError
from repo_management import convert, defaults, errors, models
def _read_db_file(db_path: Path, compression: str = "gz") -> tarfile.TarFile:
async def _read_db_file(db_path: Path, compression: str = "gz") -> tarfile.TarFile:
"""Read a repository database file
Parameters
......@@ -39,7 +40,7 @@ def _read_db_file(db_path: Path, compression: str = "gz") -> tarfile.TarFile:
return tarfile.open(name=db_path, mode=f"r:{compression}")
def _extract_db_member_package_name(name: str) -> str:
async def _extract_db_member_package_name(name: str) -> str:
"""Extract and return the package name from a repository database member name
Parameters
......@@ -55,9 +56,9 @@ def _extract_db_member_package_name(name: str) -> str:
return "".join(re.split("(-)", re.sub("(/desc|/files)$", "", name))[:-4])
def _db_file_member_as_model(
async def _db_file_member_as_model(
db_file: tarfile.TarFile, regex: str = "(/desc|/files)$"
) -> Iterator[models.RepoDbMemberData]:
) -> AsyncIterator[models.RepoDbMemberData]:
"""Iterate over the members of a database file, represented by an instance of tarfile.TarFile and yield the members
as instances of models.RepoDbMemberData
......@@ -82,7 +83,7 @@ def _db_file_member_as_model(
yield models.RepoDbMemberData(
member_type=file_type,
name=_extract_db_member_package_name(name=name),
name=await _extract_db_member_package_name(name=name),
data=io.StringIO(
io.BytesIO(
db_file.extractfile(name).read(), # type: ignore
......@@ -93,7 +94,7 @@ def _db_file_member_as_model(
)
def _json_files_in_directory(path: Path) -> Iterator[Path]:
async def _json_files_in_directory(path: Path) -> AsyncIterator[Path]:
"""Yield JSON files found in a directory
Parameters
......@@ -108,7 +109,7 @@ def _json_files_in_directory(path: Path) -> Iterator[Path]:
Returns
-------
Iterator[Path]
AsyncIterator[Path]
An iterator over the files found in the directory defined by path
"""
......@@ -120,7 +121,7 @@ def _json_files_in_directory(path: Path) -> Iterator[Path]:
yield json_file
def _read_pkgbase_json_file(path: Path) -> models.OutputPackageBase:
async def _read_pkgbase_json_file(path: Path) -> models.OutputPackageBase:
"""Read a JSON file that represents a pkgbase and return it as models.OutputPackageBase
Parameters
......@@ -141,16 +142,16 @@ def _read_pkgbase_json_file(path: Path) -> models.OutputPackageBase:
A pydantic model representing a pkgbase
"""
with open(path, "r") as input_file:
async with aiofiles.open(path, "r") as input_file:
try:
return models.OutputPackageBase(**orjson.loads(input_file.read()))
return models.OutputPackageBase(**orjson.loads(await input_file.read()))
except orjson.JSONDecodeError as e:
raise errors.RepoManagementFileError(f"The JSON file '{path}' could not be decoded!\n{e}")
except ValidationError as e:
raise errors.RepoManagementValidationError(f"The JSON file '{path}' could not be validated!\n{e}")
def _write_db_file(path: Path, compression: str = "gz") -> tarfile.TarFile:
async def _write_db_file(path: Path, compression: str = "gz") -> tarfile.TarFile:
"""Open a repository database file for writing
Parameters
......@@ -178,7 +179,7 @@ def _write_db_file(path: Path, compression: str = "gz") -> tarfile.TarFile:
return tarfile.open(name=path, mode=f"w:{compression}")
def _stream_package_base_to_db(
async def _stream_package_base_to_db(
db: tarfile.TarFile,
model: models.OutputPackageBase,
repodbfile: convert.RepoDbFile,
......@@ -198,7 +199,7 @@ def _stream_package_base_to_db(
The type of database to stream to
"""
for (desc_model, files_model) in model.get_packages_as_models():
for (desc_model, files_model) in await model.get_packages_as_models():
dirname = f"{desc_model.name}-{model.version}"
directory = tarfile.TarInfo(dirname)
directory.type = tarfile.DIRTYPE
......@@ -209,7 +210,7 @@ def _stream_package_base_to_db(
db.addfile(directory)
desc_content = io.StringIO()
repodbfile.render_desc_template(model=desc_model, output=desc_content)
await repodbfile.render_desc_template(model=desc_model, output=desc_content)
desc_file = tarfile.TarInfo(f"{dirname}/desc")
desc_file.size = len(desc_content.getvalue().encode())
desc_file.mtime = int(time.time())
......@@ -219,7 +220,7 @@ def _stream_package_base_to_db(
db.addfile(desc_file, io.BytesIO(desc_content.getvalue().encode()))
if db_type == defaults.RepoDbType.FILES:
files_content = io.StringIO()
repodbfile.render_files_template(model=files_model, output=files_content)
await repodbfile.render_files_template(model=files_model, output=files_content)
files_file = tarfile.TarInfo(f"{dirname}/files")
files_file.size = len(files_content.getvalue().encode())
files_file.mtime = int(time.time())
......
......@@ -717,7 +717,7 @@ class OutputPackageBase(
packages: List[OutputPackage]
def get_packages_as_models(self) -> List[Tuple[PackageDesc, Files]]:
async def get_packages_as_models(self) -> List[Tuple[PackageDesc, Files]]:
"""Return the list of packages as tuples of PackageDesc and Files models
Returns
......
from os.path import join
from pathlib import Path
from typing import Dict, Iterator, Tuple
from typing import AsyncIterator, Dict, Tuple
import aiofiles
import orjson
from repo_management import convert, defaults, files, models
def db_file_as_models(db_path: Path, compression: str = "gz") -> Iterator[Tuple[str, models.OutputPackageBase]]:
async def db_file_as_models(
db_path: Path, compression: str = "gz"
) -> AsyncIterator[Tuple[str, models.OutputPackageBase]]:
"""Read a repository database and yield the name of each pkgbase and the respective data (represented as an instance
of models.OutputPackageBase) in a Tuple.
......@@ -28,11 +31,13 @@ def db_file_as_models(db_path: Path, compression: str = "gz") -> Iterator[Tuple[
packages: Dict[str, models.OutputPackageBase] = {}
package_descs: Dict[str, models.PackageDesc] = {}
package_files: Dict[str, models.Files] = {}
for member in files._db_file_member_as_model(db_file=files._read_db_file(db_path=db_path, compression=compression)):
async for member in files._db_file_member_as_model( # pragma: no cover
db_file=await files._read_db_file(db_path=db_path, compression=compression)
):
if member.member_type == defaults.RepoDbMemberType.DESC:
package_descs.update({member.name: convert._desc_data_to_model(member.data)})
package_descs.update({member.name: await convert._desc_data_to_model(member.data)})
if member.member_type == defaults.RepoDbMemberType.FILES:
package_files.update({member.name: convert._files_data_to_model(member.data)})
package_files.update({member.name: await convert._files_data_to_model(member.data)})
for (name, package_desc) in package_descs.items():
if packages.get(package_desc.base):
......@@ -54,7 +59,7 @@ def db_file_as_models(db_path: Path, compression: str = "gz") -> Iterator[Tuple[
yield (name, package)
def dump_db_to_json_files(input_path: Path, output_path: Path) -> None:
async def dump_db_to_json_files(input_path: Path, output_path: Path) -> None:
"""Read a repository database file and dump each pkgbase contained in it to a separate JSON file below a defined
output directory
......@@ -66,16 +71,16 @@ def dump_db_to_json_files(input_path: Path, output_path: Path) -> None:
A directory in which to
"""
for name, model in db_file_as_models(db_path=input_path):
with open(join(output_path, f"{name}.json"), "wb") as output_file:
output_file.write(
async for name, model in db_file_as_models(db_path=input_path):
async with aiofiles.open(join(output_path, f"{name}.json"), "wb") as output_file:
await output_file.write(
orjson.dumps(
model.dict(), option=orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE | orjson.OPT_SORT_KEYS
)
)
def create_db_from_json_files(
async def create_db_from_json_files(
input_path: Path, output_path: Path, db_type: defaults.RepoDbType = defaults.RepoDbType.DEFAULT
) -> None:
"""Create a repository database from a list of JSON files found in a directory
......@@ -93,10 +98,10 @@ def create_db_from_json_files(
"""
repodbfile = convert.RepoDbFile()
database = files._write_db_file(path=output_path)
for path in files._json_files_in_directory(path=input_path):
model = files._read_pkgbase_json_file(path)
files._stream_package_base_to_db(
database = await files._write_db_file(path=output_path)
async for path in files._json_files_in_directory(path=input_path): # pragma: no cover
model = await files._read_pkgbase_json_file(path)
await files._stream_package_base_to_db(
db=database,
model=model,
repodbfile=repodbfile,
......
......@@ -21,6 +21,7 @@ def test__print_env(env: Optional[Dict[str, str]]) -> None:
(["cd", "-f"], {"FOO": "BAR"}, True, False, False, True, None, raises(CalledProcessError)),
],
)
@mark.asyncio
def test_run_command(
cmd: Union[str, List[str]],
env: Optional[Dict[str, str]],
......
......@@ -20,12 +20,13 @@ RESOURCES = join(dirname(realpath(__file__)), "resources")
("usr/%FILES%\nusr/lib/\n", raises(RuntimeError)),
],
)
def test__files_data_to_dict(
@mark.asyncio
async def test__files_data_to_model(
file_data: str,
expectation: ContextManager[str],
) -> None:
with expectation:
assert convert._files_data_to_model(data=io.StringIO(file_data))
assert await convert._files_data_to_model(data=io.StringIO(file_data))
@mark.parametrize(
......@@ -99,81 +100,26 @@ def test__files_data_to_dict(
),
],
)
def test__desc_data_to_dict(
@mark.asyncio
async def test__desc_data_to_model(
file_data: str,
expectation: ContextManager[str],
) -> None:
with expectation:
assert convert._desc_data_to_model(data=io.StringIO(file_data))
@mark.parametrize(
"desc, files",
[
(
models.PackageDesc(
arch="foo",
base="foo",
builddate=1,
csize=1,
desc="foo",
filename="foo",
isize=1,
license=["foo"],
md5sum="foo",
name="foo",
packager="foo",
pgpsig="foo",
sha256sum="foo",
url="foo",
version="foo",
),
models.Files(files=["foo", "bar"]),
),
(
models.PackageDesc(
arch="foo",
base="foo",
builddate=1,
csize=1,
desc="foo",
filename="foo",
isize=1,
license=["foo"],
md5sum="foo",
name="foo",
packager="foo",
pgpsig="foo",
sha256sum="foo",
url="foo",
version="foo",
),
None,
),
],
)
def test__transform_package_desc_to_output_package(
desc: models.PackageDesc,
files: models.Files,
) -> None:
output = convert._transform_package_desc_to_output_package(desc=desc, files=files)
assert isinstance(output, models.OutputPackage)
if files:
assert output.files
else:
assert not output.files
assert await convert._desc_data_to_model(data=io.StringIO(file_data))
def test_repodbfile__init() -> None:
assert convert.RepoDbFile()
def test_repodbfile_render_desc_template() -> None:
@mark.asyncio
async def test_repodbfile_render_desc_template() -> None:
repodbfile = convert.RepoDbFile()
assert repodbfile
output = io.StringIO()
assert not output.getvalue()
repodbfile.render_desc_template(
await repodbfile.render_desc_template(
model=models.PackageDesc(
arch="foo",
base="foo",
......@@ -196,12 +142,13 @@ def test_repodbfile_render_desc_template() -> None:
assert output.getvalue()
def test_repodbfile_render_files_template() -> None:
@mark.asyncio
async def test_repodbfile_render_files_template() -> None:
repodbfile = convert.RepoDbFile()
assert repodbfile
output = io.StringIO()
assert not output.getvalue()
repodbfile.render_files_template(
await repodbfile.render_files_template(
model=models.Files(files=["foo", "bar"]),
output=output,
)
......
......@@ -68,28 +68,33 @@ def invalid_json_file() -> Iterator[Path]:
yield Path(json_file)
def test__read_db_file(create_gz_db_file: Path) -> None:
@mark.asyncio
async def test__read_db_file(create_gz_db_file: Path) -> None:
with does_not_raise():
assert files._read_db_file(create_gz_db_file)
assert await files._read_db_file(create_gz_db_file)
def test__read_db_file_wrong_compression(create_gz_db_file: Path) -> None:
@mark.asyncio
async def test__read_db_file_wrong_compression(create_gz_db_file: Path) -> None:
with raises(tarfile.CompressionError):
assert files._read_db_file(create_gz_db_file, compression="foo")
assert await files._read_db_file(create_gz_db_file, compression="foo")
def test__read_db_file_does_not_exist(create_null_db_file: Path) -> None:
@mark.asyncio
async def test__read_db_file_does_not_exist(create_null_db_file: Path) -> None:
with raises(FileNotFoundError):
assert files._read_db_file(create_null_db_file)
assert await files._read_db_file(create_null_db_file)
def test__read_db_file_wrong_db_compression(create_bzip_db_file: Path) -> None:
@mark.asyncio
async def test__read_db_file_wrong_db_compression(create_bzip_db_file: Path) -> None:
with raises(tarfile.ReadError):
assert files._read_db_file(create_bzip_db_file)
assert await files._read_db_file(create_bzip_db_file)
def test__read_db_file_member_as_model(create_gz_db_file: Path) -> None:
for member in files._db_file_member_as_model(db_file=files._read_db_file(create_gz_db_file)):
@mark.asyncio
async def test__read_db_file_member_as_model(create_gz_db_file: Path) -> None:
async for member in files._db_file_member_as_model(db_file=await files._read_db_file(create_gz_db_file)):
assert isinstance(member, models.RepoDbMemberData)
......@@ -104,31 +109,35 @@ def test__read_db_file_member_as_model(create_gz_db_file: Path) -> None:
("foobar-1.0.0-42/files", "foobar"),
],
)
def test__extract_db_member_package_name(