Verified Commit 598f81b5 authored by David Runge's avatar David Runge
Browse files

config: Add integration to configure synchronization

arch_release_promotion/config.py:
Add a `SyncConfig` pydantic model to track a global or per project
`backlog` (amount of versions synced), `directory` (with which to
synchronize), `last_updated_file` (to which to write a timestamp upon
changes in the synchronization directory) and `temp_in_sync_dir` (a bool
to set whether temporary data is downloaded to the synchronization
directory).
Add an optional instance of SyncConfig to both `ProjectConfig` and
`Projects`.
Add a root_validator to `Projects` to allow overriding the `SyncConfig`
of each project with implicit or global defaults if no `SyncConfig` is
provided.
Add validator for `Projects.projects` to ensure, that all
`ReleaseConfig` names of each `ProjectConfig` instances are unique.

tests/test_config.py:
Extend tests for `Projects` in accordance with changes to models.
parent 086fd12e
from collections import Counter
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import toml
from dotenv import dotenv_values
from email_validator import EmailNotValidError, validate_email
from pydantic import BaseModel, BaseSettings, Extra, validator
from pydantic import BaseModel, BaseSettings, Extra, root_validator, validator
from pydantic.env_settings import SettingsSourceCallable
from xdg.BaseDirectory import xdg_config_home
......@@ -19,6 +20,9 @@ PROJECTS_CONFIGS = [
Path(f"{xdg_config_home}/arch-release-promotion/projects.toml"),
]
PROJECTS_SYNC_DIR = Path("/var/lib/arch-release-promotion/")
PROJECTS_SYNC_BACKLOG = 3
class ReleaseConfig(BaseModel):
"""A pydantic model describing the configuration of a project's release
......@@ -50,6 +54,31 @@ class ReleaseConfig(BaseModel):
create_torrent: bool = False
class SyncConfig(BaseModel):
"""A pydantic model describing configuration for synchronization
Attributes
----------
backlog: int
The backlog of releases to retain at a maximum (defaults to PROJECTS_SYNC_BACKLOG when a SysConfig instance is
used in a Projects instance or a ProjectConfig instance).
directory: Path
A directory into which to sync project release types and their respective releases (defaults to
PROJECTS_SYNC_DIR when a SysConfig instance is used in a Projects instance or a ProjectConfig instance).
last_updated_file: Optional[Path]
The optional path to a file, that is used to write a timestamp to, if the synchronization of a project leads to
the changing of data on disk (defaults to None).
temp_in_sync_dir: bool
A bool specifying whether to download temporary data to temporary directories below sync_dir (defaults to True).
If False is specified the temporary data is downloaded to the respective user's temporary directory.
"""
backlog: Optional[int]
directory: Optional[Path]
last_updated_file: Optional[Path] = None
temp_in_sync_dir: bool = True
class ProjectConfig(BaseModel):
"""A pydantic model describing the configuration of a project
......@@ -65,6 +94,8 @@ class ProjectConfig(BaseModel):
The project's metrics file
releases: List[ReleaseConfig]
The project's list of releases
sync_config: Optional[SyncConfig]
An optional SyncConfig instance, which is used to override any global defaults.
"""
name: str
......@@ -72,6 +103,7 @@ class ProjectConfig(BaseModel):
output_dir: Path
metrics_file: Path
releases: List[ReleaseConfig]
sync_config: Optional[SyncConfig]
class Projects(BaseSettings):
......@@ -81,9 +113,13 @@ class Projects(BaseSettings):
----------
projects: List[ProjectConfig]
A list of project configurations
sync_config: Optional[SyncConfig]
An optional SyncConfig instance, which is used to override any implicit defaults and sets defaults for all
ProjectConfig instances in projects.
"""
projects: List[ProjectConfig]
sync_config: Optional[SyncConfig]
class Config:
......@@ -98,6 +134,97 @@ class Projects(BaseSettings):
) -> Tuple[SettingsSourceCallable, ...]:
return (read_projects_conf,)
@validator("projects")
def validate_project_releases_unique(cls, projects: List[ProjectConfig]) -> List[ProjectConfig]:
"""Validate the list of ProjectConfig instances to only contain uniquely named ReleaseConfig instances
Parameters
----------
projects: List[ProjectConfig]
A list of ProjectConfig instances
Raises
------
ValueError
If a non-unique name is encountered among any ReleaseConfig instance
Returns
-------
List[ProjectConfig]
The unmodified list of ProjectConfig instances
"""
release_types: List[ReleaseConfig] = []
for project in projects:
release_types += project.releases
names = [release_type.name for release_type in release_types]
if len(set(names)) < len(names):
duplicates = [name for name, count in Counter(names).items() if count > 1]
raise ValueError(
f"The following release type {'name' if len(duplicates) == 1 else 'names'} "
f"{'is' if len(duplicates) == 1 else 'are'} not unique: "
f"{duplicates[0] if len(duplicates) == 1 else duplicates}"
)
return projects
@root_validator
def validate_projects(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validate the list of ProjectConfig instances and override defaults
If a ProjectConfig does not specify a SysConfig, override it with the global SysConfig. If a global SysConfig
does not exist, override with an implicit default where directory defaults to PROJECTS_SYNC_DIR and backlog to
PROJECTS_SYNC_BACKLOG.
Parameters
----------
values: Dict[str, Any]
A dict with all values of the Projects instance
Returns
-------
values: Dict[str, Any]
The (potentially modified) dict with all values of the Projects instance
"""
default_sync_config = SyncConfig(
directory=PROJECTS_SYNC_DIR,
backlog=PROJECTS_SYNC_BACKLOG,
last_updated_file=None,
)
projects: List[ProjectConfig] = values.get("projects") # type: ignore
sync_config: Optional[SyncConfig] = values.get("sync_config")
# merge global sync_config with defaults
if sync_config:
global_sync_config = SyncConfig(
directory=sync_config.directory or default_sync_config.directory,
backlog=sync_config.backlog or default_sync_config.backlog,
last_updated_file=sync_config.last_updated_file or default_sync_config.last_updated_file,
temp_in_sync_dir=False if not sync_config.temp_in_sync_dir else default_sync_config.temp_in_sync_dir,
)
else:
global_sync_config = default_sync_config
for project in projects:
if not project.sync_config:
project.sync_config = global_sync_config
else:
project.sync_config = SyncConfig(
directory=project.sync_config.directory or global_sync_config.directory,
backlog=project.sync_config.backlog or global_sync_config.backlog,
last_updated_file=project.sync_config.last_updated_file or global_sync_config.last_updated_file,
temp_in_sync_dir=False
if not project.sync_config.temp_in_sync_dir
else global_sync_config.temp_in_sync_dir,
)
values["projects"] = projects
return values
def get_project(self, name: str) -> ProjectConfig:
"""Return a ProjectConfig by name
......
......@@ -93,6 +93,38 @@ def test_settings(
"foo/bar",
does_not_raise(),
),
(
True,
[
"[sync_config]",
'directory = "foo"',
"backlog = 2",
"[[projects]]",
'name = "foo/bar"',
'job_name = "build"',
'metrics_file = "metrics.txt"',
'output_dir = "output"',
'releases = [{name = "test",version_metrics = ["bar"],extensions_to_sign = [".baz"]}]',
],
"foo/bar",
does_not_raise(),
),
(
True,
[
"[[projects]]",
'name = "foo/bar"',
'job_name = "build"',
'metrics_file = "metrics.txt"',
'output_dir = "output"',
'releases = [{name = "test",version_metrics = ["bar"],extensions_to_sign = [".baz"]}]',
"[projects.sync_config]",
'directory = "foo"',
"sync_backlog = 2",
],
"foo/bar",
does_not_raise(),
),
(
True,
[
......@@ -106,6 +138,21 @@ def test_settings(
"foo/baz",
raises(RuntimeError),
),
(
True,
[
"[[projects]]",
'name = "foo/bar"',
'job_name = "build"',
'metrics_file = "metrics.txt"',
'output_dir = "output"',
"releases = [",
'{name = "test",extensions_to_sign = [".baz"]}',
'{name = "test",extensions_to_sign = [".bar"]}]',
],
"foo/bar",
raises(ValidationError),
),
(
False,
[],
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment