Verified Commit 18bd338d authored by David Runge's avatar David Runge
Browse files

Read openmetrics of different types

arch_release_promotion/release.py:
Add `Metric`, `SizeMetric`, `AmountMetric` and |VersionMetric| models,
that track metrics with size, amount or version (respectively).
Change the `Release` model to use `amount_metrics`, `size_metrics` and
`version_metrics` attributes instead of `info`.

arch_release_promotion/config.py:
Change the `ReleaseConfig` model to track `version_metrics`,
`size_metrics` and `amount_metrics` instead of `info_metrics`.

arch_release_promotion/files.py:
Change `read_metrics_file()` to return a tuple of AmountMetric,
SizeMetric and VersionMetric lists and read the provided file only if it
exists.

arch_release_promotion/cli.py:
Change `main()` to correctly initialize instances of `Release` with the
different types of metrics.

tests/*:
Change the tests to match the changes in signature and attributes.
parent fdd9145e
......@@ -55,16 +55,19 @@ def main() -> None:
file_extensions=release_config.extensions_to_sign,
)
metrics = files.read_metrics_file(
path=metrics_file,
version_metrics_names=release_config.version_metrics,
size_metrics_names=release_config.size_metrics,
amount_metrics_names=release_config.amount_metrics,
)
artifact_release = release.Release(
name=release_config.name,
version=release_version,
files=files.files_in_dir(path=artifact_full_path),
info=files.read_metrics_file(
path=metrics_file,
metrics=release_config.info_metrics,
)
if metrics_file.exists()
else None,
amount_metrics=metrics[0],
size_metrics=metrics[1],
version_metrics=metrics[2],
torrent_file=torrent.create_torrent_file(
path=artifact_full_path,
webseeds=torrent.get_webseeds(
......
......@@ -27,8 +27,15 @@ class ReleaseConfig(BaseModel):
----------
name: str
The name of the release (type)
info_metrics: List[str]
A list of openmetrics "name" labels of type info, that should be extracted from the project's metrics file
version_metrics: Optional[List[str]]
A list of names that identify labels in metric samples of type "info", that should be extracted from the
project's metrics file
size_metrics: Optional[List[str]]
A list of names that identify labels in metric samples of type "gauge", that should be extracted from the
project's metrics file
amount_metrics: Optional[List[str]]
A list of names that identify labels in metric samples of type "summary", that should be extracted from the
project's metrics file
extensions_to_sign: List[str]
A list of file extensions for which to create detached signatures
create_torrent: bool
......@@ -36,7 +43,9 @@ class ReleaseConfig(BaseModel):
"""
name: str
info_metrics: List[str]
version_metrics: Optional[List[str]]
size_metrics: Optional[List[str]]
amount_metrics: Optional[List[str]]
extensions_to_sign: List[str]
create_torrent: bool = False
......
......@@ -2,12 +2,17 @@ import shutil
import tempfile
import zipfile
from pathlib import Path
from typing import Dict, List
from typing import List, Optional, Tuple
import orjson
from prometheus_client.parser import text_fd_to_metric_families
from arch_release_promotion.release import Release
from arch_release_promotion.release import (
AmountMetric,
Release,
SizeMetric,
VersionMetric,
)
TEMP_DIR_PREFIX = "arp-"
......@@ -193,41 +198,82 @@ def write_zip_file_to_parent_dir(path: Path, name: str = "promotion", format: st
shutil.make_archive(base_name=str(path.parent / Path(name)), format=format, root_dir=path)
def read_metrics_file(path: Path, metrics: List[str]) -> Dict[str, Dict[str, str]]:
"""Read a metrics file that contains openmetrics based metrics and return metrics that match the keywords
def read_metrics_file(
path: Path,
version_metrics_names: Optional[List[str]],
size_metrics_names: Optional[List[str]],
amount_metrics_names: Optional[List[str]],
) -> Tuple[List[AmountMetric], List[SizeMetric], List[VersionMetric]]:
"""Read a metrics file that contains openmetrics based metrics and return those that match the respective keywords
Parameters
----------
path: Path
The path of the file to read
metrics: List[str]
A list of metric names to search for in the metrics file
version_metrics_names: Optional[List[str]]
A list of metric names to search for in the labels of metric samples of type "info"
size_metrics_names: Optional[List[str]],
A list of metric names to search for in the labels of metric samples of type "gauge"
amount_metrics_names: Optional[List[str]],
A list of metric names to search for in the labels of metric samples of type "summary"
Returns
-------
Dict[str, str]:
A dictionary representing packages, their respective description and their version
Tuple[List[AmountMetric], List[SizeMetric], List[VersionMetric]]:
A Tuple with lists of AmountMetric, SizeMetric and VersionMetric instances derived from the input file
"""
output: Dict[str, Dict[str, str]] = {}
amount_metrics: List[AmountMetric] = []
size_metrics: List[SizeMetric] = []
version_metrics: List[VersionMetric] = []
with open(path, "r") as file:
for metric in text_fd_to_metric_families(file):
if metric.name == "version_info" and metric.samples:
if path.exists():
with open(path, "r") as file:
for metric in text_fd_to_metric_families(file):
for sample in metric.samples:
if (
sample.labels.get("package")
version_metrics_names
and metric.type == "info"
and metric.name == "version_info"
and sample.labels.get("name") in version_metrics_names
and sample.labels.get("description")
and sample.labels.get("version")
and sample.labels.get("package") in metrics
):
output.update(
{
sample.labels.get("package"): {
"description": sample.labels.get("description"),
"version": sample.labels.get("version"),
}
}
)
return output
version_metrics += [
VersionMetric(
name=sample.labels.get("name"),
description=sample.labels.get("description"),
version=sample.labels.get("version"),
)
]
if (
size_metrics_names
and metric.type == "gauge"
and metric.name == "artifact_bytes"
and sample.labels.get("name") in size_metrics_names
and sample.labels.get("description")
and sample.value
):
size_metrics += [
SizeMetric(
name=sample.labels.get("name"),
description=sample.labels.get("description"),
size=sample.value,
)
]
if (
amount_metrics_names
and metric.type == "summary"
and metric.name == "data_count"
and sample.labels.get("name") in amount_metrics_names
and sample.labels.get("description")
and sample.value
):
amount_metrics += [
AmountMetric(
name=sample.labels.get("name"),
description=sample.labels.get("description"),
amount=sample.value,
)
]
return (amount_metrics, size_metrics, version_metrics)
from typing import Dict, List, Optional
from typing import List, Optional
from pydantic import BaseModel
class Metric(BaseModel):
"""A pydantic model describing a metric
Attributes
----------
name: str
A name for the metric
description: str
A description for the metric
"""
name: str
description: str
class SizeMetric(Metric):
"""A pydantic model describing a size metric
Attributes
----------
size: int
"""
size: int
class AmountMetric(Metric):
"""A pydantic model describing an amount metric
Attributes
----------
amount: int
"""
amount: int
class VersionMetric(Metric):
"""A pydantic model describing a version metric
Attributes
----------
version: str
"""
version: str
class Release(BaseModel):
"""A pydantic model describing a release
......@@ -14,8 +62,12 @@ class Release(BaseModel):
The version of the artifact
files: List[str]
A list of files that belong to the release
info: Dict[str, str]
A dictionary that provides additional information about the release
amount_metrics: List[AmountMetric]
A list of AmountMetric instances that are related to the release
size_metrics: List[SizeMetric]
A list of SizeMetric instances that are related to the release
version_metrics: List[VersionMetric]
A list of VersionMetric instances that are related to the release
torrent_file: str
A string representing the name of a torrent file for the release
developer: str
......@@ -27,7 +79,9 @@ class Release(BaseModel):
name: str
version: str
files: List[str]
info: Optional[Dict[str, Dict[str, str]]]
amount_metrics: List[AmountMetric]
size_metrics: List[SizeMetric]
version_metrics: List[VersionMetric]
torrent_file: Optional[str]
developer: str
pgp_public_key: str
......@@ -88,7 +88,7 @@ def test_settings(
'job_name = "build"',
'metrics_file = "metrics.txt"',
'output_dir = "output"',
'releases = [{name = "test",info_metrics = ["bar"],extensions_to_sign = [".baz"]}]',
'releases = [{name = "test",version_metrics = ["bar"],extensions_to_sign = [".baz"]}]',
],
"foo/bar",
does_not_raise(),
......@@ -101,7 +101,7 @@ def test_settings(
'job_name = "build"',
'metrics_file = "metrics.txt"',
'output_dir = "output"',
'releases = [{name = "test",info_metrics = ["bar"],extensions_to_sign = [".baz"]}]',
'releases = [{name = "test",version_metrics = ["bar"],extensions_to_sign = [".baz"]}]',
],
"foo/baz",
raises(RuntimeError),
......
......@@ -39,10 +39,20 @@ def create_temp_dir_with_files() -> Iterator[Path]:
def create_temp_metrics_file() -> Iterator[Path]:
with tempfile.TemporaryDirectory() as temp_dir:
with tempfile.NamedTemporaryFile(dir=temp_dir, delete=False) as temp_file:
temp_file.write(b'version_info{package="foo", description="Version of foo", version="1.0.0-1"} 1\n')
temp_file.write(b'version_info{package="foo", not_description="Version of foo", version="1.0.0-1"} 1\n')
temp_file.write(b"# TYPE version_info info\n")
temp_file.write(b"# HELP version_info Package description and version information\n")
temp_file.write(b'version_info{name="foo", description="Version of foo", version="1.0.0-1"} 1\n')
temp_file.write(b'version_info{name="bar", not_description="Version of bar", version="1.0.0-1"} 1\n')
temp_file.write(b"version_info 1\n")
temp_file.write(b'foo{package="foo", description="Version of foo", version="1.0.0-1"} 1\n')
temp_file.write(b'version{name="foo", description="Version of foo", version="1.0.0-1"} 1\n')
temp_file.write(b"# TYPE artifact_bytes gauge\n")
temp_file.write(b"# HELP artifact_bytes Artifact sizes in bytes\n")
temp_file.write(b'artifact_bytes{name="foo",description="Size of ISO image in MiB"} 832\n')
temp_file.write(b'artifact_bytes{not_name="foo",description="Size of ISO image in MiB"} 832\n')
temp_file.write(b"# TYPE data_count summary\n")
temp_file.write(b"# HELP data_count The amount of packages used in specific buildmodes\n")
temp_file.write(b'data_count{name="foo",description="The amount of packages in foo"} 369\n')
temp_file.write(b'data_count{not_name="netboot",description="something else"} 369\n')
yield Path(temp_file.name)
......@@ -132,6 +142,9 @@ def test_write_release_info_to_file(create_temp_dir: Path) -> None:
name="foo",
version="1.0.0",
files=["foo", "bar", "baz"],
amount_metrics=[],
size_metrics=[],
version_metrics=[],
developer="Foobar McFoo",
torrent_file="foo-0.1.0.torrent",
pgp_public_key="SOMEONESKEY",
......@@ -145,6 +158,9 @@ def test_write_release_info_to_file(create_temp_dir: Path) -> None:
name="foo",
version="1.0.0",
files=["foo", "bar", "baz"],
amount_metrics=[],
size_metrics=[],
version_metrics=[],
developer="Foobar McFoo",
torrent_file="foo-0.1.0.torrent",
pgp_public_key="SOMEONESKEY",
......@@ -176,13 +192,33 @@ def test_write_zip_file_to_parent_dir(
assert (create_temp_dir_with_files.parent / Path(f"{name}.zip")).is_file()
@mark.parametrize("metrics", [([]), (["foo"])])
def test_read_metrics_file(metrics: List[str], create_temp_metrics_file: Path) -> None:
files.read_metrics_file(
path=create_temp_metrics_file,
metrics=metrics,
)
files.read_metrics_file(
path=create_temp_metrics_file,
metrics=metrics,
@mark.parametrize(
"file_exists, version_metrics_names, size_metrics_names, amount_metrics_names",
[
(True, [], [], []),
(False, [], [], []),
(True, ["foo"], ["foo"], ["foo"]),
(True, ["bar"], ["foo"], ["foo"]),
(True, ["bar"], ["foo"], ["bar"]),
(True, ["bar"], ["bar"], ["foo"]),
],
)
def test_read_metrics_file(
file_exists: bool,
version_metrics_names: List[str],
size_metrics_names: List[str],
amount_metrics_names: List[str],
create_temp_metrics_file: Path,
) -> None:
metrics = files.read_metrics_file(
path=create_temp_metrics_file if file_exists else Path("foo"),
version_metrics_names=version_metrics_names,
size_metrics_names=size_metrics_names,
amount_metrics_names=amount_metrics_names,
)
if version_metrics_names == "foo":
assert len(metrics[2]) == 1
if size_metrics_names == "foo":
assert len(metrics[1]) == 1
if amount_metrics_names == "foo":
assert len(metrics[0]) == 1
from arch_release_promotion import release
def test_metric() -> None:
assert release.Metric(
name="foo",
description="bar",
)
def test_amount_metric() -> None:
assert release.AmountMetric(
name="foo",
description="bar",
amount=1,
)
def test_size_metric() -> None:
assert release.SizeMetric(
name="foo",
description="bar",
size=1.1,
)
def test_version_metric() -> None:
assert release.VersionMetric(
name="foo",
description="bar",
version="1.0.0-1",
)
def test_release() -> None:
assert release.Release(
name="foo",
version="0.1.0",
files=["foo", "bar", "baz"],
info={"bar": {"description": "Version of bar when building foo", "version": "1.0.0"}},
amount_metrics=[],
size_metrics=[],
version_metrics=[],
developer="Foobar McFoo",
torrent_file="foo-0.1.0.torrent",
pgp_public_key="SOMEONESKEY",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment