fix(rules): detect unrequired Python packages

When using PEP517 to build a Python package a RECORD file is included containing all the dependencies of a package. For backports (like importlib_metadata) projects usually specify until which Python version it is required combined with an optional import usually by checking sys.version_info and comparing the Python version.

As using the AST to figure out if a package is required is tricky and a lot of work and finding it out via shadowed imports of builtins for example with ExceptionGroup can work as shown below, but does not hold up for importlib.metadata as the backport is called importlib_metadata.

{ 'builtins.ExceptionGroup': {'usr/bin/optional.py'}, 'exceptiongroup.ExceptionGroup': {'usr/bin/optional.py'} }

Instead of these methods, rely on the project to provide a marker for example python_version < 3.11 for required packages.

Namcap/rules/pydepends.py

@@ -4,8 +4,11 @@
 from collections import defaultdict
 import ast
 import importlib
+import os.path
+from io import TextIOWrapper
 import sys
 import sysconfig
+from packaging.metadata import parse_email, Metadata
 import Namcap.package
 from Namcap.util import is_script, script_type
 from Namcap.ruleclass import TarballRule
@@ -121,6 +124,46 @@ def get_imports(fileobj, filename, modules, gir_modules, gir_versions):
                     gir_versions[module.value] = version.value
 
 
+def get_unrequired_deps(metadata: TextIOWrapper) -> list[str]:
+    """
+    Parses the Python packaging METADATA file and returns a list of python
+    modules which are not required with the currently installed Python version.
+
+    The list of unrequired python modules is used in the PythonDependencyRule
+    to remove false positives of required python modules which are marked as
+    they are for example conditionally imported. Usually because a newer Python
+    version now has a builtin for this, see for example exceptiongroup or
+    importlib-metadata.
+
+    More information about the Metadata file:
+
+    https://packaging.python.org/en/latest/specifications/core-metadata/
+    """
+
+    try:
+        raw, _ = parse_email(metadata.read())
+        parsed = Metadata.from_raw(raw)
+    except Exception:
+        return []
+
+    pyversion = f"{sys.version_info.major}.{sys.version_info.minor}"
+    environment = {"python_version": pyversion}
+    unrequired_dep: list[str] = []
+
+    if parsed.requires_dist is None:
+        return unrequired_dep
+
+    for dep in parsed.requires_dist:
+        if dep.marker is None:
+            continue
+        if "python_version" in str(dep.marker) and not dep.marker.evaluate(environment):
+            # PyPi applies symbol conversion, so we replace relevant symbols
+            # too (e.g. typing-extensions -> typing_extensions)
+            unrequired_dep.append(f"python-{dep.name.replace('-', '_')}")
+
+    return unrequired_dep
+
+
 class PythonDependencyRule(TarballRule):
     name = "pydepends"
     description = "Checks python dependencies"
@@ -129,11 +172,14 @@ class PythonDependencyRule(TarballRule):
         modules: dict[str, set[str]] = defaultdict(set)
         gir_modules: dict[str, set[str]] = defaultdict(set)
         gir_versions: dict[str, str] = defaultdict(str)
+        unrequired_packages: list[str] = []
 
         for entry in tar:
             if not entry.isfile():
                 continue
             f = tar.extractfile(entry)
+            if os.path.basename(entry.name) == "METADATA":
+                unrequired_packages = get_unrequired_deps(f)
             if not entry.name.endswith(".py") and not is_script(f):
                 continue
             if is_script(f) and script_type(f) not in ["python", "python3"]:
@@ -159,6 +205,8 @@ class PythonDependencyRule(TarballRule):
 
         # Print python module deps
         for pkg, libraries in (dependlist | gir_dependlist).items():
+            if pkg in unrequired_packages:
+                continue
             if isinstance(libraries, set):
                 files = list(libraries)
                 needing = set().union(*[liblist[lib] for lib in libraries])