diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d09f080f5..0d587b86b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,6 +32,8 @@ repos: # # For now, we simply copy & paste from pyproject.toml :( additional_dependencies: + - "importlib-metadata>=3.6.0; python_version < '3.10'" + - "importlib-resources; python_version < '3.10'" - "click>=8.0.3,!=8.1.4" # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 - "docutils>=0.16" # 0.16 is the current one available for RHEL9 - "fmf>=1.3.0" @@ -80,6 +82,8 @@ repos: # # For now, we simply copy & paste from pyproject.toml :( additional_dependencies: + - "importlib-metadata>=3.6.0; python_version < '3.10'" + - "importlib-resources; python_version < '3.10'" - "click>=8.0.3,!=8.1.4" # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 - "docutils>=0.16" # 0.16 is the current one available for RHEL9 - "fmf>=1.3.0" diff --git a/docs/code/plugin-introduction.rst b/docs/code/plugin-introduction.rst index ddf34e7c77..0cbc70d3ba 100644 --- a/docs/code/plugin-introduction.rst +++ b/docs/code/plugin-introduction.rst @@ -16,6 +16,11 @@ provided in the ``TMT_PLUGINS`` environment variable and from Consider adding a static type checker (e.g. ``mypy``) in your plugin's CI using the ``main`` branch of ``tmt``. +.. versionadded:: 1.38 + You can use ``tmt.resources`` entry point to inject resource + files to be used for tmt, e.g. schemas or templates. See + :ref:`additional-resources` for more details. + Inheritance ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -62,3 +67,48 @@ features which cannot be covered by generic ssh implementation of the ``Guest`` class. __ https://github.com/teemtee/tmt/tree/main/examples/plugins + +.. additional-resources: + +Additional resource files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to make resource files available to the base ``tmt`` +execution, you need to point a ``tmt.resources`` entry point to the +root python package where the resource files are located from, e.g. +with in the ``examples/plugins``: + +.. code-block:: toml + :caption: pyproject.toml + :emphasize-lines: 5,6 + + [project.entry-points."tmt.plugin"] + ProvisionExample = "example.provision:ProvisionExample" + DiscoverExample = "example.discover:DiscoverExample" + + [project.entry-points."tmt.resources"] + ResourcesExample = "example" + +you can, for example, add a json schema file for the plugins +implemented above by including the schema files under a ``schemas`` +folder: + +.. code-block:: shell + + $ tree ./example + ./example + ├── __init__.py + ├── discover.py + ├── provision.py + └── schemas + ├── discover + │ └── example.yaml + └── provision + └── example.yaml + +.. note:: + + Both the entry-point entries as well as any resource file + under the ``tmt.resources`` path **must** have unique names. + Consider namespacing all relevant entries with the name of the + project or an unambiguous derivative of it. diff --git a/docs/releases.rst b/docs/releases.rst index c270cb6e0b..9e34e33534 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,13 @@ Releases ====================== +tmt-1.38.0 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External plugins can inject additional resources to be used by tmt using +the ``tmt.resources`` entry-point, e.g. to extend the schema validation +or tmt templates. + tmt-1.37.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/examples/plugins/example/__init__.py b/examples/plugins/example/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/plugins/discover.py b/examples/plugins/example/discover.py similarity index 78% rename from examples/plugins/discover.py rename to examples/plugins/example/discover.py index 0541e5be02..885e06bc28 100644 --- a/examples/plugins/discover.py +++ b/examples/plugins/example/discover.py @@ -1,11 +1,23 @@ +import dataclasses + import tmt import tmt.steps import tmt.steps.discover +import tmt.utils # See the online documentation for more details about writing plugins # https://tmt.readthedocs.io/en/stable/plugins.html +@dataclasses.dataclass +class DiscoverExampleData(tmt.steps.discover.DiscoverStepData): + path: str = tmt.utils.field( + default=".", + option=('-p', '--path'), + metavar='ROOT', + help='Path to the metadata tree root.') + + @tmt.steps.provides_method('example') class DiscoverExample(tmt.steps.discover.DiscoverPlugin): """ @@ -16,6 +28,8 @@ class DiscoverExample(tmt.steps.discover.DiscoverPlugin): of configuration examples as well. """ + _data_class = DiscoverExampleData + def show(self): """ Show plugin details for given or all available keys """ super().show([]) @@ -48,7 +62,8 @@ def go(self): print("Code should prepare environment for tests.") # Discover available tests - self._tests = tmt.Tree(logger=self._logger, path=".").tests() + self._tests = tmt.Tree(logger=self._logger, + path=self.data.path).tests() def tests(self): """ diff --git a/examples/plugins/provision.py b/examples/plugins/example/provision.py similarity index 100% rename from examples/plugins/provision.py rename to examples/plugins/example/provision.py diff --git a/examples/plugins/example/schemas/discover/example.yaml b/examples/plugins/example/schemas/discover/example.yaml new file mode 100644 index 0000000000..18b8b877f4 --- /dev/null +++ b/examples/plugins/example/schemas/discover/example.yaml @@ -0,0 +1,22 @@ +$id: /schemas/discover/example +$schema: https://json-schema.org/draft-07/schema + +type: object +additionalProperties: false + +properties: + how: + type: string + enum: + - example + path: + $ref: "/schemas/common#/definitions/fmf_id/properties/path" + + # Other basic discover properties + name: + type: string + order: + $ref: "/schemas/core#/definitions/order" + +required: + - how diff --git a/examples/plugins/example/schemas/provision/example.yaml b/examples/plugins/example/schemas/provision/example.yaml new file mode 100644 index 0000000000..283e71b03c --- /dev/null +++ b/examples/plugins/example/schemas/provision/example.yaml @@ -0,0 +1,20 @@ +$id: /schemas/provision/example +$schema: https://json-schema.org/draft-07/schema + +type: object +additionalProperties: false + +properties: + how: + type: string + enum: + - example + + # Other basic provision properties + name: + type: string + order: + $ref: "/schemas/core#/definitions/order" + +required: + - how diff --git a/examples/plugins/pyproject.toml b/examples/plugins/pyproject.toml index 126126bf99..a3e57c2c41 100644 --- a/examples/plugins/pyproject.toml +++ b/examples/plugins/pyproject.toml @@ -12,10 +12,12 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = [ - "discover.py", - "provision.py", + "example", ] [project.entry-points."tmt.plugin"] -ProvisionExample = "provision:ProvisionExample" -DiscoverExample = "discover:DiscoverExample" +ProvisionExample = "example.provision:ProvisionExample" +DiscoverExample = "example.discover:DiscoverExample" + +[project.entry-points."tmt.resources"] +ResourcesExample = "example" diff --git a/pyproject.toml b/pyproject.toml index 26f2d94999..f22a15cf01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ classifiers = [ "Operating System :: POSIX :: Linux", ] dependencies = [ # F39 / PyPI + "importlib-metadata>=3.6.0; python_version < '3.10'", # Backporting selectable entry_points + "importlib-resources; python_version < '3.12'", # MultiplexedPath is broken on earlier versions "click>=8.0.3,!=8.1.4", # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 "docutils>=0.16", # 0.16 is the current one available for RHEL9 "fmf>=1.3.0", @@ -413,6 +415,8 @@ builtins-ignorelist = ["help", "format", "input", "filter", "copyright", "max"] "pathlib.PosixPath".msg = "Use tmt._compat.pathlib.Path instead." "warnings.deprecated".msg = "Use tmt._compat.warnings.deprecated instead." "os.path".msg = "Use tmt._compat.pathlib.Path and pathlib instead." +"importlib.metadata.entry_points".msg = "Use tmt._compat.importlib.metadata.entry_points instead." +"importlib.readers".msg = "Use tmt._compat.importlib.readers instead." [tool.ruff.lint.isort] known-first-party = ["tmt"] diff --git a/tests/core/env/test.sh b/tests/core/env/test.sh index ed910f94a3..28a038b32d 100755 --- a/tests/core/env/test.sh +++ b/tests/core/env/test.sh @@ -12,7 +12,7 @@ rlJournalStart rlRun -s "TMT_DEBUG=3 tmt plan show" rlAssertGrep "Using the 'DiscoverFmf' plugin" $rlRun_LOG rlRun -s "TMT_DEBUG=weird tmt plan show" 2 - rlAssertGrep "Invalid debug level" $rlRun_LOG + rlAssertGrep "Invalid value.*'weird' is not a valid integer" $rlRun_LOG -E rlPhaseEnd for execute in 'tmt'; do diff --git a/tests/plugins/data/.fmf/version b/tests/plugins/data/.fmf/version new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/tests/plugins/data/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/tests/plugins/data/main.fmf b/tests/plugins/data/main.fmf new file mode 100644 index 0000000000..5312d0b083 --- /dev/null +++ b/tests/plugins/data/main.fmf @@ -0,0 +1,8 @@ +summary: Check plugin schema validation +execute: + how: tmt +provision: + how: example +discover: + how: example + path: some/random/path diff --git a/tests/plugins/test.sh b/tests/plugins/test.sh index e5dd76c97a..632c08bf84 100755 --- a/tests/plugins/test.sh +++ b/tests/plugins/test.sh @@ -5,6 +5,7 @@ rlJournalStart rlPhaseStartSetup rlRun "tmp=\$(mktemp -d)" 0 "Create tmp directory" rlRun "cp -r $(git rev-parse --show-toplevel)/examples/plugins $tmp" + rlRun "cp -a data $tmp" rlRun "pushd $tmp" # For local development this can run already in venv, do not use venv @@ -27,11 +28,16 @@ rlJournalStart rlAssertGrep "Unsupported discover method" "$rlRun_LOG" rlRun -s "$tmt run provision -h example --help" "2" rlAssertGrep "Unsupported provision method" "$rlRun_LOG" + rlRun -s "$tmt -r data lint --enable-check C000 --enforce-check C000" "1" + rlAssertGrep "fail C000 fmf node failed schema validation" "$rlRun_LOG" + rlAssertGrep "fail C000 key \"path\" not recognized" "$rlRun_LOG" + rlAssertGrep "fail C000 value of \"how\" is not" "$rlRun_LOG" # Install them to entry_point and they work now rlRun "pip install ./plugins" rlRun "$tmt run discover -h example --help" rlRun "$tmt run provision -h example --help" + rlRun -s "$tmt -r data lint --enable-check C000 --enforce-check C000" # Uninstall them rlRun "pip uninstall -y demo-plugins" @@ -47,7 +53,7 @@ rlJournalStart rlAssertGrep "Unsupported provision method" "$rlRun_LOG" # Export variable and plugins work now - rlRun "export TMT_PLUGINS=./plugins" + rlRun "export TMT_PLUGINS=./plugins/example" rlRun "$tmt run discover -h example --help" rlRun "$tmt run provision -h example --help" rlPhaseEnd diff --git a/tmt/_compat/importlib/__init__.py b/tmt/_compat/importlib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tmt/_compat/importlib/metadata.py b/tmt/_compat/importlib/metadata.py new file mode 100644 index 0000000000..b06df7f5ac --- /dev/null +++ b/tmt/_compat/importlib/metadata.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import sys + +if sys.version_info < (3, 10): + from importlib_metadata import entry_points # pyright: ignore[reportUnknownVariableType] +else: + from importlib.metadata import entry_points + +__all__ = [ + "entry_points", + ] diff --git a/tmt/_compat/importlib/readers.py b/tmt/_compat/importlib/readers.py new file mode 100644 index 0000000000..ed82eeb8b6 --- /dev/null +++ b/tmt/_compat/importlib/readers.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import sys + +if sys.version_info < (3, 12): + from importlib_resources.readers import MultiplexedPath +else: + from importlib.readers import MultiplexedPath + +__all__ = [ + "MultiplexedPath", + ] diff --git a/tmt/export/template.py b/tmt/export/template.py index b5508daf55..10137574e9 100644 --- a/tmt/export/template.py +++ b/tmt/export/template.py @@ -1,3 +1,4 @@ +from importlib.abc import Traversable from typing import Any, Optional import tmt.base @@ -15,7 +16,7 @@ class TemplateExporter(tmt.export.ExportPlugin): def render_template( cls, *, - template_filepath: Optional[Path] = None, + template_filepath: Optional[Traversable] = None, default_template_filename: str, keys: Optional[list[str]] = None, **variables: Any diff --git a/tmt/log.py b/tmt/log.py index 753e8e3902..1de96e78ed 100644 --- a/tmt/log.py +++ b/tmt/log.py @@ -108,21 +108,6 @@ def create_decolorizer(apply_colors: bool) -> Callable[[str], str]: return tmt.utils.remove_color -def _debug_level_from_global_envvar() -> int: - import tmt.utils - - raw_value = os.getenv('TMT_DEBUG', None) - - if raw_value is None: - return 0 - - try: - return int(raw_value) - - except ValueError: - raise tmt.utils.GeneralError(f"Invalid debug level '{raw_value}', use an integer.") - - def decide_colorization(no_color: bool, force_color: bool) -> tuple[bool, bool]: """ Decide whether the output and logging should be colorized. @@ -626,19 +611,13 @@ def apply_verbosity_options( else: self.verbosity_level = verbosity_level - debug_level_from_global_envvar = _debug_level_from_global_envvar() + debug_level_from_option = cast(Optional[int], actual_kwargs.get('debug', None)) - if debug_level_from_global_envvar not in (None, 0): - self.debug_level = debug_level_from_global_envvar + if debug_level_from_option is None or debug_level_from_option == 0: + pass else: - debug_level_from_option = cast(Optional[int], actual_kwargs.get('debug', None)) - - if debug_level_from_option is None or debug_level_from_option == 0: - pass - - else: - self.debug_level = debug_level_from_option + self.debug_level = debug_level_from_option quietness_level = actual_kwargs.get('quiet', False) @@ -868,7 +847,14 @@ def get_bootstrap_logger(cls) -> 'Logger': # Stay away of our future main logger actual_logger = Logger._normalize_logger(logging.getLogger('_tmt_bootstrap')) - cls._bootstrap_logger = Logger.create(actual_logger=actual_logger) + # The environment variables are usually handled at the click cli stage + # Here we enable safe parsing of those variables. + try: + debug = int(os.getenv('TMT_DEBUG', 0)) + except ValueError: + debug = None + + cls._bootstrap_logger = Logger.create(actual_logger=actual_logger, debug=debug) cls._bootstrap_logger.add_console_handler() return cls._bootstrap_logger diff --git a/tmt/options.py b/tmt/options.py index 04f5569804..b755398510 100644 --- a/tmt/options.py +++ b/tmt/options.py @@ -132,7 +132,7 @@ def option( '-v', '--verbose', count=True, default=0, help='Show more details. Use multiple times to raise verbosity.'), option( - '-d', '--debug', count=True, default=0, + '-d', '--debug', count=True, default=0, envvar="TMT_DEBUG", help='Provide debugging information. Repeat to see more details.'), option( '-q', '--quiet', is_flag=True, diff --git a/tmt/plugins/__init__.py b/tmt/plugins/__init__.py index f5dc921ce0..93df507fff 100644 --- a/tmt/plugins/__init__.py +++ b/tmt/plugins/__init__.py @@ -5,12 +5,12 @@ import pkgutil import sys from collections.abc import Iterator -from importlib.metadata import entry_points from types import ModuleType from typing import Any, Generic, Optional, TypeVar, cast import tmt import tmt.utils +from tmt._compat.importlib.metadata import entry_points from tmt.log import Logger from tmt.utils import Path @@ -128,11 +128,7 @@ def _explore_entry_point(entry_point: str, logger: Logger) -> None: logger = logger.descend() try: - eps = entry_points() - if hasattr(eps, "select"): - entry_point_group = eps.select(group=entry_point) - else: - entry_point_group = eps[entry_point] + entry_point_group = entry_points(group=entry_point) for found in entry_point_group: logger.debug(f"Loading plugin '{found.name}' ({found.value}).") diff --git a/tmt/steps/execute/__init__.py b/tmt/steps/execute/__init__.py index a5f18da10c..d45068b013 100644 --- a/tmt/steps/execute/__init__.py +++ b/tmt/steps/execute/__init__.py @@ -3,6 +3,7 @@ import functools import json import os +import pathlib import signal as _signal import subprocess import threading @@ -600,6 +601,11 @@ def prepare_scripts(self, guest: "tmt.steps.provision.Guest") -> None: # Install all scripts on guest for script in self.scripts: source = SCRIPTS_SRC_DIR / script.path.name + if not source.is_file(): + self.warn(f"Script is not a file, skipping: {script.path}") + continue + assert isinstance(source, pathlib.Path) # narrow type # noqa: TID251 + source = Path(source) for dest in [script.path, *script.aliases]: guest.push( diff --git a/tmt/steps/prepare/feature.py b/tmt/steps/prepare/feature.py index 63e7041a28..0fb62311e0 100644 --- a/tmt/steps/prepare/feature.py +++ b/tmt/steps/prepare/feature.py @@ -1,4 +1,5 @@ import dataclasses +import pathlib from typing import Optional, cast import tmt @@ -33,8 +34,9 @@ def __init__( def _find_playbook(self, filename: str) -> Optional[Path]: filepath = FEATURE_PLAYEBOOK_DIRECTORY / filename - if filepath.exists(): - return filepath + if filepath.is_file(): + assert isinstance(filepath, pathlib.Path) # narrow type # noqa: TID251 + return Path(filepath) self.warn(f"Cannot find any suitable playbook for '{filename}'.") return None diff --git a/tmt/steps/report/junit.py b/tmt/steps/report/junit.py index f82cb5e0d1..9b674a4e12 100644 --- a/tmt/steps/report/junit.py +++ b/tmt/steps/report/junit.py @@ -1,6 +1,7 @@ import dataclasses import functools from collections.abc import Iterator +from importlib.abc import Traversable from typing import TYPE_CHECKING, Any, Optional, TypedDict, cast, overload from jinja2 import FileSystemLoader, select_autoescape @@ -13,6 +14,7 @@ import tmt.steps import tmt.steps.report import tmt.utils +from tmt._compat.importlib.readers import MultiplexedPath from tmt.plugins import ModuleImporter from tmt.result import ResultOutcome from tmt.utils import Path, field @@ -31,7 +33,7 @@ CUSTOM_FLAVOR_NAME = 'custom' # Relative path to tmt junit template directory. -DEFAULT_TEMPLATE_DIR = Path('steps/report/junit/templates/') +DEFAULT_TEMPLATE_DIR = tmt.utils.resource_files('steps/report/junit/templates/') # ignore[unused-ignore]: Pyright would report that "module cannot be # used as a type", and it would be correct. On the other hand, it works, @@ -167,7 +169,7 @@ def duration(self) -> int: def make_junit_xml( phase: tmt.steps.report.ReportPlugin[Any], flavor: str = DEFAULT_FLAVOR_NAME, - template_path: Optional[Path] = None, + template_path: Optional[Traversable] = None, include_output_log: bool = True, prettify: bool = True, results_context: Optional[ResultsContext] = None, @@ -193,13 +195,18 @@ def make_junit_xml( # Prepare the template environment environment = default_template_environment() - template_path = template_path or tmt.utils.resource_files( - DEFAULT_TEMPLATE_DIR / Path(f'{flavor}.xml.j2')) + template_path = template_path or DEFAULT_TEMPLATE_DIR / f'{flavor}.xml.j2' # Use a FileSystemLoader for a non-custom flavor if flavor != CUSTOM_FLAVOR_NAME: + # TODO: Check if PackageLoader would work instead + # Note: the issue here is that jinja passes the paths through `os.fspath` which breaks the + # MultiplexedPath. This can be resolved by using `as_files` to create a context and copy + # all files to a real data folder, or jinja could learn to support generic Traversable + if isinstance(DEFAULT_TEMPLATE_DIR, MultiplexedPath): + phase.warn("Jinja template extension for report/junit/templates is not supported yet.") environment.loader = FileSystemLoader( - searchpath=tmt.utils.resource_files(DEFAULT_TEMPLATE_DIR)) + searchpath=str(DEFAULT_TEMPLATE_DIR)) def _read_log(log: Path) -> str: """ Read the contents of a given result log """ @@ -246,8 +253,7 @@ def _read_log(log: Path) -> str: # The schema check must be done only for a non-custom JUnit flavors if flavor != CUSTOM_FLAVOR_NAME: - xsd_schema_path = Path(tmt.utils.resource_files( - Path(f'steps/report/junit/schemas/{flavor}.xsd'))) + xsd_schema_path = tmt.utils.resource_files(f'steps/report/junit/schemas/{flavor}.xsd') schema_root: XMLElement = etree.XML(xsd_schema_path.read_bytes()) xml_parser_kwargs['schema'] = etree.XMLSchema(schema_root) diff --git a/tmt/templates/__init__.py b/tmt/templates/__init__.py index 9f7ffadc85..30a0666bb3 100644 --- a/tmt/templates/__init__.py +++ b/tmt/templates/__init__.py @@ -1,4 +1,6 @@ import functools +import importlib.abc +import pathlib from typing import Any, Optional import tmt @@ -29,18 +31,19 @@ def _combine(default: TemplatesType, custom: TemplatesType) -> TemplatesType: return result -def _get_template_file_paths(path: Path) -> dict[str, Path]: +def _get_template_file_paths(path: importlib.abc.Traversable) -> dict[str, Path]: """ Get a dictionary of template names and their file paths. :param path: Path to the directory to search for templates. """ return { - file.name.removesuffix(TEMPLATE_FILE_SUFFIX): file for file in path.iterdir() - if file.is_file() and file.suffix == TEMPLATE_FILE_SUFFIX + file.name.removesuffix(TEMPLATE_FILE_SUFFIX): file_path for file in path.iterdir() + if file.is_file() and isinstance(file, pathlib.Path) and (file_path := Path(file)) # noqa: TID251 + and file_path.suffix == TEMPLATE_FILE_SUFFIX } -def _get_templates(root_dir: Path) -> TemplatesType: +def _get_templates(root_dir: importlib.abc.Traversable) -> TemplatesType: """ Get all templates in given root directory. :param root_dir: Path to the directory to search for templates. @@ -48,7 +51,7 @@ def _get_templates(root_dir: Path) -> TemplatesType: templates: TemplatesType = {} for template_type in TEMPLATE_TYPES: templates_dir = root_dir / template_type - if templates_dir.exists() and templates_dir.is_dir(): + if templates_dir.is_dir(): template_files = _get_template_file_paths(templates_dir) if template_files: templates[template_type] = template_files diff --git a/tmt/utils/__init__.py b/tmt/utils/__init__.py index 587564dcb0..3f74e4fa3d 100644 --- a/tmt/utils/__init__.py +++ b/tmt/utils/__init__.py @@ -27,6 +27,7 @@ from collections import Counter from collections.abc import Iterable, Iterator, Sequence from contextlib import suppress +from importlib.abc import Traversable from math import ceil from re import Pattern from threading import Thread @@ -61,6 +62,8 @@ from ruamel.yaml.representer import Representer import tmt.log +from tmt._compat.importlib.metadata import entry_points +from tmt._compat.importlib.readers import MultiplexedPath from tmt._compat.pathlib import Path from tmt.log import LoggableValue @@ -4638,6 +4641,28 @@ def _patch_plan_schema(schema: Schema, store: SchemaStore) -> None: } +def walk( + traversable: Traversable, + ) -> Iterator[tuple[Traversable, list[str], list[str]]]: + """ + Basic equivalent of os.walk/pathlib.Path.walk for more general Traversable. + + Does not cover the full interface of os.walk, e.g. top_down, follow_symlinks, etc. + """ + paths = [traversable] + while paths: + path = paths.pop() + dirnames = [] + filenames = [] + for entry in path.iterdir(): + if entry.is_dir(): + paths.append(entry) + dirnames.append(entry.name) + else: + filenames.append(entry.name) + yield path, dirnames, filenames + + def _load_schema(schema_filepath: Path) -> Schema: """ Load a JSON schema from a given filepath. @@ -4645,11 +4670,13 @@ def _load_schema(schema_filepath: Path) -> Schema: A helper returning the raw loaded schema. """ + # Broadening the allowed types to any Traversable + schema_file: Traversable = schema_filepath if not schema_filepath.is_absolute(): - schema_filepath = resource_files('schemas') / schema_filepath + schema_file = resource_files('schemas') / str(schema_filepath) try: - with open(schema_filepath, encoding='utf-8') as f: + with schema_file.open(encoding='utf-8') as f: return cast(Schema, yaml_to_dict(f.read())) except Exception as exc: @@ -4687,14 +4714,20 @@ def load_schema_store() -> SchemaStore: schema_dirpath = resource_files('schemas') try: - for filepath in schema_dirpath.glob('**/*ml'): - # Ignore all files but YAML files. - if filepath.suffix.lower() not in ('.yaml', '.yml'): - continue + for root, _, files in walk(schema_dirpath): + for file_name in files: + filepath = root / file_name + # Files are always of type pathlib.Path, we can safely narrow type it here + assert isinstance(filepath, pathlib.Path) # narrow type # noqa: TID251 + filepath = Path(filepath) + + # Ignore all files but YAML files. + if filepath.suffix.lower() not in ('.yaml', '.yml'): + continue - schema = _load_schema(filepath) + schema = _load_schema(filepath) - store[schema['$id']] = schema + store[schema['$id']] = schema except Exception as exc: raise FileError(f"Failed to discover schema files\n{exc}") @@ -5885,21 +5918,86 @@ def is_key_origin(node: fmf.Tree, key: str) -> bool: return origin is not None and node.name == origin.name -def resource_files(path: Union[str, Path], package: Union[str, ModuleType] = "tmt") -> Path: +@functools.cache +def _get_resource_files_search_path(package: Union[str, ModuleType], + logger: tmt.log.Logger) -> Traversable: + """ + Helper (cached) function for :py:func:`resource_files`. + + :param package: primary package in which to search for the file/directory. + :param logger: logger to report plugin import failures + :returns: the search path for resource files + """ + def accumulate_path(paths: list[Traversable], + pkg_path: Traversable) -> None: + if isinstance(pkg_path, MultiplexedPath): + # The root resources.files can be a MultiplexedPath if it is a namespace + paths.extend(pkg_path._paths) + else: + # Otherwise it should be a normal Path, just add it as-is + paths.append(pkg_path) + + # Accumulate the base path of the package and entry-points + base_paths: list[Traversable] = [] + + main_path = importlib.resources.files(package) + accumulate_path(base_paths, main_path) + + # Additional resource files can be imported from entry-point + entry_point_name = 'tmt.resources' + + entry_point_group = entry_points(group=entry_point_name) + + for ep in entry_point_group: + try: + ep_module = ep.load() + ep_path = importlib.resources.files(ep_module) + accumulate_path(base_paths, ep_path) + except ModuleNotFoundError: + logger.warning(f"Failed to load plugin resources: {ep}") + except Exception as err: + # Other exceptions are rather weird + logger.warning(f"Unexpected failure in parsing: {ep}\n{err}") + + # Extract the Path if only one was found + if len(base_paths) == 1: + return base_paths[0] + # Otherwise construct a MultiplexedPath + assert (len(base_paths) > 1) + # Note: ignore[no-untyped-call]: importlib_resources does not type-hint + # the constructor correctly + return MultiplexedPath(*base_paths) # type: ignore[no-untyped-call] + + +def resource_files( + path: str, + package: Union[str, ModuleType] = "tmt", + logger: Optional[tmt.log.Logger] = None, + ) -> Traversable: """ Helper function to get path of package file or directory. A thin wrapper for :py:func:`importlib.resources.files`: - ``files()`` returns ``Traversable`` object, though in our use-case - it should always produce a :py:class:`pathlib.PosixPath` object. - Converting it to :py:class:`tmt.utils.Path` instance should be - safe and stick to the "``Path`` only!" rule in tmt's code base. - - :param path: file or directory path to retrieve, relative to the ``package`` root. - :param package: package in which to search for the file/directory. - :returns: an absolute path to the requested file or directory. + ``files()`` returns ``Traversable`` object that can be either a + :py:class:`pathlib.PosixPath` or a :py:class:`importlib.reader.MultiplexedPath`. + + If the final path is a file, go ahead and convert it to a regular ``Path``, otherwise + keep it as a traversable in order to properly support MultiplexedPaths. + + Additional search paths are introduced from entry-point definitions + + :param path: file or directory path to retrieve, relative to the ``package`` + or entry-point's root. + :param package: primary package in which to search for the file/directory. + :param logger: logger to report plugin import failures + :returns: a traversable path to the requested file or directory. """ - return Path(importlib.resources.files(package)) / path # type: ignore[arg-type] + + if not logger: + # Make sure there is a logger to report entry-point failures + logger = tmt.log.Logger.get_bootstrap_logger() + search_path = _get_resource_files_search_path(package, logger) + return search_path / path class Stopwatch(contextlib.AbstractContextManager['Stopwatch']): diff --git a/tmt/utils/templates.py b/tmt/utils/templates.py index 4e787f1d87..d45837105e 100644 --- a/tmt/utils/templates.py +++ b/tmt/utils/templates.py @@ -7,6 +7,7 @@ import re import textwrap +from importlib.abc import Traversable from re import Match from typing import ( Any, @@ -333,7 +334,7 @@ def default_template_environment() -> jinja2.Environment: def render_template( template: str, - template_filepath: Optional[Path] = None, + template_filepath: Optional[Traversable] = None, environment: Optional[jinja2.Environment] = None, **variables: Any ) -> str: @@ -374,7 +375,7 @@ def raise_error(message: str) -> None: def render_template_file( - template_filepath: Path, + template_filepath: Traversable, environment: Optional[jinja2.Environment] = None, **variables: Any ) -> str: