From 32f43b7facf8081bbbd4fcdcc74209e388773e3f Mon Sep 17 00:00:00 2001 From: Miroslav Vadkerti Date: Tue, 26 Mar 2024 00:22:24 +0100 Subject: [PATCH] Support Alpine Linux `apk` package manager Add support for `apk` package manager from Alpine Linux. Unsupported debuginfo installation, seems not have a nice `debuginfo-install`-like solution. Installing by path is limited to certain recognized paths. The existing `apk-file` solution looks flaky, it was randomly hanging and does not have a good machine readable output. For testing we need to use a locally build alpine image which contains required packages of tmt and the unit tests. Resolves #2694 Signed-off-by: Miroslav Vadkerti --- Makefile | 6 + containers/Containerfile.alpine | 4 + docs/releases.rst | 5 +- pyproject.toml | 2 +- tests/unit/test.sh | 2 + tests/unit/test_package_managers.py | 88 ++++++++++++++- tmt/package_managers/apk.py | 163 ++++++++++++++++++++++++++++ 7 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 containers/Containerfile.alpine create mode 100644 tmt/package_managers/apk.py diff --git a/Makefile b/Makefile index c422a865da..718275b980 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ # Prepare variables TMP = $(CURDIR)/tmp +UNIT_TESTS_IMAGE_TAG = tmt-unit-tests # Define special targets .DEFAULT_GOAL := help @@ -82,6 +83,11 @@ images: ## Build tmt images for podman/docker podman build -t tmt --squash -f ./containers/Containerfile.mini . podman build -t tmt-all --squash -f ./containers/Containerfile.full . +images-unit-tests: image-alpine ## Build images for unit tests + +image-alpine: ## Build local alpine image for unit tests + podman build -t alpine:$(UNIT_TESTS_IMAGE_TAG) -f ./containers/Containerfile.alpine . + ## ## Development ## diff --git a/containers/Containerfile.alpine b/containers/Containerfile.alpine new file mode 100644 index 0000000000..d5603cdcc9 --- /dev/null +++ b/containers/Containerfile.alpine @@ -0,0 +1,4 @@ +FROM docker.io/library/alpine:latest + +# tmt requires `bash` to be installed +RUN apk add --no-cache bash diff --git a/docs/releases.rst b/docs/releases.rst index 40dd468a69..246e72acff 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -28,8 +28,9 @@ Internal implementation of basic package manager actions has been refactored. tmt now supports package implementations to be shipped as plugins, therefore allowing for tmt to work natively with distributions beyond the ecosystem of rpm-based distributions. As a preview, ``apt``, -the package manager used by Debian and Ubuntu, has been included in this -release. +the package manager used by Debian and Ubuntu, ``rpm-ostree``, the +package manager used by ``rpm-ostree``-based Linux systems and ``apk``, +the package manager of Alpine Linux have been included in this release. New environment variable ``TMT_TEST_ITERATION_ID`` has been added to :ref:`test-variables`. This variable is a combination of a unique diff --git a/pyproject.toml b/pyproject.toml index 78e4281e35..81117e94e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,7 +145,7 @@ lint = ["autopep8 {args:.}", "ruff --fix {args:.}"] type = ["mypy {args:tmt}"] check = ["lint", "type"] -unit = "pytest -vvv -ra --showlocals -n 0 tests/unit" +unit = "make images-unit-tests && pytest -vvv -ra --showlocals -n 0 tests/unit" smoke = "pytest -vvv -ra --showlocals -n 0 tests/unit/test_cli.py" cov = [ "coverage run --source=tmt -m pytest -vvv -ra --showlocals -n 0 tests", diff --git a/tests/unit/test.sh b/tests/unit/test.sh index 7c2094bb6a..5386441740 100755 --- a/tests/unit/test.sh +++ b/tests/unit/test.sh @@ -35,6 +35,8 @@ rlJournalStart rlLogInfo "pip is $(which pip), $(pip --version)" rlLogInfo "hatch is $(which hatch), $(hatch --version)" + + rlRun "make -C $TMT_TREE images-unit-tests" rlPhaseEnd if [ "$WITH_SYSTEM_PACKAGES" = "yes" ]; then diff --git a/tests/unit/test_package_managers.py b/tests/unit/test_package_managers.py index 0ebc0275e9..6511ecb221 100644 --- a/tests/unit/test_package_managers.py +++ b/tests/unit/test_package_managers.py @@ -42,6 +42,8 @@ CONTAINER_CENTOS_7 = Container(url='quay.io/centos/centos:7') CONTAINER_UBUNTU_2204 = Container(url='docker.io/library/ubuntu:22.04') CONTAINER_FEDORA_COREOS = Container(url='quay.io/fedora/fedora-coreos:stable') +# Local image created via `make image-alpine`, reference to local registry +CONTAINER_ALPINE = Container(url='containers-storage:localhost/alpine:tmt-unit-tests') PACKAGE_MANAGER_DNF5 = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('dnf5') PACKAGE_MANAGER_DNF = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('dnf') @@ -49,6 +51,7 @@ PACKAGE_MANAGER_APT = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('apt') PACKAGE_MANAGER_RPMOSTREE = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY \ .get_plugin('rpm-ostree') +PACKAGE_MANAGER_APK = tmt.package_managers._PACKAGE_MANAGER_PLUGIN_REGISTRY.get_plugin('apk') CONTAINER_BASE_MATRIX = [ @@ -73,6 +76,9 @@ # Fedora CoreOS (CONTAINER_FEDORA_COREOS, PACKAGE_MANAGER_RPMOSTREE), + + # Alpine + (CONTAINER_ALPINE, PACKAGE_MANAGER_APK), ] CONTAINER_MATRIX_IDS = [ @@ -181,6 +187,13 @@ def _parametrize_test_install() -> \ 'Installing: tree', \ None # noqa: E501 + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + r"apk info -e tree \|\| apk add tree", \ + 'Installing tree', \ + None + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") @@ -277,6 +290,13 @@ def _parametrize_test_install_nonexistent() -> \ 'no package provides tree-but-spelled-wrong', \ 'error: Packages not found: tree-but-spelled-wrong' # noqa: E501 + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + r"apk info -e tree-but-spelled-wrong \|\| apk add tree-but-spelled-wrong", \ + None, \ + 'ERROR: unable to select packages:\n tree-but-spelled-wrong (no such package):\n required by: world[tree-but-spelled-wrong]' # noqa: E501 + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") @@ -373,6 +393,13 @@ def _parametrize_test_install_nonexistent_skip() -> \ 'no package provides tree-but-spelled-wrong', \ 'error: Packages not found: tree-but-spelled-wrong' # noqa: E501 + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + r"apk info -e tree-but-spelled-wrong \|\| apk add tree-but-spelled-wrong \|\| /bin/true", \ + None, \ + 'ERROR: unable to select packages:\n tree-but-spelled-wrong (no such package):\n required by: world[tree-but-spelled-wrong]' # noqa: E501 + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") @@ -453,6 +480,13 @@ def _parametrize_test_install_dont_check_first() -> \ 'Installing: tree', \ None + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + r"apk add tree", \ + 'Installing tree', \ + None + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") @@ -506,6 +540,7 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ if 'centos:7' in container.url: yield container, \ package_manager_class, \ + Package('tar'), \ True, \ r"rpm -q --whatprovides tar && yum reinstall -y tar && rpm -q --whatprovides tar", \ 'Reinstalling:\n tar', \ @@ -514,6 +549,7 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ else: yield container, \ package_manager_class, \ + Package('tar'), \ True, \ r"rpm -q --whatprovides tar && yum reinstall -y tar && rpm -q --whatprovides tar", \ 'Reinstalled:\n tar', \ @@ -522,6 +558,7 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ elif package_manager_class is tmt.package_managers.dnf.Dnf: yield container, \ package_manager_class, \ + Package('tar'), \ True, \ r"rpm -q --whatprovides tar && dnf reinstall -y tar", \ 'Reinstalled:\n tar', \ @@ -530,6 +567,7 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ elif package_manager_class is tmt.package_managers.dnf.Dnf5: yield container, \ package_manager_class, \ + Package('tar'), \ True, \ r"rpm -q --whatprovides tar && dnf5 reinstall -y tar", \ 'Reinstalling tar', \ @@ -538,6 +576,7 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ elif package_manager_class is tmt.package_managers.apt.Apt: yield container, \ package_manager_class, \ + Package('tar'), \ True, \ r"export DEBIAN_FRONTEND=noninteractive; dpkg-query --show tar && apt reinstall -y tar", \ 'Setting up tar', \ @@ -546,11 +585,21 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ elif package_manager_class is tmt.package_managers.rpm_ostree.RpmOstree: yield container, \ package_manager_class, \ + Package('tar'), \ False, \ None, \ None, \ None + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + Package('bash'), \ + True, \ + r"apk info -e bash && apk fix bash", \ + 'Reinstalling bash', \ + None + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") @@ -558,6 +607,7 @@ def _parametrize_test_reinstall() -> Iterator[tuple[ @pytest.mark.containers() @pytest.mark.parametrize(('container_per_test', 'package_manager_class', + 'package', 'supported', 'expected_command', 'expected_stdout', @@ -569,6 +619,7 @@ def test_reinstall( container_per_test: ContainerData, guest_per_test: GuestContainer, package_manager_class: PackageManagerClass, + package: Package, supported: bool, expected_command: Optional[str], expected_stdout: Optional[str], @@ -584,14 +635,14 @@ def test_reinstall( if supported: assert expected_command is not None - output = package_manager.reinstall(Package('tar')) + output = package_manager.reinstall(package) assert_log(caplog, message=MATCH( rf"Run command: podman exec .+? /bin/bash -c '{expected_command}'")) else: with pytest.raises(tmt.utils.GeneralError) as excinfo: - package_manager.reinstall(Package('tar')) + package_manager.reinstall(package) assert excinfo.value.message \ == "rpm-ostree does not support reinstall operation." @@ -666,6 +717,14 @@ def _generate_test_reinstall_nonexistent_matrix() -> Iterator[tuple[ None, \ None + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + True, \ + r"apk info -e tree-but-spelled-wrong && apk fix tree-but-spelled-wrong", \ + None, \ + '' + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") @@ -951,6 +1010,31 @@ def _generate_test_check_presence() -> Iterator[ r'\s+out:\s+util-linux-core', \ None + elif package_manager_class is tmt.package_managers.apk.Apk: + yield container, \ + package_manager_class, \ + Package('busybox'), \ + True, \ + r"apk info -e busybox", \ + r'\s+out:\s+busybox', \ + None + + yield container, \ + package_manager_class, \ + Package('tree-but-spelled-wrong'), \ + False, \ + r"apk info -e tree-but-spelled-wrong", \ + None, \ + '' + + yield container, \ + package_manager_class, \ + FileSystemPath('/usr/bin/arch'), \ + True, \ + r"apk info -e busybox", \ + r'\s+out:\s+busybox', \ + None + else: pytest.fail(f"Unhandled package manager class '{package_manager_class}'.") diff --git a/tmt/package_managers/apk.py b/tmt/package_managers/apk.py new file mode 100644 index 0000000000..6180f2c437 --- /dev/null +++ b/tmt/package_managers/apk.py @@ -0,0 +1,163 @@ +import re +from typing import Optional, Union + +import tmt.package_managers +import tmt.utils +from tmt.package_managers import ( + FileSystemPath, + Installable, + Options, + Package, + PackagePath, + escape_installables, + provides_package_manager, + ) +from tmt.utils import ( + Command, + CommandOutput, + GeneralError, + RunError, + ShellScript, + ) + +ReducedPackages = list[Union[Package, PackagePath]] + +PACKAGE_PATH: dict[FileSystemPath, str] = { + FileSystemPath('/usr/bin/arch'): 'busybox', + FileSystemPath('/usr/bin/flock'): 'flock' + } + + +@provides_package_manager('apk') +class Apk(tmt.package_managers.PackageManager): + probe_command = Command('apk', '--version') + + install_command = Command('add') + + _sudo_prefix: Command + + def prepare_command(self) -> tuple[Command, Command]: + """ Prepare installation command for apk """ + + if self.guest.facts.is_superuser is False: + self._sudo_prefix = Command('sudo') + + else: + self._sudo_prefix = Command() + + command = Command() + + command += self._sudo_prefix + command += Command('apk') + + return (command, Command()) + + def path_to_package(self, path: FileSystemPath) -> Package: + """ + Find a package providing given filesystem path. + + This is not easily possible in Alpine. There is `apk-file` utility + available but it seems unrealiable. Support only a fixed set + of mappings until a better solution is available. + """ + + if path in PACKAGE_PATH: + return Package(PACKAGE_PATH[path]) + + raise GeneralError(f"Unsupported package path '{path} for Alpine Linux.") + + def _reduce_to_packages(self, *installables: Installable) -> ReducedPackages: + packages: ReducedPackages = [] + + for installable in installables: + if isinstance(installable, (Package, PackagePath)): + packages.append(installable) + + elif isinstance(installable, FileSystemPath): + packages.append(self.path_to_package(installable)) + + else: + raise GeneralError(f"Package specification '{installable}' is not supported.") + + return packages + + def _construct_presence_script( + self, *installables: Installable) -> tuple[ReducedPackages, ShellScript]: + reduced_packages = self._reduce_to_packages(*installables) + + shell_script = ShellScript( + f'apk info -e {" ".join(escape_installables(*reduced_packages))}') + + return reduced_packages, shell_script + + def check_presence(self, *installables: Installable) -> dict[Installable, bool]: + reduced_packages, presence_script = self._construct_presence_script(*installables) + + try: + output = self.guest.execute(presence_script) + stdout, stderr = output.stdout, output.stderr + + except RunError as exc: + stdout, stderr = exc.stdout, exc.stderr + + if stdout is None or stderr is None: + raise GeneralError("apk presence check output provided no output") + + results: dict[Installable, bool] = {} + + for installable, package in zip(installables, reduced_packages): + match = re.search(rf'^{re.escape(str(package))}\s', stdout) + + if match is not None: + results[installable] = True + continue + + results[installable] = False + + return results + + def install( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + options = options or Options() + + packages = self._reduce_to_packages(*installables) + + script = ShellScript( + f'{self.command.to_script()} {self.install_command.to_script()} ' + f'{" ".join(escape_installables(*packages))}') + + if options.check_first: + script = self._construct_presence_script(*packages)[1] | script + + if options.skip_missing: + script = script | ShellScript('/bin/true') + + return self.guest.execute(script) + + def reinstall( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + options = options or Options() + + packages = self._reduce_to_packages(*installables) + + script = ShellScript( + f'{self.command.to_script()} fix ' + f'{" ".join(escape_installables(*packages))}') + + if options.check_first: + script = self._construct_presence_script(*packages)[1] & script + + if options.skip_missing: + script = script | ShellScript('/bin/true') + + return self.guest.execute(script) + + def install_debuginfo( + self, + *installables: Installable, + options: Optional[Options] = None) -> CommandOutput: + raise tmt.utils.GeneralError("There is no support for debuginfo packages in apk.")