From 22d2c7c95e7f492245d462757f7cfde695b31c90 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Sun, 20 Oct 2019 02:18:43 +0700 Subject: [PATCH] Add support options from requirements.txt in pip-sync --- piptools/repositories/pypi.py | 5 +- piptools/scripts/sync.py | 109 +++++++++++++++++++++++++++------- tests/test_cli_sync.py | 73 +++++++++++++++-------- 3 files changed, 139 insertions(+), 48 deletions(-) diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 588a8dc23..d836c97e2 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -67,7 +67,10 @@ class PyPIRepository(BaseRepository): changed/configured on the Finder. """ - def __init__(self, pip_args, build_isolation=False): + def __init__(self, pip_args=None, build_isolation=False): + if pip_args is None: + pip_args = [] + self.build_isolation = build_isolation # Use pip's parser for pip.conf management and defaults. diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 40c086a4f..cf1cbc3a1 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -1,13 +1,15 @@ # coding: utf-8 from __future__ import absolute_import, division, print_function, unicode_literals +import itertools import os import sys from .. import click, sync -from .._compat import get_installed_distributions, parse_requirements +from .._compat import PIP_VERSION, get_installed_distributions, parse_requirements from ..exceptions import PipToolsError from ..logging import log +from ..repositories import PyPIRepository from ..utils import flat_map DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @@ -104,8 +106,15 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) + repository = PyPIRepository() + + # Parse requirements file. Note, all options inside requirements file + # will be collected by the finder. requirements = flat_map( - lambda src: parse_requirements(src, session=True), src_files + lambda src: parse_requirements( + src, finder=repository.finder, session=repository.session + ), + src_files, ) try: @@ -117,26 +126,17 @@ def cli( installed_dists = get_installed_distributions(skip=[], user_only=user_only) to_install, to_uninstall = sync.diff(requirements, installed_dists) - install_flags = [] - for link in find_links or []: - install_flags.extend(["-f", link]) - if no_index: - install_flags.append("--no-index") - if index_url: - install_flags.extend(["-i", index_url]) - if extra_index_url: - for extra_index in extra_index_url: - install_flags.extend(["--extra-index-url", extra_index]) - if trusted_host: - for host in trusted_host: - install_flags.extend(["--trusted-host", host]) - if user_only: - install_flags.append("--user") - if cert: - install_flags.extend(["--cert", cert]) - if client_cert: - install_flags.extend(["--client-cert", client_cert]) - + install_flags = _build_install_flags( + repository.finder, + no_index=no_index, + index_url=index_url, + extra_index_url=extra_index_url, + trusted_host=trusted_host, + find_links=find_links, + user_only=user_only, + cert=cert, + client_cert=client_cert, + ) sys.exit( sync.sync( to_install, @@ -147,3 +147,68 @@ def cli( ask=ask, ) ) + + +def _build_install_flags( + finder, + no_index=False, + index_url=None, + extra_index_url=None, + trusted_host=None, + find_links=None, + user_only=False, + cert=None, + client_cert=None, +): + """ + Builds install flags with the given finder and CLI options. + """ + result = [] + + # Build --index-url/--extra-index-url/--no-index + if no_index: + result.append("--no-index") + elif index_url: + result.extend(["--index-url", index_url]) + elif finder.index_urls: + finder_index_url = finder.index_urls[0] + if finder_index_url != PyPIRepository.DEFAULT_INDEX_URL: + result.extend(["--index-url", finder_index_url]) + for extra_index in finder.index_urls[1:]: + result.extend(["--extra-index-url", extra_index]) + else: + result.append("--no-index") + + for extra_index in extra_index_url or []: + result.extend(["--extra-index-url", extra_index]) + + # Build --trusted-hosts + if PIP_VERSION < (19, 2): + finder_trusted_hosts = (host for _, host, _ in finder.secure_origins) + else: + finder_trusted_hosts = finder.trusted_hosts + for host in itertools.chain(trusted_host or [], finder_trusted_hosts): + result.extend(["--trusted-host", host]) + + # Build --find-links + for link in itertools.chain(find_links or [], finder.find_links): + result.extend(["--find-links", link]) + + # Build format controls --no-binary/--only-binary + for format_control in ("no_binary", "only_binary"): + formats = getattr(finder.format_control, format_control) + if formats: + result.extend( + ["--" + format_control.replace("_", "-"), ",".join(sorted(formats))] + ) + + if user_only: + result.append("--user") + + if cert: + result.extend(["--cert", cert]) + + if client_cert: + result.extend(["--client-cert", client_cert]) + + return result diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index fe54594b5..787028475 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -5,7 +5,7 @@ from .utils import invoke -from piptools.scripts.sync import cli +from piptools.scripts.sync import DEFAULT_REQUIREMENTS_FILE, cli def test_run_as_module_sync(): @@ -111,42 +111,65 @@ def test_merge_error(runner): @pytest.mark.parametrize( - ("cli_flags", "expected_install_flags"), + "install_flags", [ - (["--find-links", "./libs"], ["-f", "./libs"]), - (["--no-index"], ["--no-index"]), - (["--index-url", "https://example.com"], ["-i", "https://example.com"]), - ( - ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"], - ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"], - ), - ( - ["--trusted-host", "https://foo", "--trusted-host", "https://bar"], - ["--trusted-host", "https://foo", "--trusted-host", "https://bar"], - ), - ( - ["--extra-index-url", "https://foo", "--trusted-host", "https://bar"], - ["--extra-index-url", "https://foo", "--trusted-host", "https://bar"], - ), - (["--user"], ["--user"]), - (["--cert", "foo.crt"], ["--cert", "foo.crt"]), - (["--client-cert", "foo.pem"], ["--client-cert", "foo.pem"]), + ["--find-links", "./libs1", "--find-links", "./libs2"], + ["--no-index"], + ["--index-url", "https://example.com"], + ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"], + ["--trusted-host", "foo", "--trusted-host", "bar"], + ["--user"], + ["--cert", "foo.crt"], + ["--client-cert", "foo.pem"], ], ) @mock.patch("piptools.sync.check_call") -def test_pip_install_flags(check_call, cli_flags, expected_install_flags, runner): +def test_pip_install_flags(check_call, install_flags, runner): """ Test the cli flags have to be passed to the pip install command. """ with open("requirements.txt", "w") as req_in: req_in.write("six==1.10.0") - runner.invoke(cli, cli_flags) + runner.invoke(cli, install_flags) call_args = [call[0][0] for call in check_call.call_args_list] - assert [args[6:] for args in call_args if args[3] == "install"] == [ - expected_install_flags - ] + called_install_options = [args[6:] for args in call_args if args[3] == "install"] + assert called_install_options == [install_flags], "Called args: {}".format( + call_args + ) + + +@pytest.mark.parametrize( + "install_flags", + [ + ["--no-index"], + ["--index-url", "https://example.com"], + ["--extra-index-url", "https://example.com"], + ["--find-links", "./libs1"], + ["--trusted-host", "example.com"], + ["--no-binary", ":all:"], + ["--only-binary", ":all:"], + ], +) +@mock.patch("piptools.sync.check_call") +def test_pip_install_flags_in_requirements_file(check_call, runner, install_flags): + """ + Test the options from requirements.txt file pass to the pip install command. + """ + with open(DEFAULT_REQUIREMENTS_FILE, "w") as reqs: + reqs.write(" ".join(install_flags) + "\n") + reqs.write("six==1.10.0") + + out = runner.invoke(cli) + assert out.exit_code == 0, out + + # Make sure pip install command has expected options + call_args = [call[0][0] for call in check_call.call_args_list] + called_install_options = [args[6:] for args in call_args if args[3] == "install"] + assert called_install_options == [install_flags], "Called args: {}".format( + call_args + ) @mock.patch("piptools.sync.check_call")