From 8a2efa1374677c9064e90ea175493d7019982e9f Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Tue, 18 Feb 2020 19:01:51 -0500 Subject: [PATCH] Implement pip argument forwarding for pip-sync and pip-compile Pass arbitrary args to 'pip install' from pip-sync, with '--pip-args "ARG..."' Pass arbitrary args to 'pip' from pip-compile, with '--pip-args "ARG..."' Add tests Add examples to README Account for ' -- ' (filename escapes) in get_compile_command, for more accurate pip-compile output headers User repr rather than shlex_quote in get_compile_command for --pip-args, to avoid noisy quoting --- README.rst | 17 ++++++++++++++ piptools/scripts/compile.py | 5 +++++ piptools/scripts/sync.py | 5 ++++- piptools/utils.py | 22 ++++++++++++++---- tests/test_cli_compile.py | 16 ++++++++++++- tests/test_cli_sync.py | 45 +++++++++++++++++++++++++++---------- tests/test_utils.py | 19 ++++++++++++++++ 7 files changed, 111 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index 8506b32d8..ebb3e5205 100644 --- a/README.rst +++ b/README.rst @@ -201,6 +201,16 @@ Or to output to standard output, use ``--output-file=-``: $ pip-compile --output-file=- > requirements.txt $ pip-compile - --output-file=- < requirements.in > requirements.txt +Forwarding options to ``pip`` +----------------------------- + +Any valid ``pip`` flags or arguments may be passed on with ``pip-compile``'s +``--pip-args`` option, e.g. + +.. code-block:: bash + + $ pip-compile requirements.in --pip-args '--retries 10 --timeout 30' + Configuration ------------- @@ -368,6 +378,13 @@ line arguments, e.g. Passing in empty arguments would cause it to default to ``requirements.txt``. +Any valid ``pip install`` flags or arguments may be passed with ``pip-sync``'s +``--pip-args`` option, e.g. + +.. code-block:: bash + + $ pip-sync requirements.txt --pip-args '--no-cache-dir --no-deps' + If you use multiple Python versions, you can run ``pip-sync`` as ``py -X.Y -m piptools sync ...`` on Windows and ``pythonX.Y -m piptools sync ...`` on other systems. diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index aa2a2a917..03232a849 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import os +import shlex import sys import tempfile @@ -177,6 +178,7 @@ show_envvar=True, type=click.Path(file_okay=False, writable=True), ) +@click.option("--pip-args", help="Arguments to pass directly to the pip command.") def cli( ctx, verbose, @@ -204,6 +206,7 @@ def cli( build_isolation, emit_find_links, cache_dir, + pip_args, ): """Compiles requirements.txt from requirements.in specs.""" log.verbosity = verbose - quiet @@ -247,6 +250,7 @@ def cli( # Setup ### + right_args = shlex.split(pip_args or "") pip_args = [] if find_links: for link in find_links: @@ -268,6 +272,7 @@ def cli( if not build_isolation: pip_args.append("--no-build-isolation") + pip_args.extend(right_args) repository = PyPIRepository(pip_args, cache_dir=cache_dir) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 94f738b59..137e8132e 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -3,6 +3,7 @@ import itertools import os +import shlex import sys from pip._internal.commands import create_command @@ -73,6 +74,7 @@ "the private key and the certificate in PEM format.", ) @click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1) +@click.option("--pip-args", help="Arguments to pass directly to pip install.") def cli( ask, dry_run, @@ -87,6 +89,7 @@ def cli( cert, client_cert, src_files, + pip_args, ): """Synchronize virtual environment with requirements.txt.""" if not src_files: @@ -139,7 +142,7 @@ def cli( user_only=user_only, cert=cert, client_cert=client_cert, - ) + ) + shlex.split(pip_args or "") sys.exit( sync.sync( to_install, diff --git a/piptools/utils.py b/piptools/utils.py index d90597d17..77334474f 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -328,6 +328,10 @@ def get_compile_command(click_ctx): # Collect variadic args separately, they will be added # at the end of the command later if option.nargs < 0: + # These will necessarily be src_files + # Re-add click-stripped '--' if any start with '-' + if any(val.startswith("-") and val != "-" for val in value): + right_args.append("--") right_args.extend([shlex_quote(force_text(val)) for val in value]) continue @@ -366,10 +370,20 @@ def get_compile_command(click_ctx): left_args.append(shlex_quote(arg)) # Append to args the option with a value else: - left_args.append( - "{option}={value}".format( - option=option_long_name, value=shlex_quote(force_text(val)) + if option.name == "pip_args": + # shlex_quote would produce functional but noisily quoted results, + # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"'' + # Instead, we try to get more legible quoting via repr: + left_args.append( + "{option}={value}".format( + option=option_long_name, value=repr(fs_str(force_text(val))) + ) + ) + else: + left_args.append( + "{option}={value}".format( + option=option_long_name, value=shlex_quote(force_text(val)) + ) ) - ) return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args)) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 8965bf021..3b3771ea7 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -730,7 +730,7 @@ def test_allow_unsafe_option(pip_conf, monkeypatch, runner, option, expected): @mock.patch("piptools.scripts.compile.parse_requirements") def test_cert_option(parse_requirements, runner, option, attr, expected): """ - The options --cert and --client-crt have to be passed to the PyPIRepository. + The options --cert and --client-cert have to be passed to the PyPIRepository. """ with open("requirements.in", "w"): pass @@ -759,6 +759,20 @@ def test_build_isolation_option(parse_requirements, runner, option, expected): assert parse_requirements.call_args.kwargs["options"].build_isolation is expected +@mock.patch("piptools.scripts.compile.PyPIRepository") +def test_forwarded_args(PyPIRepository, runner): + """ + Test the forwarded cli args (--pip-args 'arg...') are passed to the pip command. + """ + with open("requirements.in", "w"): + pass + + cli_args = ("--no-annotate", "--generate-hashes") + pip_args = ("--no-color", "--isolated", "--disable-pip-version-check") + runner.invoke(cli, cli_args + ("--pip-args", " ".join(pip_args))) + assert set(pip_args).issubset(set(PyPIRepository.call_args.args[0])) + + @pytest.mark.parametrize( "cli_option, infile_option, expected_package", [ diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 787028475..ffc21720c 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -111,31 +111,52 @@ def test_merge_error(runner): @pytest.mark.parametrize( - "install_flags", + ("cli_flags", "expected_install_flags"), [ - ["--find-links", "./libs1", "--find-links", "./libs2"], - ["--no-index"], - ["--index-url", "https://example.com"], - ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"], - ["--trusted-host", "foo", "--trusted-host", "bar"], - ["--user"], - ["--cert", "foo.crt"], - ["--client-cert", "foo.pem"], + ( + ["--find-links", "./libs1", "--find-links", "./libs2"], + ["--find-links", "./libs1", "--find-links", "./libs2"], + ), + (["--no-index"], ["--no-index"]), + ( + ["--index-url", "https://example.com"], + ["--index-url", "https://example.com"], + ), + ( + ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"], + ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"], + ), + ( + ["--trusted-host", "foo", "--trusted-host", "bar"], + ["--trusted-host", "foo", "--trusted-host", "bar"], + ), + (["--user"], ["--user"]), + (["--cert", "foo.crt"], ["--cert", "foo.crt"]), + (["--client-cert", "foo.pem"], ["--client-cert", "foo.pem"]), + ( + ["--pip-args", "--no-cache-dir --no-deps --no-warn-script-location"], + ["--no-cache-dir", "--no-deps", "--no-warn-script-location"], + ), + (["--pip-args='--cache-dir=/tmp'"], ["--cache-dir=/tmp"]), + ( + ["--pip-args=\"--cache-dir='/tmp/cache dir with spaces/'\""], + ["--cache-dir='/tmp/cache dir with spaces/'"], + ), ], ) @mock.patch("piptools.sync.check_call") -def test_pip_install_flags(check_call, install_flags, runner): +def test_pip_install_flags(check_call, cli_flags, expected_install_flags, runner): """ Test the cli flags have to be passed to the pip install command. """ with open("requirements.txt", "w") as req_in: req_in.write("six==1.10.0") - runner.invoke(cli, install_flags) + runner.invoke(cli, cli_flags) call_args = [call[0][0] for call in check_call.call_args_list] called_install_options = [args[6:] for args in call_args if args[3] == "install"] - assert called_install_options == [install_flags], "Called args: {}".format( + assert called_install_options == [expected_install_flags], "Called args: {}".format( call_args ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 567c3e41d..5a92dae87 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -265,6 +265,15 @@ def test_force_text(value, expected_text): (["-f", "συνδέσεις"], "pip-compile --find-links='συνδέσεις'"), (["-o", "my file.txt"], "pip-compile --output-file='my file.txt'"), (["-o", "απαιτήσεις.txt"], "pip-compile --output-file='απαιτήσεις.txt'"), + # Check '--pip-args' (forwarded) arguments + ( + ["--pip-args", "--disable-pip-version-check"], + "pip-compile --pip-args='--disable-pip-version-check'", + ), + ( + ["--pip-args", "--disable-pip-version-check --isolated"], + "pip-compile --pip-args='--disable-pip-version-check --isolated'", + ), ], ) def test_get_compile_command(tmpdir_cwd, cli_args, expected_command): @@ -275,6 +284,16 @@ def test_get_compile_command(tmpdir_cwd, cli_args, expected_command): assert get_compile_command(ctx) == expected_command +def test_get_compile_command_escaped_filenames(tmpdir_cwd): + """ + Test that get_compile_command output (re-)escapes ' -- '-escaped filenames. + """ + with open("--requirements.in", "w"): + pass + with compile_cli.make_context("pip-compile", ["--", "--requirements.in"]) as ctx: + assert get_compile_command(ctx) == "pip-compile -- --requirements.in" + + @mark.parametrize( "filename", ["requirements.in", "my requirements.in", "απαιτήσεις.txt"] )