Pass arbitrary args to 'pip' from pip-compile and pip-sync, with '--pip-args'

README.rst

@@ -201,6 +201,16 @@ Or to output to standard output, use ``--output-file=-``:
  $ pip-compile --output-file=- > requirements.txt
  $ pip-compile - --output-file=- < requirements.in > requirements.txt
 
+Forwarding options to ``pip``
+-----------------------------
+
+Any valid ``pip`` flags or arguments may be passed on with ``pip-compile``'s
+``--pip-args`` option, e.g.
+
+.. code-block:: bash
+
+ $ pip-compile requirements.in --pip-args '--retries 10 --timeout 30'
+
 Configuration
 -------------
 
@@ -368,6 +378,13 @@ line arguments, e.g.
 
 Passing in empty arguments would cause it to default to ``requirements.txt``.
 
+Any valid ``pip install`` flags or arguments may be passed with ``pip-sync``'s
+``--pip-args`` option, e.g.
+
+.. code-block:: bash
+
+ $ pip-sync requirements.txt --pip-args '--no-cache-dir --no-deps'
+
 If you use multiple Python versions, you can run ``pip-sync`` as
 ``py -X.Y -m piptools sync ...`` on Windows and
 ``pythonX.Y -m piptools sync ...`` on other systems.

piptools/scripts/compile.py

@@ -2,6 +2,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
+import shlex
 import sys
 import tempfile
 
@@ -177,6 +178,7 @@
  show_envvar=True,
  type=click.Path(file_okay=False, writable=True),
 )
+@click.option("--pip-args", help="Arguments to pass directly to the pip command.")
 def cli(
  ctx,
  verbose,
@@ -204,6 +206,7 @@ def cli(
  build_isolation,
  emit_find_links,
  cache_dir,
+ pip_args,
 ):
  """Compiles requirements.txt from requirements.in specs."""
  log.verbosity = verbose - quiet
@@ -247,6 +250,7 @@ def cli(
  # Setup
  ###
 
+ right_args = shlex.split(pip_args or "")
  pip_args = []
  if find_links:
  for link in find_links:
@@ -268,6 +272,7 @@ def cli(
 
  if not build_isolation:
  pip_args.append("--no-build-isolation")
+ pip_args.extend(right_args)
 
  repository = PyPIRepository(pip_args, cache_dir=cache_dir)
 

piptools/scripts/sync.py

@@ -3,6 +3,7 @@
 
 import itertools
 import os
+import shlex
 import sys
 
 from pip._internal.commands import create_command
@@ -73,6 +74,7 @@
  "the private key and the certificate in PEM format.",
 )
 @click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1)
+@click.option("--pip-args", help="Arguments to pass directly to pip install.")
 def cli(
  ask,
  dry_run,
@@ -87,6 +89,7 @@ def cli(
  cert,
  client_cert,
  src_files,
+ pip_args,
 ):
  """Synchronize virtual environment with requirements.txt."""
  if not src_files:
@@ -139,7 +142,7 @@ def cli(
  user_only=user_only,
  cert=cert,
  client_cert=client_cert,
- )
+ ) + shlex.split(pip_args or "")
  sys.exit(
  sync.sync(
  to_install,

piptools/utils.py

@@ -328,6 +328,10 @@ def get_compile_command(click_ctx):
  # Collect variadic args separately, they will be added
  # at the end of the command later
  if option.nargs < 0:
+ # These will necessarily be src_files
+ # Re-add click-stripped '--' if any start with '-'
+ if any(val.startswith("-") and val != "-" for val in value):
+ right_args.append("--")
  right_args.extend([shlex_quote(force_text(val)) for val in value])
  continue
 
@@ -366,10 +370,20 @@ def get_compile_command(click_ctx):
  left_args.append(shlex_quote(arg))
  # Append to args the option with a value
  else:
- left_args.append(
- "{option}={value}".format(
- option=option_long_name, value=shlex_quote(force_text(val))
+ if option.name == "pip_args":
+ # shlex_quote would produce functional but noisily quoted results,
+ # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"''
+ # Instead, we try to get more legible quoting via repr:
+ left_args.append(
+ "{option}={value}".format(
+ option=option_long_name, value=repr(fs_str(force_text(val)))
+ )
+ )
+ else:
+ left_args.append(
+ "{option}={value}".format(
+ option=option_long_name, value=shlex_quote(force_text(val))
+ )
  )
- )
 
  return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))

tests/test_cli_compile.py

@@ -730,7 +730,7 @@ def test_allow_unsafe_option(pip_conf, monkeypatch, runner, option, expected):
 @mock.patch("piptools.scripts.compile.parse_requirements")
 def test_cert_option(parse_requirements, runner, option, attr, expected):
  """
- The options --cert and --client-crt have to be passed to the PyPIRepository.
+ The options --cert and --client-cert have to be passed to the PyPIRepository.
  """
  with open("requirements.in", "w"):
  pass
@@ -759,6 +759,20 @@ def test_build_isolation_option(parse_requirements, runner, option, expected):
  assert parse_requirements.call_args.kwargs["options"].build_isolation is expected
 
 
+@mock.patch("piptools.scripts.compile.PyPIRepository")
+def test_forwarded_args(PyPIRepository, runner):
+ """
+ Test the forwarded cli args (--pip-args 'arg...') are passed to the pip command.
+ """
+ with open("requirements.in", "w"):
+ pass
+
+ cli_args = ("--no-annotate", "--generate-hashes")
+ pip_args = ("--no-color", "--isolated", "--disable-pip-version-check")
+ runner.invoke(cli, cli_args + ("--pip-args", " ".join(pip_args)))
+ assert set(pip_args).issubset(set(PyPIRepository.call_args.args[0]))
+
+
 @pytest.mark.parametrize(
  "cli_option, infile_option, expected_package",
  [

tests/test_cli_sync.py

@@ -111,31 +111,52 @@ def test_merge_error(runner):
 
 
 @pytest.mark.parametrize(
- "install_flags",
+ ("cli_flags", "expected_install_flags"),
  [
- ["--find-links", "./libs1", "--find-links", "./libs2"],
- ["--no-index"],
- ["--index-url", "https://example.com"],
- ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
- ["--trusted-host", "foo", "--trusted-host", "bar"],
- ["--user"],
- ["--cert", "foo.crt"],
- ["--client-cert", "foo.pem"],
+ (
+ ["--find-links", "./libs1", "--find-links", "./libs2"],
+ ["--find-links", "./libs1", "--find-links", "./libs2"],
+ ),
+ (["--no-index"], ["--no-index"]),
+ (
+ ["--index-url", "https://example.com"],
+ ["--index-url", "https://example.com"],
+ ),
+ (
+ ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
+ ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
+ ),
+ (
+ ["--trusted-host", "foo", "--trusted-host", "bar"],
+ ["--trusted-host", "foo", "--trusted-host", "bar"],
+ ),
+ (["--user"], ["--user"]),
+ (["--cert", "foo.crt"], ["--cert", "foo.crt"]),
+ (["--client-cert", "foo.pem"], ["--client-cert", "foo.pem"]),
+ (
+ ["--pip-args", "--no-cache-dir --no-deps --no-warn-script-location"],
+ ["--no-cache-dir", "--no-deps", "--no-warn-script-location"],
+ ),
+ (["--pip-args='--cache-dir=/tmp'"], ["--cache-dir=/tmp"]),
+ (
+ ["--pip-args=\"--cache-dir='/tmp/cache dir with spaces/'\""],
+ ["--cache-dir='/tmp/cache dir with spaces/'"],
+ ),
  ],
 )
 @mock.patch("piptools.sync.check_call")
-def test_pip_install_flags(check_call, install_flags, runner):
+def test_pip_install_flags(check_call, cli_flags, expected_install_flags, runner):
  """
  Test the cli flags have to be passed to the pip install command.
  """
  with open("requirements.txt", "w") as req_in:
  req_in.write("six==1.10.0")
 
- runner.invoke(cli, install_flags)
+ runner.invoke(cli, cli_flags)
 
  call_args = [call[0][0] for call in check_call.call_args_list]
  called_install_options = [args[6:] for args in call_args if args[3] == "install"]
- assert called_install_options == [install_flags], "Called args: {}".format(
+ assert called_install_options == [expected_install_flags], "Called args: {}".format(
  call_args
  )
 

tests/test_utils.py

@@ -265,6 +265,15 @@ def test_force_text(value, expected_text):
  (["-f", "συνδέσεις"], "pip-compile --find-links='συνδέσεις'"),
  (["-o", "my file.txt"], "pip-compile --output-file='my file.txt'"),
  (["-o", "απαιτήσεις.txt"], "pip-compile --output-file='απαιτήσεις.txt'"),
+ # Check '--pip-args' (forwarded) arguments
+ (
+ ["--pip-args", "--disable-pip-version-check"],
+ "pip-compile --pip-args='--disable-pip-version-check'",
+ ),
+ (
+ ["--pip-args", "--disable-pip-version-check --isolated"],
+ "pip-compile --pip-args='--disable-pip-version-check --isolated'",
+ ),
  ],
 )
 def test_get_compile_command(tmpdir_cwd, cli_args, expected_command):
@@ -275,6 +284,16 @@ def test_get_compile_command(tmpdir_cwd, cli_args, expected_command):
  assert get_compile_command(ctx) == expected_command
 
 
+def test_get_compile_command_escaped_filenames(tmpdir_cwd):
+ """
+ Test that get_compile_command output (re-)escapes ' -- '-escaped filenames.
+ """
+ with open("--requirements.in", "w"):
+ pass
+ with compile_cli.make_context("pip-compile", ["--", "--requirements.in"]) as ctx:
+ assert get_compile_command(ctx) == "pip-compile -- --requirements.in"
+
+
 @mark.parametrize(
  "filename", ["requirements.in", "my requirements.in", "απαιτήσεις.txt"]
 )