Support options in requirements.txt in pip-sync

piptools/scripts/sync.py

@@ -1,14 +1,16 @@
 # coding: utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import itertools
 import os
 import sys
 
 from .. import click, sync
 from .._compat import get_installed_distributions, parse_requirements
 from ..exceptions import PipToolsError
 from ..logging import log
-from ..utils import flat_map
+from ..repositories import PyPIRepository
+from ..utils import create_install_command, flat_map, get_trusted_hosts
 
 DEFAULT_REQUIREMENTS_FILE = "requirements.txt"
 
@@ -104,8 +106,15 @@ def cli(
  log.error("ERROR: " + msg)
  sys.exit(2)
 
+ install_command = create_install_command()
+ options, _ = install_command.parse_args([])
+ session = install_command._build_session(options)
+ finder = install_command._build_package_finder(options=options, session=session)
+
+ # Parse requirements file. Note, all options inside requirements file
+ # will be collected by the finder.
  requirements = flat_map(
- lambda src: parse_requirements(src, session=True), src_files
+ lambda src: parse_requirements(src, finder=finder, session=session), src_files
  )
 
  try:
@@ -117,26 +126,17 @@ def cli(
  installed_dists = get_installed_distributions(skip=[], user_only=user_only)
  to_install, to_uninstall = sync.diff(requirements, installed_dists)
 
- install_flags = []
- for link in find_links or []:
- install_flags.extend(["-f", link])
- if no_index:
- install_flags.append("--no-index")
- if index_url:
- install_flags.extend(["-i", index_url])
- if extra_index_url:
- for extra_index in extra_index_url:
- install_flags.extend(["--extra-index-url", extra_index])
- if trusted_host:
- for host in trusted_host:
- install_flags.extend(["--trusted-host", host])
- if user_only:
- install_flags.append("--user")
- if cert:
- install_flags.extend(["--cert", cert])
- if client_cert:
- install_flags.extend(["--client-cert", client_cert])
-
+ install_flags = _compose_install_flags(
+ finder,
+ no_index=no_index,
+ index_url=index_url,
+ extra_index_url=extra_index_url,
+ trusted_host=trusted_host,
+ find_links=find_links,
+ user_only=user_only,
+ cert=cert,
+ client_cert=client_cert,
+ )
  sys.exit(
  sync.sync(
  to_install,
@@ -147,3 +147,65 @@ def cli(
  ask=ask,
  )
  )
+
+
+def _compose_install_flags(
+ finder,
+ no_index=False,
+ index_url=None,
+ extra_index_url=None,
+ trusted_host=None,
+ find_links=None,
+ user_only=False,
+ cert=None,
+ client_cert=None,
+):
+ """
+ Compose install flags with the given finder and CLI options.
+ """
+ result = []
+
+ # Build --index-url/--extra-index-url/--no-index
+ if no_index:
+ result.append("--no-index")
+ elif index_url:
+ result.extend(["--index-url", index_url])
+ elif finder.index_urls:
+ finder_index_url = finder.index_urls[0]
+ if finder_index_url != PyPIRepository.DEFAULT_INDEX_URL:
+ result.extend(["--index-url", finder_index_url])
+ for extra_index in finder.index_urls[1:]:
+ result.extend(["--extra-index-url", extra_index])
+ else:
+ result.append("--no-index")
+
+ for extra_index in extra_index_url or []:
+ result.extend(["--extra-index-url", extra_index])
+
+ # Build --trusted-hosts
+ for host in itertools.chain(trusted_host or [], get_trusted_hosts(finder)):
+ result.extend(["--trusted-host", host])
+
+ # Build --find-links
+ for link in itertools.chain(find_links or [], finder.find_links):
+ result.extend(["--find-links", link])
+
+ # Build format controls --no-binary/--only-binary
+ for format_control in ("no_binary", "only_binary"):
+ formats = getattr(finder.format_control, format_control)
+ if not formats:
+ continue
+ result.extend(
+ ["--" + format_control.replace("_", "-"), ",".join(sorted(formats))]
+ )
+
+ if user_only:
+ result.append("--user")
+
+ if cert:
+ result.extend(["--cert", cert])
+
+ if client_cert:
+ result.extend(["--client-cert", client_cert])
+
+ return result

tests/test_cli_sync.py

@@ -5,7 +5,7 @@
 
 from .utils import invoke
 
-from piptools.scripts.sync import cli
+from piptools.scripts.sync import DEFAULT_REQUIREMENTS_FILE, cli
 
 
 def test_run_as_module_sync():
@@ -111,42 +111,65 @@ def test_merge_error(runner):
 
 
 @pytest.mark.parametrize(
- ("cli_flags", "expected_install_flags"),
+ "install_flags",
  [
- (["--find-links", "./libs"], ["-f", "./libs"]),
- (["--no-index"], ["--no-index"]),
- (["--index-url", "https://example.com"], ["-i", "https://example.com"]),
- (
- ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
- ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
- ),
- (
- ["--trusted-host", "https://foo", "--trusted-host", "https://bar"],
- ["--trusted-host", "https://foo", "--trusted-host", "https://bar"],
- ),
- (
- ["--extra-index-url", "https://foo", "--trusted-host", "https://bar"],
- ["--extra-index-url", "https://foo", "--trusted-host", "https://bar"],
- ),
- (["--user"], ["--user"]),
- (["--cert", "foo.crt"], ["--cert", "foo.crt"]),
- (["--client-cert", "foo.pem"], ["--client-cert", "foo.pem"]),
+ ["--find-links", "./libs1", "--find-links", "./libs2"],
+ ["--no-index"],
+ ["--index-url", "https://example.com"],
+ ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
+ ["--trusted-host", "foo", "--trusted-host", "bar"],
+ ["--user"],
+ ["--cert", "foo.crt"],
+ ["--client-cert", "foo.pem"],
  ],
 )
 @mock.patch("piptools.sync.check_call")
-def test_pip_install_flags(check_call, cli_flags, expected_install_flags, runner):
+def test_pip_install_flags(check_call, install_flags, runner):
  """
  Test the cli flags have to be passed to the pip install command.
  """
  with open("requirements.txt", "w") as req_in:
  req_in.write("six==1.10.0")
 
- runner.invoke(cli, cli_flags)
+ runner.invoke(cli, install_flags)
 
  call_args = [call[0][0] for call in check_call.call_args_list]
- assert [args[6:] for args in call_args if args[3] == "install"] == [
- expected_install_flags
- ]
+ called_install_options = [args[6:] for args in call_args if args[3] == "install"]
+ assert called_install_options == [install_flags], "Called args: {}".format(
+ call_args
+ )
+
+
+@pytest.mark.parametrize(
+ "install_flags",
+ [
+ ["--no-index"],
+ ["--index-url", "https://example.com"],
+ ["--extra-index-url", "https://example.com"],
+ ["--find-links", "./libs1"],
+ ["--trusted-host", "example.com"],
+ ["--no-binary", ":all:"],
+ ["--only-binary", ":all:"],
+ ],
+)
+@mock.patch("piptools.sync.check_call")
+def test_pip_install_flags_in_requirements_file(check_call, runner, install_flags):
+ """
+ Test the options from requirements.txt file pass to the pip install command.
+ """
+ with open(DEFAULT_REQUIREMENTS_FILE, "w") as reqs:
+ reqs.write(" ".join(install_flags) + "\n")
+ reqs.write("six==1.10.0")
+
+ out = runner.invoke(cli)
+ assert out.exit_code == 0, out
+
+ # Make sure pip install command has expected options
+ call_args = [call[0][0] for call in check_call.call_args_list]
+ called_install_options = [args[6:] for args in call_args if args[3] == "install"]
+ assert called_install_options == [install_flags], "Called args: {}".format(
+ call_args
+ )
 
 
 @mock.patch("piptools.sync.check_call")