Skip to content

Commit

Permalink
Merge pull request #9522 from uranusjr/no-dup-fetch
Browse files Browse the repository at this point in the history
Make candidate download even lazier
  • Loading branch information
pradyunsg authored Jan 29, 2021
2 parents fa0ee31 + 7d43ec5 commit 55d2969
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 30 deletions.
3 changes: 3 additions & 0 deletions news/9516.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
New resolver: Download and prepare a distribution only at the last possible
moment to avoid unnecessary network access when the same version is already
installed locally.
19 changes: 8 additions & 11 deletions src/pip/_internal/resolution/resolvelib/factory.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import functools
import logging

from pip._vendor.packaging.utils import canonicalize_name
Expand Down Expand Up @@ -65,6 +66,7 @@

from .base import Candidate, Requirement
from .candidates import BaseCandidate
from .found_candidates import IndexCandidateInfo

C = TypeVar("C")
Cache = Dict[Link, C]
Expand Down Expand Up @@ -213,8 +215,8 @@ def _iter_found_candidates(
template=template,
)

def iter_index_candidates():
# type: () -> Iterator[Candidate]
def iter_index_candidate_infos():
# type: () -> Iterator[IndexCandidateInfo]
result = self._finder.find_best_candidate(
project_name=name,
specifier=specifier,
Expand All @@ -228,26 +230,21 @@ def iter_index_candidates():
all_yanked = all(ican.link.is_yanked for ican in icans)

# PackageFinder returns earlier versions first, so we reverse.
versions_found = set() # type: Set[_BaseVersion]
for ican in reversed(icans):
if not all_yanked and ican.link.is_yanked:
continue
if ican.version in versions_found:
continue
candidate = self._make_candidate_from_link(
func = functools.partial(
self._make_candidate_from_link,
link=ican.link,
extras=extras,
template=template,
name=name,
version=ican.version,
)
if candidate is None:
continue
yield candidate
versions_found.add(ican.version)
yield ican.version, func

return FoundCandidates(
iter_index_candidates,
iter_index_candidate_infos,
installed_candidate,
prefers_installed,
)
Expand Down
82 changes: 63 additions & 19 deletions src/pip/_internal/resolution/resolvelib/found_candidates.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,62 @@
"""

import functools
import itertools

from pip._vendor.six.moves import collections_abc # type: ignore

from pip._internal.utils.typing import MYPY_CHECK_RUNNING

if MYPY_CHECK_RUNNING:
from typing import Callable, Iterator, Optional
from typing import Callable, Iterator, Optional, Set, Tuple

from pip._vendor.packaging.version import _BaseVersion

from .base import Candidate

IndexCandidateInfo = Tuple[_BaseVersion, Callable[[], Optional[Candidate]]]


def _iter_built(infos):
# type: (Iterator[IndexCandidateInfo]) -> Iterator[Candidate]
"""Iterator for ``FoundCandidates``.
This iterator is used the package is not already installed. Candidates
from index come later in their normal ordering.
"""
versions_found = set() # type: Set[_BaseVersion]
for version, func in infos:
if version in versions_found:
continue
candidate = func()
if candidate is None:
continue
yield candidate
versions_found.add(version)


def _iter_built_with_prepended(installed, infos):
# type: (Candidate, Iterator[IndexCandidateInfo]) -> Iterator[Candidate]
"""Iterator for ``FoundCandidates``.
This iterator is used when the resolver prefers the already-installed
candidate and NOT to upgrade. The installed candidate is therefore
always yielded first, and candidates from index come later in their
normal ordering, except skipped when the version is already installed.
"""
yield installed
versions_found = {installed.version} # type: Set[_BaseVersion]
for version, func in infos:
if version in versions_found:
continue
candidate = func()
if candidate is None:
continue
yield candidate
versions_found.add(version)


def _insert_installed(installed, others):
# type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
def _iter_built_with_inserted(installed, infos):
# type: (Candidate, Iterator[IndexCandidateInfo]) -> Iterator[Candidate]
"""Iterator for ``FoundCandidates``.
This iterator is used when the resolver prefers to upgrade an
Expand All @@ -33,16 +75,22 @@ def _insert_installed(installed, others):
the installed candidate exactly once before we start yielding older or
equivalent candidates, or after all other candidates if they are all newer.
"""
installed_yielded = False
for candidate in others:
versions_found = set() # type: Set[_BaseVersion]
for version, func in infos:
if version in versions_found:
continue
# If the installed candidate is better, yield it first.
if not installed_yielded and installed.version >= candidate.version:
if installed.version >= version:
yield installed
installed_yielded = True
versions_found.add(installed.version)
candidate = func()
if candidate is None:
continue
yield candidate
versions_found.add(version)

# If the installed candidate is older than all other candidates.
if not installed_yielded:
if installed.version not in versions_found:
yield installed


Expand All @@ -56,11 +104,11 @@ class FoundCandidates(collections_abc.Sequence):
"""
def __init__(
self,
get_others, # type: Callable[[], Iterator[Candidate]]
get_infos, # type: Callable[[], Iterator[IndexCandidateInfo]]
installed, # type: Optional[Candidate]
prefers_installed, # type: bool
):
self._get_others = get_others
self._get_infos = get_infos
self._installed = installed
self._prefers_installed = prefers_installed

Expand All @@ -73,16 +121,12 @@ def __getitem__(self, index):

def __iter__(self):
# type: () -> Iterator[Candidate]
infos = self._get_infos()
if not self._installed:
return self._get_others()
others = (
candidate
for candidate in self._get_others()
if candidate.version != self._installed.version
)
return _iter_built(infos)
if self._prefers_installed:
return itertools.chain([self._installed], others)
return _insert_installed(self._installed, others)
return _iter_built_with_prepended(self._installed, infos)
return _iter_built_with_inserted(self._installed, infos)

def __len__(self):
# type: () -> int
Expand Down
25 changes: 25 additions & 0 deletions tests/functional/test_new_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -1253,3 +1253,28 @@ def test_new_resolver_lazy_fetch_candidates(script, upgrade):
# But should reach there in the best route possible, without trying
# candidates it does not need to.
assert "myuberpkg-2" not in result.stdout, str(result)


def test_new_resolver_no_fetch_no_satisfying(script):
create_basic_wheel_for_package(script, "myuberpkg", "1")

# Install the package. This should emit a "Processing" message for
# fetching the distribution from the --find-links page.
result = script.pip(
"install",
"--no-cache-dir", "--no-index",
"--find-links", script.scratch_path,
"myuberpkg",
)
assert "Processing " in result.stdout, str(result)

# Try to upgrade the package. This should NOT emit the "Processing"
# message because the currently installed version is latest.
result = script.pip(
"install",
"--no-cache-dir", "--no-index",
"--find-links", script.scratch_path,
"--upgrade",
"myuberpkg",
)
assert "Processing " not in result.stdout, str(result)

0 comments on commit 55d2969

Please sign in to comment.