From c624090f7af108965be1dabd2c4ccf05e06e8474 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:01:10 +0100 Subject: [PATCH 01/22] Try to build wheels for directories/VCS Follow up from gh-4144 To allow build system abstractions, we want installation to go through wheels in more cases. In particular, installing packages from a local directory or a VCS URL currently runs 'setup.py install'. The aim of this PR is to have it build a wheel, which is stored in an ephemeral cache directory, used for installation, and then discarded. We can't cache it permanently based on the path/URL, because the code there might change, but our cache wouldn't be invalidated. --- pip/commands/wheel.py | 52 +++++++++++++++++++++---------------------- pip/req/req_set.py | 3 +++ pip/wheel.py | 26 +++++++++++++--------- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index cc6476c27a5..155256d82cb 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -146,35 +146,35 @@ def run(self, options, args): require_hashes=options.require_hashes, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=options.wheel_dir, - progress_bar=options.progress_bar, - ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=options.wheel_dir, + progress_bar=options.progress_bar, + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=True, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + resolver = Resolver( + preparer=preparer, + finder=finder, + session=session, + wheel_cache=wheel_cache, + use_user_site=False, + upgrade_strategy="to-satisfy-only", + force_reinstall=False, + ignore_dependencies=options.ignore_dependencies, + ignore_requires_python=options.ignore_requires_python, + ignore_installed=True, + isolated=options.isolated_mode, + ) + resolver.resolve(requirement_set) - try: # build wheels wb = WheelBuilder( requirement_set, diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 1c8a35e29a5..5146211a27d 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -198,6 +198,9 @@ def cleanup_files(self): for req in self.reqs_to_cleanup: req.remove_temporary_source() + if self._wheel_cache: + self._wheel_cache.cleanup() + def _to_install(self): """Create the installation order. diff --git a/pip/wheel.py b/pip/wheel.py index ca080e24381..a6e2748b362 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -736,6 +736,7 @@ def build(self, session, autobuilding=False): buildset = [] for req in reqset: + ephem_cache = False if req.constraint: continue if req.is_wheel: @@ -745,7 +746,8 @@ def build(self, session, autobuilding=False): elif autobuilding and req.editable: pass elif autobuilding and req.link and not req.link.is_artifact: - pass + # VCS checkout. Build wheel just for this run. + ephem_cache = True elif autobuilding and not req.source_dir: pass else: @@ -753,17 +755,16 @@ def build(self, session, autobuilding=False): link = req.link base, ext = link.splitext() if pip.index.egg_info_matches(base, None, link) is None: - # Doesn't look like a package - don't autobuild a wheel - # because we'll have no way to lookup the result sanely - continue - if "binary" not in pip.index.fmt_ctl_formats( + # E.g. local directory. Build wheel just for this run. + ephem_cache = True + elif "binary" not in pip.index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue - buildset.append(req) + buildset.append((req, ephem_cache)) if not buildset: return True @@ -775,14 +776,19 @@ def build(self, session, autobuilding=False): ) with indent_log(): build_success, build_failure = [], [] - for req in buildset: + for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag # NOTE: Should move out a method on the cache directly. - output_dir = get_cache_path_for_link( - self.wheel_cache._cache_dir, req.link - ) + if ephem: + output_dir = get_cache_path_for_link( + self.wheel_cache._cache_dir, req.link + ) + else: + output_dir = get_cache_path_for_link( + self.wheel_cache._ephem_cache_dir, req.link + ) try: ensure_dir(output_dir) except OSError as e: From a32ebea3c423721ea10ee778f77ea1f034dc4cb9 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:06:07 +0100 Subject: [PATCH 02/22] Add news file --- news/4501.feature | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/4501.feature diff --git a/news/4501.feature b/news/4501.feature new file mode 100644 index 00000000000..475a1317224 --- /dev/null +++ b/news/4501.feature @@ -0,0 +1,3 @@ +Installing from a local directory or a VCS URL now builds a wheel to install, +rather than running ``setup.py install``. Wheels from these sources are not +cached. From fe5e9cc59069dbab63f499adffdb36697ae2d512 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:08:45 +0100 Subject: [PATCH 03/22] Fix list comp --- pip/wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/wheel.py b/pip/wheel.py index a6e2748b362..7d444bd1381 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -772,7 +772,7 @@ def build(self, session, autobuilding=False): # Build the wheels. logger.info( 'Building wheels for collected packages: %s', - ', '.join([req.name for req in buildset]), + ', '.join([req.name for (req, _) in buildset]), ) with indent_log(): build_success, build_failure = [], [] From 30109c3c1b0aaaec30cdbafa0659c13da0d27e8b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:11:27 +0100 Subject: [PATCH 04/22] Code style fixes --- pip/wheel.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pip/wheel.py b/pip/wheel.py index 7d444bd1381..1d7050aef8b 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -781,14 +781,9 @@ def build(self, session, autobuilding=False): if autobuilding: python_tag = pep425tags.implementation_tag # NOTE: Should move out a method on the cache directly. - if ephem: - output_dir = get_cache_path_for_link( - self.wheel_cache._cache_dir, req.link - ) - else: - output_dir = get_cache_path_for_link( - self.wheel_cache._ephem_cache_dir, req.link - ) + cache_root = (self.wheel_cache._ephem_cache_dir if ephem + else self.wheel_cache._cache_dir) + output_dir = get_cache_path_for_link(cache_root, req.link) try: ensure_dir(output_dir) except OSError as e: From ee67867369caf56fb3aa56aad3ec5275a95a06ce Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:26:06 +0100 Subject: [PATCH 05/22] Allow temp files for another test When there's a previous build directory, no_clean is set --- tests/functional/test_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index d5e05187adf..b310cf2d069 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -190,7 +190,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(script, data): result = script.pip( 'wheel', '--no-index', '--find-links=%s' % data.find_links, '--build', script.venv_path / 'build', - 'simple==3.0', expect_error=True, + 'simple==3.0', expect_error=True, expect_temp=True, ) # Then I see that the error code is the right one From 46c2c54014f25c0f1236b72e560d5365d1a423bb Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 17:41:51 +0100 Subject: [PATCH 06/22] Fix cleanup on 'pip install' with no args --- pip/commands/install.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 6c7dab73e5d..c228ff74fc5 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -247,17 +247,6 @@ def run(self, options, args): use_user_site=options.use_user_site, ) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) try: resolver = Resolver( preparer=preparer, @@ -274,10 +263,24 @@ def run(self, options, args): ) resolver.resolve(requirement_set) - # on -d don't do complex things like building - # wheels, and don't try to build wheels when wheel is - # not installed. - if wheel and options.cache_dir: + self.populate_requirement_set( + requirement_set, args, options, finder, session, self.name, + wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + + if (not wheel or not options.cache_dir): + # on -d don't do complex things like building + # wheels, and don't try to build wheels when wheel is + # not installed. + requirement_set.prepare_files(finder) + else: # build wheels before install. wb = WheelBuilder( requirement_set, From 9fdcfaf1b62f16aa0fbaeca20ee958fd431e6ce3 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:02:44 +0100 Subject: [PATCH 07/22] Cleanup wheel cache after 'pip freeze' --- pip/commands/freeze.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pip/commands/freeze.py b/pip/commands/freeze.py index e02fdcf8799..94500fd4439 100644 --- a/pip/commands/freeze.py +++ b/pip/commands/freeze.py @@ -88,5 +88,8 @@ def run(self, options, args): skip=skip, exclude_editable=options.exclude_editable) - for line in freeze(**freeze_kwargs): - sys.stdout.write(line + '\n') + try: + for line in freeze(**freeze_kwargs): + sys.stdout.write(line + '\n') + finally: + wheel_cache.cleanup() From 9109322ed573d76a550b005ab302c2324d9f5fa9 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:06:20 +0100 Subject: [PATCH 08/22] Expect temp files in another test --- tests/functional/test_install_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index 79fff57e293..ca3f658433c 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data): build = script.base_path / 'pip-build' script.pip( 'install', '--no-clean', '--no-index', '--build', build, - '--find-links=%s' % data.find_links, 'simple', + '--find-links=%s' % data.find_links, 'simple', expect_temp=True, ) assert exists(build) From ad242d179dfd6e70685ff69afacaea68005b6a5d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:10:32 +0100 Subject: [PATCH 09/22] Update test of install output --- tests/functional/test_install.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 033ad3dc81a..df8fb6ecd65 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -983,16 +983,15 @@ def test_install_builds_wheels(script, data): # and built wheels for upper and wheelbroken assert "Running setup.py bdist_wheel for upper" in str(res), str(res) assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # wheelbroken has to run install # into the cache assert wheels != [], str(res) # and installed from the wheel assert "Running setup.py install for upper" not in str(res), str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached. + assert "Running setup.py install for requir" not in str(res), str(res) # wheelbroken has to run install assert "Running setup.py install for wheelb" in str(res), str(res) # We want to make sure we used the correct implementation tag From a51843a92ab4fb8cb53d519fc45269b319ae010a Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:11:06 +0100 Subject: [PATCH 10/22] Expect temp files on another test --- tests/functional/test_install_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index ca3f658433c..c1ba9fc3ba0 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data): result = script.pip( 'install', '-f', data.find_links, '--no-index', 'simple', '--build', build, - expect_error=True, + expect_error=True, expect_temp=True, ) assert result.returncode == PREVIOUS_BUILD_DIR_ERROR From 73440ede2371348f91a15ecd21b731c069bdb657 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 19:12:28 +0100 Subject: [PATCH 11/22] Update another install output test --- tests/functional/test_install.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index df8fb6ecd65..9805f00e8d8 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1018,16 +1018,15 @@ def test_install_no_binary_disables_building_wheels(script, data): wheels.extend(files) # and built wheels for wheelbroken only assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res) - # But not requires_wheel... which is a local dir and thus uncachable. - assert "Running setup.py bdist_wheel for requir" not in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py bdist_wheel for requir" in str(res), str(res) # Nor upper, which was blacklisted assert "Running setup.py bdist_wheel for upper" not in str(res), str(res) # wheelbroken has to run install # into the cache assert wheels != [], str(res) - # the local tree can't build a wheel (because we can't assume that every - # build will have a suitable unique key to cache on). - assert "Running setup.py install for requires-wheel" in str(res), str(res) + # Wheels are built for local directories, but not cached + assert "Running setup.py install for requir" not in str(res), str(res) # And these two fell back to sdist based installed. assert "Running setup.py install for wheelb" in str(res), str(res) assert "Running setup.py install for upper" in str(res), str(res) From 56106365002d72930e11e2c7d9de7a2d9127cfb4 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 21:49:36 +0100 Subject: [PATCH 12/22] Update test to check for dist-info --- tests/functional/test_install_user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 6faea6149cb..c4d17b55f99 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -80,12 +80,12 @@ def test_install_curdir_usersite(self, script, virtualenv, data): expect_error=False, ) fspkg_folder = script.user_site / 'fspkg' - egg_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion + dist_info_folder = ( + script.user_site / 'FSPkg-0.1.dev0.dist-info' % pyversion ) assert fspkg_folder in result.files_created, result.stdout - assert egg_info_folder in result.files_created + assert dist_info_folder in result.files_created def test_install_user_venv_nositepkgs_fails(self, script, data): """ From 1c03c1b092839494761676b4d64c92afd84ed74c Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 22:21:29 +0100 Subject: [PATCH 13/22] Remove string formatting operation --- tests/functional/test_install_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index c4d17b55f99..2d3320f71f3 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -81,7 +81,7 @@ def test_install_curdir_usersite(self, script, virtualenv, data): ) fspkg_folder = script.user_site / 'fspkg' dist_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0.dist-info' % pyversion + script.user_site / 'FSPkg-0.1.dev0.dist-info' ) assert fspkg_folder in result.files_created, result.stdout From ad50ac98ba0a6abbaef41a30d057b0c2286ee73c Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 20 May 2017 09:43:51 +0100 Subject: [PATCH 14/22] Install 'wheel' for test that now requires it --- tests/functional/test_install_user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 2d3320f71f3..0c8edfd7241 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -68,11 +68,13 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed('INITools', use_user_site=True) + @pytest.mark.network def test_install_curdir_usersite(self, script, virtualenv, data): """ Test installing current directory ('.') into usersite """ virtualenv.system_site_packages = True + script.pip("install", "wheel") run_from = data.packages.join("FSPkg") result = script.pip( 'install', '-vvv', '--user', curdir, From d045cc264c4aa2c5e41c9fb482497914ba0250ee Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 22 Jun 2017 17:34:05 -0400 Subject: [PATCH 15/22] Fix some changes that I screwed up in the rebase --- pip/cache.py | 88 +++++++++++++++++++++++++---------------- pip/commands/install.py | 15 ++++--- 2 files changed, 62 insertions(+), 41 deletions(-) diff --git a/pip/cache.py b/pip/cache.py index 87fd0e1b595..7413011420e 100644 --- a/pip/cache.py +++ b/pip/cache.py @@ -4,12 +4,14 @@ import errno import logging import os +import tempfile from pip._vendor.packaging.utils import canonicalize_name import pip.index from pip.compat import expanduser from pip.download import path_to_url +from pip.utils import rmtree from pip.utils.cache import get_cache_path_for_link from pip.wheel import InvalidWheelFilename, Wheel @@ -27,45 +29,61 @@ def __init__(self, cache_dir, format_control): binaries being read from the cache. """ self._cache_dir = expanduser(cache_dir) if cache_dir else None + # Ephemeral cache: store wheels just for this run + self._ephem_cache_dir = tempfile.mkdtemp(suffix='-pip-ephem-cache') self._format_control = format_control def cached_wheel(self, link, package_name): - not_cached = ( - not self._cache_dir or - not link or - link.is_wheel or - not link.is_artifact or - not package_name - ) + orig_link = link + link = cached_wheel( + self._cache_dir, link, self._format_control, package_name) + if link is orig_link: + link = cached_wheel( + self._ephem_cache_dir, link, self._format_control, + package_name) + return link - if not_cached: - return link + def cleanup(self): + rmtree(self._ephem_cache_dir) + + +def cached_wheel(cache_dir, link, format_control, package_name): + not_cached = ( + not cache_dir or + not link or + link.is_wheel or + not link.is_artifact or + not package_name + ) - canonical_name = canonicalize_name(package_name) - formats = pip.index.fmt_ctl_formats( - self._format_control, canonical_name - ) - if "binary" not in formats: + if not_cached: + return link + + canonical_name = canonicalize_name(package_name) + formats = pip.index.fmt_ctl_formats( + format_control, canonical_name + ) + if "binary" not in formats: + return link + root = get_cache_path_for_link(cache_dir, link) + try: + wheel_names = os.listdir(root) + except OSError as err: + if err.errno in {errno.ENOENT, errno.ENOTDIR}: return link - root = get_cache_path_for_link(self._cache_dir, link) + raise + candidates = [] + for wheel_name in wheel_names: try: - wheel_names = os.listdir(root) - except OSError as err: - if err.errno in {errno.ENOENT, errno.ENOTDIR}: - return link - raise - candidates = [] - for wheel_name in wheel_names: - try: - wheel = Wheel(wheel_name) - except InvalidWheelFilename: - continue - if not wheel.supported(): - # Built for a different python/arch/etc - continue - candidates.append((wheel.support_index_min(), wheel_name)) - if not candidates: - return link - candidates.sort() - path = os.path.join(root, candidates[0][1]) - return pip.index.Link(path_to_url(path)) + wheel = Wheel(wheel_name) + except InvalidWheelFilename: + continue + if not wheel.supported(): + # Built for a different python/arch/etc + continue + candidates.append((wheel.support_index_min(), wheel_name)) + if not candidates: + return link + candidates.sort() + path = os.path.join(root, candidates[0][1]) + return pip.index.Link(path_to_url(path)) diff --git a/pip/commands/install.py b/pip/commands/install.py index c228ff74fc5..032b76d1da0 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -248,6 +248,11 @@ def run(self, options, args): ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + self.name, wheel_cache + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -275,12 +280,10 @@ def run(self, options, args): progress_bar=options.progress_bar, ) - if (not wheel or not options.cache_dir): - # on -d don't do complex things like building - # wheels, and don't try to build wheels when wheel is - # not installed. - requirement_set.prepare_files(finder) - else: + # on -d don't do complex things like building + # wheels, and don't try to build wheels when wheel is + # not installed. + if wheel and options.cache_dir: # build wheels before install. wb = WheelBuilder( requirement_set, From 01d226b3794ac0bc29e3595df91ebd8dee8d5e5c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 26 Jun 2017 07:19:16 -0400 Subject: [PATCH 16/22] Switch to using the existing tempdir abstraction --- pip/cache.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pip/cache.py b/pip/cache.py index 7413011420e..b3206000aae 100644 --- a/pip/cache.py +++ b/pip/cache.py @@ -4,14 +4,13 @@ import errno import logging import os -import tempfile from pip._vendor.packaging.utils import canonicalize_name import pip.index from pip.compat import expanduser from pip.download import path_to_url -from pip.utils import rmtree +from pip.utils import temp_dir from pip.utils.cache import get_cache_path_for_link from pip.wheel import InvalidWheelFilename, Wheel @@ -30,7 +29,8 @@ def __init__(self, cache_dir, format_control): """ self._cache_dir = expanduser(cache_dir) if cache_dir else None # Ephemeral cache: store wheels just for this run - self._ephem_cache_dir = tempfile.mkdtemp(suffix='-pip-ephem-cache') + self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache") + self._ephem_cache_dir.create() self._format_control = format_control def cached_wheel(self, link, package_name): @@ -39,12 +39,12 @@ def cached_wheel(self, link, package_name): self._cache_dir, link, self._format_control, package_name) if link is orig_link: link = cached_wheel( - self._ephem_cache_dir, link, self._format_control, + self._ephem_cache_dir.path, link, self._format_control, package_name) return link def cleanup(self): - rmtree(self._ephem_cache_dir) + self._ephem_cache_dir.cleanup() def cached_wheel(cache_dir, link, format_control, package_name): From d5e3082f84ddc0a3bcf68e51ad1205cd4df4ee2f Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 26 Jun 2017 07:33:11 -0400 Subject: [PATCH 17/22] flake8 --- pip/commands/install.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 032b76d1da0..4154d688231 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -248,11 +248,6 @@ def run(self, options, args): ) try: - self.populate_requirement_set( - requirement_set, args, options, finder, session, - self.name, wheel_cache - ) - resolver = Resolver( preparer=preparer, finder=finder, From 1997456855079b7102599552bc774855803f54bc Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:08:37 -0400 Subject: [PATCH 18/22] fix --- pip/commands/install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 4154d688231..a33774f298a 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -248,6 +248,18 @@ def run(self, options, args): ) try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, self.name, + wheel_cache + ) + preparer = RequirementPreparer( + build_dir=directory.path, + src_dir=options.src_dir, + download_dir=None, + wheel_download_dir=None, + progress_bar=options.progress_bar, + ) + resolver = Resolver( preparer=preparer, finder=finder, @@ -263,18 +275,6 @@ def run(self, options, args): ) resolver.resolve(requirement_set) - self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - ) - # on -d don't do complex things like building # wheels, and don't try to build wheels when wheel is # not installed. From 328f607bf0f1a6013d83854bf84e3df95c8455fd Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:12:38 -0400 Subject: [PATCH 19/22] not a thing anymore --- pip/req/req_set.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pip/req/req_set.py b/pip/req/req_set.py index 5146211a27d..403bb0f2301 100644 --- a/pip/req/req_set.py +++ b/pip/req/req_set.py @@ -44,9 +44,6 @@ def __init__(self, require_hashes=False, target_dir=None, use_user_site=False, pycompile=True): """Create a RequirementSet. - - :param wheel_cache: The pip wheel cache, for passing to - InstallRequirement. """ self.requirements = Requirements() @@ -198,9 +195,6 @@ def cleanup_files(self): for req in self.reqs_to_cleanup: req.remove_temporary_source() - if self._wheel_cache: - self._wheel_cache.cleanup() - def _to_install(self): """Create the installation order. From 8ef8400a9b7f2a31af7a7e22519fec6e1c761ded Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:15:21 -0400 Subject: [PATCH 20/22] these are probably needed --- pip/commands/install.py | 1 + pip/commands/wheel.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pip/commands/install.py b/pip/commands/install.py index a33774f298a..328ccf00643 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -351,6 +351,7 @@ def run(self, options, args): # Clean up if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index 155256d82cb..644451bf653 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -196,3 +196,4 @@ def run(self, options, args): finally: if not options.no_clean: requirement_set.cleanup_files() + wheel_cache.cleanup() From cb56bb36552fe75aa5b0a48dfd7f4148cf99d885 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:16:06 -0400 Subject: [PATCH 21/22] flake8 --- pip/commands/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 328ccf00643..b6713b2cb8a 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -249,8 +249,8 @@ def run(self, options, args): try: self.populate_requirement_set( - requirement_set, args, options, finder, session, self.name, - wheel_cache + requirement_set, args, options, finder, session, + self.name, wheel_cache ) preparer = RequirementPreparer( build_dir=directory.path, From 5f88252213be8f572b931df983e3913168553af8 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 5 Jul 2017 20:16:51 -0400 Subject: [PATCH 22/22] fix --- pip/wheel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pip/wheel.py b/pip/wheel.py index 1d7050aef8b..499ff6c33c6 100644 --- a/pip/wheel.py +++ b/pip/wheel.py @@ -781,8 +781,10 @@ def build(self, session, autobuilding=False): if autobuilding: python_tag = pep425tags.implementation_tag # NOTE: Should move out a method on the cache directly. - cache_root = (self.wheel_cache._ephem_cache_dir if ephem - else self.wheel_cache._cache_dir) + cache_root = ( + self.wheel_cache._ephem_cache_dir.path if ephem + else self.wheel_cache._cache_dir + ) output_dir = get_cache_path_for_link(cache_root, req.link) try: ensure_dir(output_dir)