From 4e6d935eafab56695f74f89bde04aae885441b7a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 13 Mar 2015 15:03:01 -0700 Subject: [PATCH] Speed up unpack_file_url E.g.: `pip install /path/to/dir` by building an sdist and then unpacking it instead of doing `shutil.copytree`. `shutil.copytree` may copy files that aren't included in the sdist, which means that: 1. If those files are large, a `pip install` could take much longer than it should. 2. Folks that are using `python setup.py install` to test their package are not fully testing it, because it may copy over files that wouldn't be there if they built and sdist and installed that. So this method building an sdist is both more accurate and faster. Fixes https://github.com/pypa/pip/issues/2195 --- pip/download.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/pip/download.py b/pip/download.py index 0490150f4d9..bb12a497c38 100644 --- a/pip/download.py +++ b/pip/download.py @@ -22,8 +22,10 @@ from pip.exceptions import InstallationError, HashMismatch from pip.models import PyPI from pip.utils import (splitext, rmtree, format_size, display_path, - backup_dir, ask_path_exists, unpack_file) + backup_dir, ask_path_exists, unpack_file, + call_subprocess) from pip.utils.filesystem import check_path_owner +from pip.utils.logging import indent_log from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner from pip.locations import write_delete_marker_file from pip.vcs import vcs @@ -725,6 +727,45 @@ def unpack_file_url(link, location, download_dir=None): _copy_file(from_path, download_dir, content_type, link) +def _copy_dist_from_dir(link_path, location): + """Copy distribution files in `link_path` to `location`. + + Invoked when user requests to install a local directory. E.g.: + + pip install . + pip install ~/dev/git-repos/python-prompt-toolkit + + """ + + # Note: This is currently VERY SLOW if you have a lot of data in the + # directory, because it copies everything with `shutil.copytree`. + # What it should really do is build an sdist and install that. + # See https://github.com/pypa/pip/issues/2195 + + if os.path.isdir(location): + rmtree(location) + + # build an sdist + setup_py = 'setup.py' + sdist_args = [sys.executable] + sdist_args.append('-c') + sdist_args.append( + "import setuptools, tokenize;__file__=%r;" + "exec(compile(getattr(tokenize, 'open', open)(__file__).read()" + ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % setup_py) + sdist_args.append('sdist') + sdist_args += ['--dist-dir', location] + logger.info('Running setup.py sdist for %s', link_path) + + with indent_log(): + call_subprocess(sdist_args, cwd=link_path, show_stdout=False) + + # unpack sdist into `location` + sdist = os.path.join(location, os.listdir(location)[0]) + logger.info('Unpacking sdist %s into %s', sdist, location) + unpack_file(sdist, location, content_type=None, link=None) + + class PipXmlrpcTransport(xmlrpc_client.Transport): """Provide a `xmlrpclib.Transport` implementation via a `PipSession` object.