-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- check build requirements for conflicts - better isolation (ignore system site packages) - support 2 prefixes: a "normal" one, and an "overlay" one (with higher priority over "normal")
- Loading branch information
1 parent
b47b2fa
commit 744b8cf
Showing
13 changed files
with
359 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,7 @@ | |
import os | ||
import sys | ||
import textwrap | ||
from collections import OrderedDict | ||
from distutils.sysconfig import get_python_lib | ||
from sysconfig import get_paths | ||
|
||
|
@@ -18,6 +19,25 @@ | |
logger = logging.getLogger(__name__) | ||
|
||
|
||
class _Prefix: | ||
|
||
def __init__(self, path): | ||
self.path = path | ||
self.setup = False | ||
self.bin_dir = get_paths( | ||
'nt' if os.name == 'nt' else 'posix_prefix', | ||
vars={'base': path, 'platbase': path} | ||
)['scripts'] | ||
# Note: prefer distutils' sysconfig to get the | ||
# library paths so PyPy is correctly supported. | ||
purelib = get_python_lib(plat_specific=0, prefix=path) | ||
platlib = get_python_lib(plat_specific=1, prefix=path) | ||
if purelib == platlib: | ||
self.lib_dirs = [purelib] | ||
else: | ||
self.lib_dirs = [purelib, platlib] | ||
|
||
|
||
class BuildEnvironment(object): | ||
"""Creates and manages an isolated environment to install build deps | ||
""" | ||
|
@@ -26,86 +46,113 @@ def __init__(self): | |
self._temp_dir = TempDirectory(kind="build-env") | ||
self._temp_dir.create() | ||
|
||
@property | ||
def path(self): | ||
return self._temp_dir.path | ||
|
||
def __enter__(self): | ||
self.save_path = os.environ.get('PATH', None) | ||
self.save_pythonpath = os.environ.get('PYTHONPATH', None) | ||
self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) | ||
|
||
install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' | ||
install_dirs = get_paths(install_scheme, vars={ | ||
'base': self.path, | ||
'platbase': self.path, | ||
}) | ||
|
||
scripts = install_dirs['scripts'] | ||
if self.save_path: | ||
os.environ['PATH'] = scripts + os.pathsep + self.save_path | ||
else: | ||
os.environ['PATH'] = scripts + os.pathsep + os.defpath | ||
|
||
# Note: prefer distutils' sysconfig to get the | ||
# library paths so PyPy is correctly supported. | ||
purelib = get_python_lib(plat_specific=0, prefix=self.path) | ||
platlib = get_python_lib(plat_specific=1, prefix=self.path) | ||
if purelib == platlib: | ||
lib_dirs = purelib | ||
else: | ||
lib_dirs = purelib + os.pathsep + platlib | ||
if self.save_pythonpath: | ||
os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ | ||
self.save_pythonpath | ||
else: | ||
os.environ['PYTHONPATH'] = lib_dirs | ||
|
||
os.environ['PYTHONNOUSERSITE'] = '1' | ||
|
||
# Ensure .pth files are honored. | ||
with open(os.path.join(purelib, 'sitecustomize.py'), 'w') as fp: | ||
self._prefixes = OrderedDict(( | ||
(name, _Prefix(os.path.join(self._temp_dir.path, name))) | ||
for name in ('normal', 'overlay') | ||
)) | ||
|
||
self._bin_dirs = [] | ||
self._lib_dirs = [] | ||
for prefix in reversed(list(self._prefixes.values())): | ||
self._bin_dirs.append(prefix.bin_dir) | ||
self._lib_dirs.extend(prefix.lib_dirs) | ||
|
||
# Customize site to: | ||
# - ensure .pth files are honored | ||
# - prevent access to system site packages | ||
system_sites = { | ||
os.path.normcase(site) for site in ( | ||
get_python_lib(plat_specific=0), | ||
get_python_lib(plat_specific=1), | ||
) | ||
} | ||
self._site_dir = os.path.join(self._temp_dir.path, 'site') | ||
if not os.path.exists(self._site_dir): | ||
os.mkdir(self._site_dir) | ||
with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: | ||
fp.write(textwrap.dedent( | ||
''' | ||
import site | ||
site.addsitedir({!r}) | ||
import os, site, sys | ||
This comment has been minimized.
Sorry, something went wrong.
This comment has been minimized.
Sorry, something went wrong.
pfmoore
Member
|
||
# First, drop system-sites related paths. | ||
original_sys_path = sys.path[:] | ||
known_paths = set() | ||
for path in {system_sites!r}: | ||
site.addsitedir(path, known_paths=known_paths) | ||
system_paths = set( | ||
os.path.normcase(path) | ||
for path in sys.path[len(original_sys_path):] | ||
) | ||
original_sys_path = [ | ||
path for path in original_sys_path | ||
if os.path.normcase(path) not in system_paths | ||
] | ||
sys.path = original_sys_path | ||
# Second, add lib directories. | ||
# ensuring .pth file are processed. | ||
for path in {lib_dirs!r}: | ||
assert not path in sys.path | ||
site.addsitedir(path) | ||
''' | ||
).format(purelib)) | ||
).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) | ||
|
||
return self.path | ||
def __enter__(self): | ||
self._save_env = { | ||
name: os.environ.get(name, None) | ||
for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') | ||
} | ||
|
||
path = self._bin_dirs[:] | ||
old_path = self._save_env['PATH'] | ||
if old_path: | ||
path.extend(old_path.split(os.pathsep)) | ||
|
||
pythonpath = [self._site_dir] | ||
|
||
os.environ.update({ | ||
'PATH': os.pathsep.join(path), | ||
'PYTHONNOUSERSITE': '1', | ||
'PYTHONPATH': os.pathsep.join(pythonpath), | ||
}) | ||
|
||
def __exit__(self, exc_type, exc_val, exc_tb): | ||
def restore_var(varname, old_value): | ||
for varname, old_value in self._save_env.items(): | ||
if old_value is None: | ||
os.environ.pop(varname, None) | ||
else: | ||
os.environ[varname] = old_value | ||
|
||
restore_var('PATH', self.save_path) | ||
restore_var('PYTHONPATH', self.save_pythonpath) | ||
restore_var('PYTHONNOUSERSITE', self.save_nousersite) | ||
|
||
def cleanup(self): | ||
self._temp_dir.cleanup() | ||
|
||
def missing_requirements(self, reqs): | ||
"""Return a list of the requirements from reqs that are not present | ||
def check_requirements(self, reqs): | ||
"""Return 2 sets: | ||
- conflicting requirements: set of (installed, wanted) reqs tuples | ||
- missing requirements: set of reqs | ||
""" | ||
missing = [] | ||
with self: | ||
ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) | ||
missing = set() | ||
conflicting = set() | ||
if reqs: | ||
ws = WorkingSet(self._lib_dirs) | ||
for req in reqs: | ||
try: | ||
if ws.find(Requirement.parse(req)) is None: | ||
missing.append(req) | ||
except VersionConflict: | ||
missing.append(req) | ||
return missing | ||
|
||
def install_requirements(self, finder, requirements, message): | ||
missing.add(req) | ||
except VersionConflict as e: | ||
conflicting.add((str(e.args[0].as_requirement()), | ||
str(e.args[1]))) | ||
return conflicting, missing | ||
|
||
def install_requirements(self, finder, requirements, prefix, message): | ||
prefix = self._prefixes[prefix] | ||
assert not prefix.setup | ||
prefix.setup = True | ||
if not requirements: | ||
return | ||
args = [ | ||
sys.executable, os.path.dirname(pip_location), 'install', | ||
'--ignore-installed', '--no-user', '--prefix', self.path, | ||
'--ignore-installed', '--no-user', '--prefix', prefix.path, | ||
'--no-warn-script-location', | ||
] | ||
if logger.getEffectiveLevel() <= logging.DEBUG: | ||
|
@@ -150,5 +197,5 @@ def __exit__(self, exc_type, exc_val, exc_tb): | |
def cleanup(self): | ||
pass | ||
|
||
def install_requirements(self, finder, requirements, message): | ||
def install_requirements(self, finder, requirements, prefix, message): | ||
raise NotImplementedError() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
include pyproject.toml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
#dummy |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
[build-system] | ||
requires = ["setuptools==1.0", "wheel"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/usr/bin/env python | ||
from setuptools import setup | ||
|
||
setup( | ||
name='pep518_conflicting_requires', | ||
version='1.0.0', | ||
py_modules=['pep518'], | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
I don't think this code should run as
sitecustomize.py
... but instead should be a pth file so that the sys.path is extended before another pth file has a chance to perform imports that initialize state that might depend on these operations (such as pkg_resources working set build) - see #7778 for details