Ensure builtin modules are from typeshed sooner

It should work now with custom-typeshed-dir.

Fixes python#1876

build-requirements.txt

@@ -1,2 +1,3 @@
 -r mypy-requirements.txt
+types-setuptools
 types-typed-ast>=1.5.0,<1.6.0

mypy-requirements.txt

@@ -1,4 +1,5 @@
 typing_extensions>=3.10
 mypy_extensions>=0.4.3
+importlib_resources; python_version<'3.7'
 typed_ast>=1.4.0,<2; python_version<'3.8'
 tomli>=1.1.0; python_version<'3.11'

mypy/build.py

@@ -605,6 +605,31 @@ def __init__(self, data_dir: str,
  self.fscache = fscache
  self.find_module_cache = FindModuleCache(self.search_paths, self.fscache, self.options,
  source_set=self.source_set)
+ for module in CORE_BUILTIN_MODULES:
+ if options.use_builtins_fixtures:
+ continue
+ if module == "_importlib_modulespec":
+ continue
+ path = self.find_module_cache.find_module(module)
+ if not isinstance(path, str):
+ raise CompileError([
+ f"Failed to find builtin module {module}, perhaps typeshed is broken?",
+ ])
+ if is_typeshed_file(path):
+ continue
+ if is_stub_package_file(path):
+ continue
+ if options.custom_typeshed_dir is not None:
+ # Check if module lives under custom_typeshed_dir subtree
+ custom_typeshed_dir = os.path.abspath(options.custom_typeshed_dir)
+ if os.path.commonpath((path, custom_typeshed_dir)) == custom_typeshed_dir:
+ continue
+
+ raise CompileError([
+ f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"',
+ f'note: A user-defined top-level module with name "{module}" is not supported'
+ ])
+
  self.metastore = create_metastore(options)
 
  # a mapping from source files to their corresponding shadow files
@@ -2459,15 +2484,6 @@ def find_module_and_diagnose(manager: BuildManager,
  if is_sub_path(result, dir):
  # Silence errors in site-package dirs and typeshed
  follow_imports = 'silent'
- if (id in CORE_BUILTIN_MODULES
- and not is_typeshed_file(result)
- and not is_stub_package_file(result)
- and not options.use_builtins_fixtures
- and not options.custom_typeshed_dir):
- raise CompileError([
- f'mypy: "{os.path.relpath(result)}" shadows library module "{id}"',
- f'note: A user-defined top-level module with name "{id}" is not supported'
- ])
  return (result, follow_imports)
  else:
  # Could not find a module. Typically the reason is a

mypy/test/testgraph.py

@@ -38,6 +38,7 @@ def test_scc(self) -> None:
  def _make_manager(self) -> BuildManager:
  errors = Errors()
  options = Options()
+ options.use_builtins_fixtures = True
  fscache = FileSystemCache()
  search_paths = SearchPaths((), (), (), ())
  manager = BuildManager(

mypy/util.py

@@ -9,6 +9,10 @@
 import io
 import shutil
 import time
+try:
+ from importlib import resources as importlib_resources # type: ignore[attr-defined]
+except ImportError: # <python3.7
+ import importlib_resources # type: ignore
 
 from typing import (
  TypeVar, List, Tuple, Optional, Dict, Sequence, Iterable, Container, IO, Callable, Union, Sized
@@ -22,6 +26,12 @@
 except ImportError:
  CURSES_ENABLED = False
 
+with importlib_resources.path(
+ "mypy", # mypy-c doesn't support __package__
+ "py.typed", # a marker file for type information, we assume typeshed to live in the same dir
+) as _resource:
+ TYPESHED_DIR: Final = str(_resource.parent / "typeshed")
+
 T = TypeVar('T')
 
 ENCODING_RE: Final = re.compile(br"([ \t\v]*#.*(\r\n?|\n))??[ \t\v]*#.*coding[:=][ \t]*([-\w.]+)")
@@ -745,16 +755,18 @@ def format_error(
 
 
 def is_typeshed_file(file: str) -> bool:
- # gross, but no other clear way to tell
- return 'typeshed' in os.path.abspath(file).split(os.sep)
+ try:
+ return os.path.commonpath((TYPESHED_DIR, os.path.abspath(file))) == TYPESHED_DIR
+ except ValueError: # Different drives on Windows
+ return False
 
 
 def is_stub_package_file(file: str) -> bool:
  # Use hacky heuristics to check whether file is part of a PEP 561 stub package.
  if not file.endswith('.pyi'):
  return False
  return any(component.endswith('-stubs')
- for component in os.path.abspath(file).split(os.sep))
+ for component in os.path.split(os.path.abspath(file)))
 
 
 def unnamed_function(name: Optional[str]) -> bool:

mypyc/build.py

@@ -47,7 +47,7 @@
 
 try:
  # Import setuptools so that it monkey-patch overrides distutils
- import setuptools # type: ignore # noqa
+ import setuptools # noqa
 except ImportError:
  if sys.version_info >= (3, 12):
  # Raise on Python 3.12, since distutils will go away forever
@@ -63,7 +63,7 @@ def get_extension() -> Type['Extension']:
  if not use_setuptools:
  from distutils.core import Extension
  else:
- from setuptools import Extension # type: ignore # noqa
+ from setuptools import Extension # noqa
 
  return Extension
 

test-data/unit/cmdline.test

@@ -1425,3 +1425,26 @@ b\.c \d+
 # cmd: mypy --enable-incomplete-features a.py
 [file a.py]
 pass
+
+[case testShadowTypingModuleEarlyLoad]
+# cmd: mypy dir
+[file dir/__init__.py]
+from typing import Union
+
+def foo(a: Union[int, str]) -> str:
+ return str
+[file typing.py]
+import sys
+import os
+del sys.modules["typing"]
+path = sys.path
+try:
+ sys.path.remove(os.getcwd())
+except ValueError:
+ sys.path.remove(".")
+from typing import *
+sys.path = path
+[out]
+mypy: "typing.py" shadows library module "typing"
+note: A user-defined top-level module with name "typing" is not supported
+== Return code: 2