improve test coverage and documentation

docs/source/running_mypy.rst

@@ -516,6 +516,32 @@ same directory on the search path, only the stub file is used.
 (However, if the files are in different directories, the one found
 in the earlier directory is used.)
 
+If a namespace package is spread across many distinct folders, for
+instance::
+
+ foo/
+ company/
+ foo/
+ a.py
+ bar/
+ company/
+ bar/
+ b.py
+ baz/
+ company/
+ baz/
+ c.py
+ ...
+
+Then the default logic used to scan through search paths to resolve
+imports can become very slow. Specifically it becomes quadratic in
+the number of folders sharing the top-level ``company`` namespace.
+To work around this, it is possible to enable an experimental fast path
+that can more efficiently resolve imports within the set of input files
+to be typechecked. This is controlled by setting the :option:`--fast-module-lookup`
+option.
+
+
 Other advice and best practices
 *******************************
 

mypy/modulefinder.py

@@ -189,6 +189,9 @@ def clear(self) -> None:
  self.ns_ancestors.clear()
 
  def find_module_via_source_set(self, id: str) -> Optional[ModuleSearchResult]:
+ """Fast path to find modules by looking through the input sources
+
+ This is only used when --fast-module-lookup is passed on the command line."""
  if not self.source_set:
  return None
 

mypy/test/helpers.py

@@ -365,7 +365,8 @@ def assert_type(typ: type, value: object) -> None:
 
 
 def parse_options(program_text: str, testcase: DataDrivenTestCase,
- incremental_step: int) -> Options:
+ incremental_step: int,
+ extra_flags: List[str] = []) -> Options:
  """Parse comments like '# flags: --foo' in a test case."""
  options = Options()
  flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
@@ -378,12 +379,13 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase,
  if flags:
  flag_list = flags.group(1).split()
  flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python
+ flag_list.extend(extra_flags)
  targets, options = process_options(flag_list, require_targets=False)
  if targets:
  # TODO: support specifying targets via the flags pragma
  raise RuntimeError('Specifying targets via the flags pragma is not supported.')
  else:
- flag_list = []
+ flag_list = extra_flags
  options = Options()
  # TODO: Enable strict optional in test cases by default (requires *many* test case changes)
  options.strict_optional = False

mypy/test/testcheck.py

@@ -40,6 +40,7 @@
  'check-multiple-inheritance.test',
  'check-super.test',
  'check-modules.test',
+ 'check-modules-fast.test',
  'check-typevar-values.test',
  'check-unsupported.test',
  'check-unreachable-code.test',
@@ -111,6 +112,13 @@
 if sys.platform in ('darwin', 'win32'):
  typecheck_files.extend(['check-modules-case.test'])
 
+# some test cases are run multiple times with various combinations of extra flags
+EXTRA_FLAGS = {
+ 'check-modules.test': [['--fast-module-lookup']],
+ 'check-modules-fast.test': [['--fast-module-lookup']],
+ 'check-modules-case.test': [['--fast-module-lookup']],
+}
+
 
 class TypeCheckSuite(DataSuite):
  files = typecheck_files
@@ -138,10 +146,13 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
  self.run_case_once(testcase, ops, step)
  else:
  self.run_case_once(testcase)
+ for extra_flags in EXTRA_FLAGS.get(os.path.basename(testcase.file), []):
+ self.run_case_once(testcase, extra_flags=extra_flags)
 
  def run_case_once(self, testcase: DataDrivenTestCase,
  operations: List[FileOperation] = [],
- incremental_step: int = 0) -> None:
+ incremental_step: int = 0,
+ extra_flags: List[str] = []) -> None:
  original_program_text = '\n'.join(testcase.input)
  module_data = self.parse_module(original_program_text, incremental_step)
 
@@ -162,7 +173,8 @@ def run_case_once(self, testcase: DataDrivenTestCase,
  perform_file_operations(operations)
 
  # Parse options after moving files (in case mypy.ini is being moved).
- options = parse_options(original_program_text, testcase, incremental_step)
+ options = parse_options(original_program_text, testcase, incremental_step,
+ extra_flags=extra_flags)
  options.use_builtins_fixtures = True
  options.show_traceback = True
 

test-data/unit/check-modules-fast.test

@@ -0,0 +1,126 @@
+-- Type checker test cases dealing with module lookup edge cases
+-- to ensure that --fast-module-lookup matches regular lookup behavior
+
+[case testModuleLookup]
+import m
+reveal_type(m.a) # N: Revealed type is "m.A"
+
+[file m.py]
+class A: pass
+a = A()
+
+[case testModuleLookupStub]
+import m
+reveal_type(m.a) # N: Revealed type is "m.A"
+
+[file m.pyi]
+class A: pass
+a = A()
+
+[case testModuleLookupFromImport]
+from m import a
+reveal_type(a) # N: Revealed type is "m.A"
+
+[file m.py]
+class A: pass
+a = A()
+
+[case testModuleLookupStubFromImport]
+from m import a
+reveal_type(a) # N: Revealed type is "m.A"
+
+[file m.pyi]
+class A: pass
+a = A()
+
+
+[case testModuleLookupWeird]
+from m import a
+reveal_type(a) # N: Revealed type is "builtins.object"
+reveal_type(a.b) # N: Revealed type is "m.a.B"
+
+[file m.py]
+class A: pass
+a = A()
+
+[file m/__init__.py]
+[file m/a.py]
+class B: pass
+b = B()
+
+
+[case testModuleLookupWeird2]
+from m.a import b
+reveal_type(b) # N: Revealed type is "m.a.B"
+
+[file m.py]
+class A: pass
+a = A()
+
+[file m/__init__.py]
+[file m/a.py]
+class B: pass
+b = B()
+
+
+[case testModuleLookupWeird3]
+from m.a import b
+reveal_type(b) # N: Revealed type is "m.a.B"
+
+[file m.py]
+class A: pass
+a = A()
+[file m/__init__.py]
+class B: pass
+a = B()
+[file m/a.py]
+class B: pass
+b = B()
+
+
+[case testModuleLookupWeird4]
+import m.a
+m.a.b # E: "str" has no attribute "b"
+
+[file m.py]
+class A: pass
+a = A()
+[file m/__init__.py]
+class B: pass
+a = 'foo'
+b = B()
+[file m/a.py]
+class C: pass
+b = C()
+
+
+[case testModuleLookupWeird5]
+import m.a as ma
+reveal_type(ma.b) # N: Revealed type is "m.a.C"
+
+[file m.py]
+class A: pass
+a = A()
+[file m/__init__.py]
+class B: pass
+a = 'foo'
+b = B()
+[file m/a.py]
+class C: pass
+b = C()
+
+
+[case testModuleLookupWeird6]
+from m.a import b
+reveal_type(b) # N: Revealed type is "m.a.C"
+
+[file m.py]
+class A: pass
+a = A()
+[file m/__init__.py]
+class B: pass
+a = 'foo'
+b = B()
+[file m/a.py]
+class C: pass
+b = C()