Refactor to allow a safer source=None

Author: bzoracler

mypy/build.py

@@ -163,7 +163,7 @@
 from mypy.modules_state import modules_state
 from mypy.nodes import Expression
 from mypy.options import Options
-from mypy.parse import load_from_raw, parse
+from mypy.parse import load_from_raw, parse, parse_native
 from mypy.plugin import ChainedPlugin, Plugin, ReportConfigContext
 from mypy.plugins.default import DefaultPlugin
 from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor
@@ -1114,7 +1114,9 @@ def parse_files_threaded_raw(self, states: list[State]) -> tuple[list[State], se
                     ignore_errors = state.ignore_all or state.options.ignore_errors
                     if ignore_errors:
                         self.errors.ignored_files.add(state.xpath)
-                    futures.append(executor.submit(state.parse_file_inner, state.source))
+                    futures.append(
+                        executor.submit(state.parse_file_inner, state.source, parallel=True)
+                    )
                     parallel_parsed_states.append(state)
                     parallel_parsed_states_set.add(state)
                 else:
@@ -1264,9 +1266,10 @@ def parse_file(
         self,
         id: str,
         path: str,
-        source: str,
+        source: str | None,
         options: Options,
         raw_data: FileRawData | None = None,
+        parallel: bool = False,
     ) -> MypyFile:
         """Parse the source of a file with the given name.
 
@@ -1277,7 +1280,11 @@ def parse_file(
             # If possible, deserialize from known binary data instead of parsing from scratch.
             tree = load_from_raw(path, id, raw_data, self.errors, options)
         else:
-            tree = parse(source, path, id, self.errors, options=options)
+            if source is not None:
+                tree = parse(source, path, id, self.errors, options=options)
+            else:
+                assert parallel
+                tree = parse_native(source, path, id, self.errors, options=options)
         tree._fullname = id
         if self.stats_enabled:
             with self.stats_lock:
@@ -3176,10 +3183,12 @@ def get_source(self) -> str:
         self.time_spent_us += time_spent_us(t0)
         return source
 
-    def parse_file_inner(self, source: str, raw_data: FileRawData | None = None) -> None:
+    def parse_file_inner(
+        self, source: str | None, raw_data: FileRawData | None = None, parallel: bool = False
+    ) -> None:
         t0 = time_ref()
         self.tree = self.manager.parse_file(
-            self.id, self.xpath, source, options=self.options, raw_data=raw_data
+            self.id, self.xpath, source, self.options, raw_data, parallel
         )
         self.time_spent_us += time_spent_us(t0)
 

mypy/parse.py

@@ -28,21 +28,7 @@ def parse(
     the parse errors, use eager=True.
     """
     if options.native_parser:
-        import mypy.nativeparse
-
-        ignore_errors = options.ignore_errors or fnam in errors.ignored_files
-        # If errors are ignored, we can drop many function bodies to speed up type checking.
-        strip_function_bodies = ignore_errors and not options.preserve_asts
-        tree, _, _ = mypy.nativeparse.native_parse(
-            fnam, options, source, skip_function_bodies=strip_function_bodies
-        )
-        # Set is_stub based on file extension
-        tree.is_stub = fnam.endswith(".pyi")
-        # Note: tree.imports is populated directly by load_from_raw() with deserialized
-        # import metadata, so we don't need to collect imports via AST traversal
-        if eager and tree.raw_data is not None:
-            tree = load_from_raw(fnam, module, tree.raw_data, errors, options)
-        return tree
+        return parse_native(source, fnam, module, errors, options, eager)
 
     if options.transform_source is not None:
         source = options.transform_source(source)
@@ -97,6 +83,31 @@ def load_from_raw(
     return tree
 
 
+def parse_native(
+    source: str | bytes | None,
+    fnam: str,
+    module: str | None,
+    errors: Errors,
+    options: Options,
+    eager: bool = False,
+) -> MypyFile:
+    import mypy.nativeparse
+
+    ignore_errors = options.ignore_errors or fnam in errors.ignored_files
+    # If errors are ignored, we can drop many function bodies to speed up type checking.
+    strip_function_bodies = ignore_errors and not options.preserve_asts
+    tree, _, _ = mypy.nativeparse.native_parse(
+        fnam, options, source, skip_function_bodies=strip_function_bodies
+    )
+    # Set is_stub based on file extension
+    tree.is_stub = fnam.endswith(".pyi")
+    # Note: tree.imports is populated directly by load_from_raw() with deserialized
+    # import metadata, so we don't need to collect imports via AST traversal
+    if eager and tree.raw_data is not None:
+        tree = load_from_raw(fnam, module, tree.raw_data, errors, options)
+    return tree
+
+
 def report_parse_error(error: ParseError, errors: Errors) -> None:
     message = error["message"]
     # Standardize error message by capitalizing the first word