[Dependency Scanning] Re-use cached queried Clang dependency info in subsequent scanning stages

Dependency Scanning is recursive over discovered Swift module overlays. In the main 'resolveImportedModuleDependencies' routine, after all Swift and Clang dependencies of the main module are discovered, the scanner looks up Swift overlays for discovered Clang modules. For each such Swift overlay, the scanner will then proceed to call 'resolveImportedModuleDependencies' on the overlay module.

On these subsequent recursive calls to 'resolveImportedModuleDependencies', Clang dependency resolution will re-query all imports of the overlay module which do not resolve to Swift modules, even if they had been queried previously in a parent invocation. This change adds the ability to re-use previously-queried-and-cached Clang module dependency information by having the dependency cache also store the set of visible modules which resulted from each by-name lookup.

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -893,7 +893,7 @@ class ModuleDependencyInfo {
       llvm_unreachable("Unexpected module dependency kind");
   }
 
-  llvm::StringSet<> &getVisibleClangModules() const {
+  llvm::StringSet<> getVisibleClangModules() const {
     return storage->visibleClangModules;
   }
 
@@ -1076,6 +1076,11 @@ class ModuleDependenciesCache {
   /// A set of module identifiers for which a scanning action failed
   /// to discover a Swift module dependency
   llvm::StringSet<> negativeSwiftDependencyCache;
+
+  /// A map from Clang module name to all visible modules to a client
+  /// of a by-name import of this Clang module
+  llvm::StringMap<std::vector<std::string>> clangModulesVisibleFrom;
+
   /// Set containing all of the Clang modules that have already been seen.
   llvm::DenseSet<clang::tooling::dependencies::ModuleID> alreadySeenClangModules;
   /// Name of the module under scan
@@ -1153,8 +1158,12 @@ class ModuleDependenciesCache {
   llvm::ArrayRef<ModuleDependencyID>
   getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const;
   /// Query all visible Clang modules for a given Swift dependency
-  llvm::StringSet<>&
+  llvm::StringSet<>
   getVisibleClangModules(ModuleDependencyID moduleID) const;
+  /// Query all Clang modules visible via a by-name lookup of given
+  /// Clang dependency
+  std::vector<std::string>
+  getVisibleClangModulesFrom(ModuleDependencyID moduleID) const;
 
   /// Look for module dependencies for a module with the given ID
   ///
@@ -1231,7 +1240,11 @@ class ModuleDependenciesCache {
   /// Add to this module's set of visible Clang modules
   void
   addVisibleClangModules(ModuleDependencyID moduleID,
+                         ModuleDependencyID clangDependencyID,
                          const std::vector<std::string> &moduleNames);
+  void
+  addHeaderVisibleClangModules(ModuleDependencyID moduleID,
+                               const std::vector<std::string> &moduleNames);
   /// Add an identifier to the set of failed Swift module queries
   void cacheNegativeSwiftDependency(StringRef moduleIdentifier);
 

lib/AST/ModuleDependencies.cpp

@@ -850,6 +850,34 @@ void ModuleDependenciesCache::recordClangDependency(
   }
 }
 
+void ModuleDependenciesCache::addHeaderVisibleClangModules(
+    ModuleDependencyID moduleID,
+    const std::vector<std::string> &visibleModules) {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+
+  if (visibleModules.empty())
+    return;
+  auto dependencyInfo = findKnownDependency(moduleID);
+  auto updatedDependencyInfo = dependencyInfo;
+  updatedDependencyInfo.addVisibleClangModules(visibleModules);
+  updateDependency(moduleID, updatedDependencyInfo);
+}
+
+void ModuleDependenciesCache::addVisibleClangModules(
+    ModuleDependencyID moduleID,
+    ModuleDependencyID clangDependencyID,
+    const std::vector<std::string> &visibleModules) {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+  ASSERT(clangDependencyID.Kind == ModuleDependencyKind::Clang);
+
+  addHeaderVisibleClangModules(moduleID, visibleModules);
+  clangModulesVisibleFrom[clangDependencyID.ModuleName] = visibleModules;
+}
+
 void ModuleDependenciesCache::updateDependency(
     ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) {
   auto &map = getDependenciesMap(moduleID.Kind);
@@ -944,29 +972,26 @@ ModuleDependencyIDCollectionView ModuleDependenciesCache::getAllDependencies(
       moduleInfo.getImportedClangDependencies());
 }
 
-void ModuleDependenciesCache::addVisibleClangModules(
-    ModuleDependencyID moduleID, const std::vector<std::string> &moduleNames) {
-  if (moduleNames.empty())
-    return;
-  auto dependencyInfo = findKnownDependency(moduleID);
-  auto updatedDependencyInfo = dependencyInfo;
-  updatedDependencyInfo.addVisibleClangModules(moduleNames);
-  updateDependency(moduleID, updatedDependencyInfo);
-}
-
 void ModuleDependenciesCache::cacheNegativeSwiftDependency(
     StringRef moduleIdentifier) {
   negativeSwiftDependencyCache.insert(moduleIdentifier);
 }
 
-llvm::StringSet<> &ModuleDependenciesCache::getVisibleClangModules(
+llvm::StringSet<> ModuleDependenciesCache::getVisibleClangModules(
     ModuleDependencyID moduleID) const {
   ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
          moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
          moduleID.Kind == ModuleDependencyKind::SwiftBinary);
   return findKnownDependency(moduleID).getVisibleClangModules();
 }
 
+std::vector<std::string> ModuleDependenciesCache::getVisibleClangModulesFrom(
+    ModuleDependencyID moduleID) const {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::Clang);
+  return clangModulesVisibleFrom.contains(moduleID.ModuleName) ?
+         clangModulesVisibleFrom.at(moduleID.ModuleName) : std::vector<std::string>() ;
+}
+
 ModuleDependencyIDCollectionView
 ModuleDependenciesCache::getDirectImportedDependencies(
     const ModuleDependencyID &moduleID) const {

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -961,13 +961,57 @@ void ModuleDependencyScanner::resolveSwiftModuleDependencies(
   return;
 }
 
-static void
-gatherUnresolvedImports(ModuleDependenciesCache &cache,
-                        ASTContext &scanASTContext,
-                        ArrayRef<ModuleDependencyID> swiftModuleDependents,
-                        ModuleDependencyIDSetVector &allDiscoveredClangModules,
-                        ImportStatementInfoMap &unresolvedImportsMap,
-                        ImportStatementInfoMap &unresolvedOptionalImportsMap) {
+static void resolveClangDependenciesFromCache(
+    ModuleDependenciesCache &cache, ASTContext &scanASTContext,
+    ArrayRef<ModuleDependencyID> swiftModuleDependents,
+    ModuleDependencyIDSetVector &allDiscoveredClangModules,
+    std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
+        &resolvedClangDependenciesMap) {
+  for (const auto &moduleID : swiftModuleDependents) {
+    ModuleDependencyInfo moduleDependencyInfo =
+        cache.findKnownDependency(moduleID);
+    if (!moduleDependencyInfo.getImportedClangDependencies().empty())
+      continue;
+
+    auto resolveImportIfCached =
+        [&](const ScannerImportStatementInfo &importInfo) {
+          auto dependencyID = ModuleDependencyID{importInfo.importIdentifier,
+                                                 ModuleDependencyKind::Clang};
+          auto visibleFromDependencyID =
+              cache.getVisibleClangModulesFrom(dependencyID);
+          if (cache.hasDependency(dependencyID) &&
+              !visibleFromDependencyID.empty()) {
+            cache.addVisibleClangModules(moduleID, dependencyID,
+                                         visibleFromDependencyID);
+            resolvedClangDependenciesMap[moduleID].insert(dependencyID);
+          }
+        };
+
+    for (const auto &depImport : moduleDependencyInfo.getModuleImports())
+      resolveImportIfCached(depImport);
+    for (const auto &depImport :
+         moduleDependencyInfo.getOptionalModuleImports())
+      resolveImportIfCached(depImport);
+  }
+}
+
+static void findAllReachableClangModules(ModuleDependencyID moduleID,
+                                         const ModuleDependenciesCache &cache,
+                                         ModuleDependencyIDSetVector &reachableClangModules) {
+  if (!reachableClangModules.insert(moduleID))
+    return;
+  for (const auto &depID : cache.getImportedClangDependencies(moduleID))
+    findAllReachableClangModules(depID, cache, reachableClangModules);
+}
+
+static void gatherUnresolvedImports(
+    ModuleDependenciesCache &cache, ASTContext &scanASTContext,
+    ArrayRef<ModuleDependencyID> swiftModuleDependents,
+    ModuleDependencyIDSetVector &allDiscoveredClangModules,
+    ImportStatementInfoMap &unresolvedImportsMap,
+    ImportStatementInfoMap &unresolvedOptionalImportsMap,
+    const std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
+                                    &resolvedClangDependenciesMap) {
   for (const auto &moduleID : swiftModuleDependents) {
     auto moduleDependencyInfo = cache.findKnownDependency(moduleID);
     auto unresolvedImports =
@@ -986,27 +1030,24 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache,
     if (!moduleDependencyInfo.getImportedClangDependencies().empty()) {
       auto directClangDeps = cache.getImportedClangDependencies(moduleID);
       ModuleDependencyIDSetVector reachableClangModules;
-      reachableClangModules.insert(directClangDeps.begin(),
-                                   directClangDeps.end());
-      for (unsigned currentModuleIdx = 0;
-           currentModuleIdx < reachableClangModules.size();
-           ++currentModuleIdx) {
-        auto moduleID = reachableClangModules[currentModuleIdx];
-        auto dependencies =
-            cache.findKnownDependency(moduleID).getImportedClangDependencies();
-        reachableClangModules.insert(dependencies.begin(), dependencies.end());
-      }
-      allDiscoveredClangModules.insert(reachableClangModules.begin(),
-                                       reachableClangModules.end());
+      for (const auto &depID : directClangDeps)
+        findAllReachableClangModules(depID, cache, reachableClangModules);
+      allDiscoveredClangModules.insert(reachableClangModules.getArrayRef().begin(),
+                                       reachableClangModules.getArrayRef().end());
       continue;
     } else {
       // We need to query the Clang dependency scanner for this module's
-      // non-Swift imports
+      // unresolved imports
       llvm::StringSet<> resolvedImportIdentifiers;
       for (const auto &resolvedDep :
            moduleDependencyInfo.getImportedSwiftDependencies())
         resolvedImportIdentifiers.insert(resolvedDep.ModuleName);
 
+      if (resolvedClangDependenciesMap.find(moduleID) != resolvedClangDependenciesMap.end())
+        for (const auto &cacheResolvedDep :
+             resolvedClangDependenciesMap.at(moduleID))
+          resolvedImportIdentifiers.insert(cacheResolvedDep.ModuleName);
+
       // When querying a *clang* module 'CxxStdlib' we must
       // instead expect a module called 'std'...
       auto addCanonicalClangModuleImport =
@@ -1074,7 +1115,8 @@ void ModuleDependencyScanner::reQueryMissedModulesFromCache(
       resolvedClangDependenciesMap[unresolvedImport.first].insert(
           unresolvedModuleID);
       DependencyCache.addVisibleClangModules(
-          unresolvedImport.first, {unresolvedImport.second.importIdentifier});
+          unresolvedImport.first, unresolvedModuleID,
+          {unresolvedImport.second.importIdentifier});
     } else {
       // Failed to resolve module dependency.
       IssueReporter.diagnoseModuleNotFoundFailure(
@@ -1156,7 +1198,8 @@ void ModuleDependencyScanner::cacheComputedClangModuleLookupResults(
           // Swift module
           if (lookupResult.visibleModules.contains(moduleIdentifier))
             DependencyCache.addVisibleClangModules(
-                moduleID, lookupResult.visibleModules.at(moduleIdentifier));
+                moduleID, dependencyID,
+                lookupResult.visibleModules.at(moduleIdentifier));
 
           // Add the resolved dependency ID
           if (lookupResult.discoveredDependencyInfos.contains(
@@ -1204,13 +1247,21 @@ void ModuleDependencyScanner::cacheComputedClangModuleLookupResults(
 void ModuleDependencyScanner::resolveAllClangModuleDependencies(
     ArrayRef<ModuleDependencyID> swiftModuleDependents,
     ModuleDependencyIDSetVector &allDiscoveredClangModules) {
-  // Gather all unresolved imports which must correspond to
+  std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
+      resolvedClangDependenciesMap;
+  resolveClangDependenciesFromCache(DependencyCache, ScanASTContext,
+                                    swiftModuleDependents,
+                                    allDiscoveredClangModules,
+                                    resolvedClangDependenciesMap);
+
+  // Gather all remaining unresolved imports which must correspond to
   // Clang modules (since no Swift module for them was found).
   ImportStatementInfoMap unresolvedImportsMap;
   ImportStatementInfoMap unresolvedOptionalImportsMap;
-  gatherUnresolvedImports(DependencyCache, ScanASTContext, swiftModuleDependents,
-                          allDiscoveredClangModules, unresolvedImportsMap,
-                          unresolvedOptionalImportsMap);
+  gatherUnresolvedImports(DependencyCache, ScanASTContext,
+                          swiftModuleDependents, allDiscoveredClangModules,
+                          unresolvedImportsMap, unresolvedOptionalImportsMap,
+                          resolvedClangDependenciesMap);
 
   // Execute parallel lookup of all unresolved import
   // identifiers as Clang modules.
@@ -1222,8 +1273,6 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies(
   // dependencies.
   std::vector<std::pair<ModuleDependencyID, ScannerImportStatementInfo>>
       failedToResolveImports;
-  std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
-      resolvedClangDependenciesMap;
   cacheComputedClangModuleLookupResults(
       lookupResult, unresolvedImportsMap, unresolvedOptionalImportsMap,
       swiftModuleDependents, allDiscoveredClangModules,
@@ -1506,6 +1555,8 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule(
           // Update the set of visible Clang modules
           moduleDependencyInfo.addVisibleClangModules(
               headerScanResult->VisibleModules);
+          DependencyCache.addHeaderVisibleClangModules(moduleID,
+              headerScanResult->VisibleModules);
           // Update the dependency in the cache
           DependencyCache.updateDependency(moduleID, moduleDependencyInfo);
         } else {

lib/DependencyScan/ScanDependencies.cpp

@@ -1599,7 +1599,7 @@ void swift::dependencies::incremental::validateInterModuleDependenciesCache(
                       emitRemarks, visited, modulesRequiringRescan);
   for (const auto &outOfDateModID : modulesRequiringRescan)
     cache.removeDependency(outOfDateModID);
-
+  // ACTODO: Invalidate visible modules
   // Regardless of invalidation, always re-scan main module.
   cache.removeDependency(rootModuleID);
 }