[Dependency Scanning] Reduce the number of redundant dependency queries

include/swift/AST/ModuleDependencies.h

@@ -893,7 +893,7 @@ class ModuleDependencyInfo {
       llvm_unreachable("Unexpected module dependency kind");
   }
 
-  llvm::StringSet<> &getVisibleClangModules() const {
+  llvm::StringSet<> getVisibleClangModules() const {
     return storage->visibleClangModules;
   }
 
@@ -1073,6 +1073,14 @@ class ModuleDependenciesCache {
 private:
   /// Discovered dependencies
   ModuleDependenciesKindMap ModuleDependenciesMap;
+  /// A set of module identifiers for which a scanning action failed
+  /// to discover a Swift module dependency
+  llvm::StringSet<> negativeSwiftDependencyCache;
+
+  /// A map from Clang module name to all visible modules to a client
+  /// of a by-name import of this Clang module
+  llvm::StringMap<std::vector<std::string>> clangModulesVisibleFrom;
+
   /// Set containing all of the Clang modules that have already been seen.
   llvm::DenseSet<clang::tooling::dependencies::ModuleID> alreadySeenClangModules;
   /// Name of the module under scan
@@ -1111,6 +1119,8 @@ class ModuleDependenciesCache {
   bool hasDependency(StringRef moduleName) const;
   /// Whether we have cached dependency information for the given Swift module.
   bool hasSwiftDependency(StringRef moduleName) const;
+  /// Whether we have cached a failed lookup of a Swift dependency for the given identifier.
+  bool hasNegativeSwiftDependency(StringRef moduleName) const;
 
   const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &
   getAlreadySeenClangModules() const {
@@ -1148,8 +1158,12 @@ class ModuleDependenciesCache {
   llvm::ArrayRef<ModuleDependencyID>
   getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const;
   /// Query all visible Clang modules for a given Swift dependency
-  llvm::StringSet<>&
+  llvm::StringSet<>
   getVisibleClangModules(ModuleDependencyID moduleID) const;
+  /// Query all Clang modules visible via a by-name lookup of given
+  /// Clang dependency
+  std::vector<std::string>
+  getVisibleClangModulesFrom(ModuleDependencyID moduleID) const;
 
   /// Look for module dependencies for a module with the given ID
   ///
@@ -1226,7 +1240,13 @@ class ModuleDependenciesCache {
   /// Add to this module's set of visible Clang modules
   void
   addVisibleClangModules(ModuleDependencyID moduleID,
+                         ModuleDependencyID clangDependencyID,
                          const std::vector<std::string> &moduleNames);
+  void
+  addHeaderVisibleClangModules(ModuleDependencyID moduleID,
+                               const std::vector<std::string> &moduleNames);
+  /// Add an identifier to the set of failed Swift module queries
+  void cacheNegativeSwiftDependency(StringRef moduleIdentifier);
 
   StringRef getMainModuleName() const { return mainScanModuleName; }
 

lib/AST/ModuleDependencies.cpp

@@ -738,8 +738,6 @@ ModuleDependenciesCache::findSwiftDependency(StringRef moduleName) const {
     return found;
   if (auto found = findDependency(moduleName, ModuleDependencyKind::SwiftBinary))
     return found;
-  if (auto found = findDependency(moduleName, ModuleDependencyKind::SwiftSource))
-    return found;
   return std::nullopt;
 }
 
@@ -772,6 +770,10 @@ bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const {
   return findSwiftDependency(moduleName).has_value();
 }
 
+bool ModuleDependenciesCache::hasNegativeSwiftDependency(StringRef moduleName) const {
+  return negativeSwiftDependencyCache.contains(moduleName);
+}
+
 void ModuleDependenciesCache::recordDependency(
     StringRef moduleName, ModuleDependencyInfo dependency) {
   auto dependenciesKind = dependency.getKind();
@@ -848,6 +850,34 @@ void ModuleDependenciesCache::recordClangDependency(
   }
 }
 
+void ModuleDependenciesCache::addHeaderVisibleClangModules(
+    ModuleDependencyID moduleID,
+    const std::vector<std::string> &visibleModules) {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+
+  if (visibleModules.empty())
+    return;
+  auto dependencyInfo = findKnownDependency(moduleID);
+  auto updatedDependencyInfo = dependencyInfo;
+  updatedDependencyInfo.addVisibleClangModules(visibleModules);
+  updateDependency(moduleID, updatedDependencyInfo);
+}
+
+void ModuleDependenciesCache::addVisibleClangModules(
+    ModuleDependencyID moduleID,
+    ModuleDependencyID clangDependencyID,
+    const std::vector<std::string> &visibleModules) {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+  ASSERT(clangDependencyID.Kind == ModuleDependencyKind::Clang);
+
+  addHeaderVisibleClangModules(moduleID, visibleModules);
+  clangModulesVisibleFrom[clangDependencyID.ModuleName] = visibleModules;
+}
+
 void ModuleDependenciesCache::updateDependency(
     ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) {
   auto &map = getDependenciesMap(moduleID.Kind);
@@ -942,24 +972,26 @@ ModuleDependencyIDCollectionView ModuleDependenciesCache::getAllDependencies(
       moduleInfo.getImportedClangDependencies());
 }
 
-void ModuleDependenciesCache::addVisibleClangModules(
-    ModuleDependencyID moduleID, const std::vector<std::string> &moduleNames) {
-  if (moduleNames.empty())
-    return;
-  auto dependencyInfo = findKnownDependency(moduleID);
-  auto updatedDependencyInfo = dependencyInfo;
-  updatedDependencyInfo.addVisibleClangModules(moduleNames);
-  updateDependency(moduleID, updatedDependencyInfo);
+void ModuleDependenciesCache::cacheNegativeSwiftDependency(
+    StringRef moduleIdentifier) {
+  negativeSwiftDependencyCache.insert(moduleIdentifier);
 }
 
-llvm::StringSet<> &ModuleDependenciesCache::getVisibleClangModules(
+llvm::StringSet<> ModuleDependenciesCache::getVisibleClangModules(
     ModuleDependencyID moduleID) const {
   ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
          moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
          moduleID.Kind == ModuleDependencyKind::SwiftBinary);
   return findKnownDependency(moduleID).getVisibleClangModules();
 }
 
+std::vector<std::string> ModuleDependenciesCache::getVisibleClangModulesFrom(
+    ModuleDependencyID moduleID) const {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::Clang);
+  return clangModulesVisibleFrom.contains(moduleID.ModuleName) ?
+         clangModulesVisibleFrom.at(moduleID.ModuleName) : std::vector<std::string>() ;
+}
+
 ModuleDependencyIDCollectionView
 ModuleDependenciesCache::getDirectImportedDependencies(
     const ModuleDependencyID &moduleID) const {

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -26,6 +26,7 @@
 #include "swift/Basic/Defer.h"
 #include "swift/Basic/FileTypes.h"
 #include "swift/Basic/PrettyStackTrace.h"
+#include "swift/Basic/Statistic.h"
 #include "swift/ClangImporter/ClangImporter.h"
 #include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
@@ -960,13 +961,57 @@ void ModuleDependencyScanner::resolveSwiftModuleDependencies(
   return;
 }
 
-static void
-gatherUnresolvedImports(ModuleDependenciesCache &cache,
-                        ASTContext &scanASTContext,
-                        ArrayRef<ModuleDependencyID> swiftModuleDependents,
-                        ModuleDependencyIDSetVector &allDiscoveredClangModules,
-                        ImportStatementInfoMap &unresolvedImportsMap,
-                        ImportStatementInfoMap &unresolvedOptionalImportsMap) {
+static void resolveClangDependenciesFromCache(
+    ModuleDependenciesCache &cache, ASTContext &scanASTContext,
+    ArrayRef<ModuleDependencyID> swiftModuleDependents,
+    ModuleDependencyIDSetVector &allDiscoveredClangModules,
+    std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
+        &resolvedClangDependenciesMap) {
+  for (const auto &moduleID : swiftModuleDependents) {
+    ModuleDependencyInfo moduleDependencyInfo =
+        cache.findKnownDependency(moduleID);
+    if (!moduleDependencyInfo.getImportedClangDependencies().empty())
+      continue;
+
+    auto resolveImportIfCached =
+        [&](const ScannerImportStatementInfo &importInfo) {
+          auto dependencyID = ModuleDependencyID{importInfo.importIdentifier,
+                                                 ModuleDependencyKind::Clang};
+          auto visibleFromDependencyID =
+              cache.getVisibleClangModulesFrom(dependencyID);
+          if (cache.hasDependency(dependencyID) &&
+              !visibleFromDependencyID.empty()) {
+            cache.addVisibleClangModules(moduleID, dependencyID,
+                                         visibleFromDependencyID);
+            resolvedClangDependenciesMap[moduleID].insert(dependencyID);
+          }
+        };
+
+    for (const auto &depImport : moduleDependencyInfo.getModuleImports())
+      resolveImportIfCached(depImport);
+    for (const auto &depImport :
+         moduleDependencyInfo.getOptionalModuleImports())
+      resolveImportIfCached(depImport);
+  }
+}
+
+static void findAllReachableClangModules(ModuleDependencyID moduleID,
+                                         const ModuleDependenciesCache &cache,
+                                         ModuleDependencyIDSetVector &reachableClangModules) {
+  if (!reachableClangModules.insert(moduleID))
+    return;
+  for (const auto &depID : cache.getImportedClangDependencies(moduleID))
+    findAllReachableClangModules(depID, cache, reachableClangModules);
+}
+
+static void gatherUnresolvedImports(
+    ModuleDependenciesCache &cache, ASTContext &scanASTContext,
+    ArrayRef<ModuleDependencyID> swiftModuleDependents,
+    ModuleDependencyIDSetVector &allDiscoveredClangModules,
+    ImportStatementInfoMap &unresolvedImportsMap,
+    ImportStatementInfoMap &unresolvedOptionalImportsMap,
+    const std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
+                                    &resolvedClangDependenciesMap) {
   for (const auto &moduleID : swiftModuleDependents) {
     auto moduleDependencyInfo = cache.findKnownDependency(moduleID);
     auto unresolvedImports =
@@ -985,27 +1030,24 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache,
     if (!moduleDependencyInfo.getImportedClangDependencies().empty()) {
       auto directClangDeps = cache.getImportedClangDependencies(moduleID);
       ModuleDependencyIDSetVector reachableClangModules;
-      reachableClangModules.insert(directClangDeps.begin(),
-                                   directClangDeps.end());
-      for (unsigned currentModuleIdx = 0;
-           currentModuleIdx < reachableClangModules.size();
-           ++currentModuleIdx) {
-        auto moduleID = reachableClangModules[currentModuleIdx];
-        auto dependencies =
-            cache.findKnownDependency(moduleID).getImportedClangDependencies();
-        reachableClangModules.insert(dependencies.begin(), dependencies.end());
-      }
-      allDiscoveredClangModules.insert(reachableClangModules.begin(),
-                                       reachableClangModules.end());
+      for (const auto &depID : directClangDeps)
+        findAllReachableClangModules(depID, cache, reachableClangModules);
+      allDiscoveredClangModules.insert(reachableClangModules.getArrayRef().begin(),
+                                       reachableClangModules.getArrayRef().end());
       continue;
     } else {
       // We need to query the Clang dependency scanner for this module's
-      // non-Swift imports
+      // unresolved imports
       llvm::StringSet<> resolvedImportIdentifiers;
       for (const auto &resolvedDep :
            moduleDependencyInfo.getImportedSwiftDependencies())
         resolvedImportIdentifiers.insert(resolvedDep.ModuleName);
 
+      if (resolvedClangDependenciesMap.find(moduleID) != resolvedClangDependenciesMap.end())
+        for (const auto &cacheResolvedDep :
+             resolvedClangDependenciesMap.at(moduleID))
+          resolvedImportIdentifiers.insert(cacheResolvedDep.ModuleName);
+
       // When querying a *clang* module 'CxxStdlib' we must
       // instead expect a module called 'std'...
       auto addCanonicalClangModuleImport =
@@ -1073,7 +1115,8 @@ void ModuleDependencyScanner::reQueryMissedModulesFromCache(
       resolvedClangDependenciesMap[unresolvedImport.first].insert(
           unresolvedModuleID);
       DependencyCache.addVisibleClangModules(
-          unresolvedImport.first, {unresolvedImport.second.importIdentifier});
+          unresolvedImport.first, unresolvedModuleID,
+          {unresolvedImport.second.importIdentifier});
     } else {
       // Failed to resolve module dependency.
       IssueReporter.diagnoseModuleNotFoundFailure(
@@ -1155,7 +1198,8 @@ void ModuleDependencyScanner::cacheComputedClangModuleLookupResults(
           // Swift module
           if (lookupResult.visibleModules.contains(moduleIdentifier))
             DependencyCache.addVisibleClangModules(
-                moduleID, lookupResult.visibleModules.at(moduleIdentifier));
+                moduleID, dependencyID,
+                lookupResult.visibleModules.at(moduleIdentifier));
 
           // Add the resolved dependency ID
           if (lookupResult.discoveredDependencyInfos.contains(
@@ -1203,13 +1247,21 @@ void ModuleDependencyScanner::cacheComputedClangModuleLookupResults(
 void ModuleDependencyScanner::resolveAllClangModuleDependencies(
     ArrayRef<ModuleDependencyID> swiftModuleDependents,
     ModuleDependencyIDSetVector &allDiscoveredClangModules) {
-  // Gather all unresolved imports which must correspond to
+  std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
+      resolvedClangDependenciesMap;
+  resolveClangDependenciesFromCache(DependencyCache, ScanASTContext,
+                                    swiftModuleDependents,
+                                    allDiscoveredClangModules,
+                                    resolvedClangDependenciesMap);
+
+  // Gather all remaining unresolved imports which must correspond to
   // Clang modules (since no Swift module for them was found).
   ImportStatementInfoMap unresolvedImportsMap;
   ImportStatementInfoMap unresolvedOptionalImportsMap;
-  gatherUnresolvedImports(DependencyCache, ScanASTContext, swiftModuleDependents,
-                          allDiscoveredClangModules, unresolvedImportsMap,
-                          unresolvedOptionalImportsMap);
+  gatherUnresolvedImports(DependencyCache, ScanASTContext,
+                          swiftModuleDependents, allDiscoveredClangModules,
+                          unresolvedImportsMap, unresolvedOptionalImportsMap,
+                          resolvedClangDependenciesMap);
 
   // Execute parallel lookup of all unresolved import
   // identifiers as Clang modules.
@@ -1221,8 +1273,6 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies(
   // dependencies.
   std::vector<std::pair<ModuleDependencyID, ScannerImportStatementInfo>>
       failedToResolveImports;
-  std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
-      resolvedClangDependenciesMap;
   cacheComputedClangModuleLookupResults(
       lookupResult, unresolvedImportsMap, unresolvedOptionalImportsMap,
       swiftModuleDependents, allDiscoveredClangModules,
@@ -1337,6 +1387,13 @@ void ModuleDependencyScanner::resolveSwiftImportsForModule(
     // Avoid querying the underlying Clang module here
     if (moduleID.ModuleName == dependsOn.importIdentifier)
       continue;
+    // Avoid querying Swift module dependencies previously discovered
+    if (DependencyCache.hasSwiftDependency(dependsOn.importIdentifier))
+      continue;
+    // Avoid querying Swift module dependencies which have already produced
+    // in a negative (not found) result
+    if (DependencyCache.hasNegativeSwiftDependency(dependsOn.importIdentifier))
+      continue;
     ScanningThreadPool.async(
         scanForSwiftModuleDependency,
         getModuleImportIdentifier(dependsOn.importIdentifier),
@@ -1376,10 +1433,13 @@ void ModuleDependencyScanner::resolveSwiftImportsForModule(
                        moduleImport.importIdentifier))
           importedSwiftDependencies.insert(
               {moduleImport.importIdentifier, cachedInfo.value()->getKind()});
-        else
+        else {
           IssueReporter.diagnoseFailureOnOnlyIncompatibleCandidates(
-              moduleImport, lookupResult.incompatibleCandidates,
-              DependencyCache, std::nullopt);
+                     moduleImport, lookupResult.incompatibleCandidates,
+                     DependencyCache, std::nullopt);
+          DependencyCache
+            .cacheNegativeSwiftDependency(moduleImport.importIdentifier);
+        }
       };
 
   for (const auto &importInfo : moduleDependencyInfo.getModuleImports())
@@ -1495,6 +1555,8 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule(
           // Update the set of visible Clang modules
           moduleDependencyInfo.addVisibleClangModules(
               headerScanResult->VisibleModules);
+          DependencyCache.addHeaderVisibleClangModules(moduleID,
+              headerScanResult->VisibleModules);
           // Update the dependency in the cache
           DependencyCache.updateDependency(moduleID, moduleDependencyInfo);
         } else {
@@ -1523,10 +1585,8 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule(
     auto moduleName = moduleIdentifier.str();
     {
       std::lock_guard<std::mutex> guard(lookupResultLock);
-      if (DependencyCache.hasDependency(moduleName,
-                                        ModuleDependencyKind::SwiftInterface) ||
-          DependencyCache.hasDependency(moduleName,
-                                        ModuleDependencyKind::SwiftBinary))
+      if (DependencyCache.hasSwiftDependency(moduleName) ||
+          DependencyCache.hasNegativeSwiftDependency(moduleName))
         return;
     }
 

lib/DependencyScan/ScanDependencies.cpp

@@ -1599,7 +1599,7 @@ void swift::dependencies::incremental::validateInterModuleDependenciesCache(
                       emitRemarks, visited, modulesRequiringRescan);
   for (const auto &outOfDateModID : modulesRequiringRescan)
     cache.removeDependency(outOfDateModID);
-
+  // ACTODO: Invalidate visible modules
   // Regardless of invalidation, always re-scan main module.
   cache.removeDependency(rootModuleID);
 }