[clang][depscan] Support prefix mappings when deciding modules that come

from "stable" directories

To support compiler caching, clang may strip out common prefixes from
paths like the sysroot. When this happens, include both the initial
input paths that would get mapped away and the resulting mapped prefix
for determining module dependencies that come from stable directories.

This is to handle all of the ways paths may come streaming. For example,
input paths that represented included headers will contain an absolute
paths for handling indirections like vfsoverlays but sysroot's from `CompilerInstance`'s will be mapped.

Author: cyndyishida

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -586,6 +586,11 @@ class DependencyScanningAction : public tooling::ToolAction {
     auto *FileMgr = ScanInstance.createFileManager(FS);
     ScanInstance.createSourceManager(*FileMgr);
 
+    // Initialize PrefixMapper incase mappings exist.
+    DepscanPrefixMapping::configurePrefixMapper(
+        ScanInstance.getFrontendOpts().PathPrefixMappings,
+        ScanInstance.getPrefixMapper());
+
     // Create a collection of stable directories derived from the ScanInstance
     // for determining whether module dependencies would fully resolve from
     // those directories.
@@ -595,6 +600,20 @@ class DependencyScanningAction : public tooling::ToolAction {
         (llvm::sys::path::root_directory(Sysroot) != Sysroot))
       StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir};
 
+    // When remapping is disabled, the stable directory paths are references to
+    // strings tied to the CompilerInstance. Otherwise, these potentially mapped
+    // strings need to be allocated. To reduce the number of downstream changes
+    // required to this support allocation, tie the lifetime of these strings to
+    // visiting all dependencies of a prebuilt module.
+    llvm::PrefixMapper &Mapper = ScanInstance.getPrefixMapper();
+    llvm::BumpPtrAllocator Allocator;
+    llvm::StringSaver StableDirStorage(Allocator);
+    if (!StableDirs.empty() && !Mapper.empty()) {
+      StableDirs.push_back(StableDirStorage.save(Mapper.mapToString(Sysroot)));
+      StableDirs.push_back(StableDirStorage.save(
+          Mapper.mapToString(ScanInstance.getHeaderSearchOpts().ResourceDir)));
+    }
+
     // Store a mapping of prebuilt module files and their properties like header
     // search options. This will prevent the implicit build to create duplicate
     // modules and will force reuse of the existing prebuilt module files

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -255,12 +255,11 @@ bool dependencies::isPathInStableDir(const ArrayRef<StringRef> Directories,
 
 bool dependencies::areOptionsInStableDir(const ArrayRef<StringRef> Directories,
                                          const HeaderSearchOptions &HSOpts) {
-  // FIXME: This breaks CAS prefix mapping tests.
-  // assert(isPathInStableDir(Directories, HSOpts.Sysroot) &&
-  //       "Sysroots differ between module dependencies and current TU");
+  assert(isPathInStableDir(Directories, HSOpts.Sysroot) &&
+         "Sysroots differ between module dependencies and current TU");
 
-  // assert(isPathInStableDir(Directories, HSOpts.ResourceDir) &&
-  //        "ResourceDirs differ between module dependencies and current TU");
+  assert(isPathInStableDir(Directories, HSOpts.ResourceDir) &&
+         "ResourceDirs differ between module dependencies and current TU");
 
   for (const auto &Entry : HSOpts.UserEntries) {
     if (!Entry.IgnoreSysRoot)

clang/test/ClangScanDeps/include-tree-with-pch.c

@@ -3,7 +3,7 @@
 // RUN: split-file %s %t
 // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
 
-// RUN: %clang -x c-header %t/prefix.h -target x86_64-apple-macos12 -o %t/prefix.pch -fdepscan=inline -fdepscan-include-tree -Xclang -fcas-path -Xclang %t/cas
+// RUN: %clang -x c-header %t/prefix.h -target x86_64-apple-macos12 -isysroot %t -o %t/prefix.pch -fdepscan=inline -fdepscan-include-tree -Xclang -fcas-path -Xclang %t/cas
 // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-include-tree -cas-path %t/cas > %t/result.txt
 // RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t
 

clang/test/ClangScanDeps/modules-cas-trees-input-files.c

@@ -37,7 +37,7 @@
 [
   {
     "directory" : "DIR",
-    "command" : "clang_tool -I DIR -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
+    "command" : "clang -I DIR -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
     "file" : "DIR/prefix.h"
   },
 ]
@@ -46,7 +46,7 @@
 [
   {
     "directory" : "DIR",
-    "command" : "clang_tool -I DIR -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
+    "command" : "clang -I DIR -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
     "file" : "DIR/tu.c"
   },
 ]

clang/test/ClangScanDeps/modules-cas-trees-with-pch.c

@@ -191,7 +191,7 @@
 [
   {
     "directory" : "DIR",
-    "command" : "clang_tool -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
+    "command" : "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
     "file" : "DIR/prefix.h"
   },
 ]
@@ -200,7 +200,7 @@
 [
   {
     "directory" : "DIR",
-    "command" : "clang_tool -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
+    "command" : "clang -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache",
     "file" : "DIR/tu.c"
   },
 ]

clang/test/ClangScanDeps/modules-in-stable-dirs-prefix-mapping.c

@@ -0,0 +1,94 @@
+// This test verifies modules that are entirely comprised from stable directory inputs are captured in
+// dependency information when paths are prefix mapped.
+ 
+// REQUIRES: shell,ondisk_cas
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/overlay.json.template > %t/overlay.json
+// RUN: sed -e "s|DIR|%/t|g" %t/compile.json.in > %t/compile.json
+
+// RUN: clang-scan-deps -compilation-database %t/compile.json \
+// RUN:   -prefix-map=%t/modules=/^modules -prefix-map=%t=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \
+// RUN:   -cas-path %t/cas -module-files-dir %t/modules \
+// RUN:   -j 1 -format experimental-full > %t/deps.db
+
+// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix DEP
+
+// DEP: "is-in-stable-directories": true
+// DEP: "name": "A"
+
+// CHECK-NOT: "is-in-stable-directories": true
+// CHECK-NOT: warning:
+// CHECK-NOT: error:
+
+//--- compile.json.in
+[
+{
+    "directory": "DIR",
+    "command": "clang -x c -c DIR/client.c -isysroot DIR/MacOSX.sdk -IDIR/BuildDir -ivfsoverlay DIR/overlay.json -IDIR/MacOSX.sdk/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -o DIR/client.c.o",
+    "file": "DIR/client.c"
+}
+]
+
+//--- overlay.json.template
+{
+  "version": 0,
+  "case-sensitive": "false",
+  "roots": [
+    {
+          "external-contents": "DIR/BuildDir/B_vfs.h",
+          "name": "DIR/MacOSX.sdk/usr/include/B/B_vfs.h",
+          "type": "file"
+    }
+  ]
+}
+
+//--- MacOSX.sdk/usr/include/A/module.modulemap
+module A [system] {
+  umbrella "."
+}
+
+//--- MacOSX.sdk/usr/include/A/A.h
+typedef int A_type;
+
+//--- MacOSX.sdk/usr/include/B/module.modulemap
+module B [system] {
+  umbrella "."
+}
+
+//--- MacOSX.sdk/usr/include/B/B.h
+#include <B/B_vfs.h>
+
+//--- BuildDir/B_vfs.h
+typedef int local_t;
+
+//--- MacOSX.sdk/usr/include/sys/sys.h
+#include <A/A.h>
+typedef int sys_t_m;
+
+//--- MacOSX.sdk/usr/include/sys/module.modulemap
+module sys [system] {
+  umbrella "."
+}
+
+//--- MacOSX.sdk/usr/include/B_transitive/B.h
+#include <B/B.h>
+
+//--- MacOSX.sdk/usr/include/B_transitive/module.modulemap
+module B_transitive [system] {
+  umbrella "."
+}
+
+//--- MacOSX.sdk/usr/include/C/module.modulemap
+module C [system] {
+  umbrella "."
+}
+
+//--- MacOSX.sdk/usr/include/C/C.h
+#include <B_transitive/B.h>
+
+
+//--- client.c
+#include <A/A.h>
+#include <C/C.h> // This dependency transitively depends on a local header.