Improve fully cached build times. (#6127)

alliepiper · web-flow · commit 542862e59757 · 2025-10-13T11:25:07.000-04:00
* Improve fully-cached build performance in CI.

Reduces the build time of a fully-cached build on linux from ~15m to &lt;5m,
and on windows from &gt;1hr to ~15m.

- Switches to using the rapids fork of sccache (better nvcc whole-compilation caching)
- Use Github Cache to persist sccache preprocessor caching.
- Remove the scccache summary tools and replace with raw dumps.
  - The sccache stats output is not stable, and was broken in windows.
  - The full advanced stats output is more useful, as it breaks down
    cache by tool (nvcc, ptxas, cicc, host compiler, etc) and helps
    identify where cache misses are occurring.

* Ensure devcontainers are using rapids sccache.

Move DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 into the devcontainer files from the GHA action, per review comment.
diff --git a/.devcontainer/cuda12.0-gcc10/devcontainer.json b/.devcontainer/cuda12.0-gcc10/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc10",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "10",
diff --git a/.devcontainer/cuda12.0-gcc11/devcontainer.json b/.devcontainer/cuda12.0-gcc11/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc11",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "11",
diff --git a/.devcontainer/cuda12.0-gcc12/devcontainer.json b/.devcontainer/cuda12.0-gcc12/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc12",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "12",
diff --git a/.devcontainer/cuda12.0-gcc13/devcontainer.json b/.devcontainer/cuda12.0-gcc13/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc13",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "13",
diff --git a/.devcontainer/cuda12.0-gcc7/devcontainer.json b/.devcontainer/cuda12.0-gcc7/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc7",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "7",
diff --git a/.devcontainer/cuda12.0-gcc8/devcontainer.json b/.devcontainer/cuda12.0-gcc8/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc8",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "8",
diff --git a/.devcontainer/cuda12.0-gcc9/devcontainer.json b/.devcontainer/cuda12.0-gcc9/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-gcc9",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "9",
diff --git a/.devcontainer/cuda12.0-llvm14/devcontainer.json b/.devcontainer/cuda12.0-llvm14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.0-llvm14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda12.9-gcc10/devcontainer.json b/.devcontainer/cuda12.9-gcc10/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc10",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "10",
diff --git a/.devcontainer/cuda12.9-gcc11/devcontainer.json b/.devcontainer/cuda12.9-gcc11/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc11",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "11",
diff --git a/.devcontainer/cuda12.9-gcc12/devcontainer.json b/.devcontainer/cuda12.9-gcc12/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc12",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "12",
diff --git a/.devcontainer/cuda12.9-gcc13/devcontainer.json b/.devcontainer/cuda12.9-gcc13/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc13",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "13",
diff --git a/.devcontainer/cuda12.9-gcc14/devcontainer.json b/.devcontainer/cuda12.9-gcc14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda12.9-gcc7/devcontainer.json b/.devcontainer/cuda12.9-gcc7/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc7",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "7",
diff --git a/.devcontainer/cuda12.9-gcc8/devcontainer.json b/.devcontainer/cuda12.9-gcc8/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc8",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "8",
diff --git a/.devcontainer/cuda12.9-gcc9/devcontainer.json b/.devcontainer/cuda12.9-gcc9/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-gcc9",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "9",
diff --git a/.devcontainer/cuda12.9-llvm14/devcontainer.json b/.devcontainer/cuda12.9-llvm14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda12.9-llvm15/devcontainer.json b/.devcontainer/cuda12.9-llvm15/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm15",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "15",
diff --git a/.devcontainer/cuda12.9-llvm16/devcontainer.json b/.devcontainer/cuda12.9-llvm16/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm16",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "16",
diff --git a/.devcontainer/cuda12.9-llvm17/devcontainer.json b/.devcontainer/cuda12.9-llvm17/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm17",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "17",
diff --git a/.devcontainer/cuda12.9-llvm18/devcontainer.json b/.devcontainer/cuda12.9-llvm18/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm18",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "18",
diff --git a/.devcontainer/cuda12.9-llvm19/devcontainer.json b/.devcontainer/cuda12.9-llvm19/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm19",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "19",
diff --git a/.devcontainer/cuda12.9-llvm20/devcontainer.json b/.devcontainer/cuda12.9-llvm20/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-llvm20",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "20",
diff --git a/.devcontainer/cuda12.9-nvhpc25.5/devcontainer.json b/.devcontainer/cuda12.9-nvhpc25.5/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-nvhpc25.5",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "nvhpc",
     "CCCL_HOST_COMPILER_VERSION": "25.5",
diff --git a/.devcontainer/cuda12.9-nvhpc25.7/devcontainer.json b/.devcontainer/cuda12.9-nvhpc25.7/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9-nvhpc25.7",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "nvhpc",
     "CCCL_HOST_COMPILER_VERSION": "25.7",
diff --git a/.devcontainer/cuda12.9ext-gcc14/devcontainer.json b/.devcontainer/cuda12.9ext-gcc14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9ext-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda12.9ext-llvm20/devcontainer.json b/.devcontainer/cuda12.9ext-llvm20/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda12.9ext-llvm20",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "12.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "20",
diff --git a/.devcontainer/cuda13.0-gcc11/devcontainer.json b/.devcontainer/cuda13.0-gcc11/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-gcc11",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "11",
diff --git a/.devcontainer/cuda13.0-gcc12/devcontainer.json b/.devcontainer/cuda13.0-gcc12/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-gcc12",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "12",
diff --git a/.devcontainer/cuda13.0-gcc13/devcontainer.json b/.devcontainer/cuda13.0-gcc13/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-gcc13",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "13",
diff --git a/.devcontainer/cuda13.0-gcc14/devcontainer.json b/.devcontainer/cuda13.0-gcc14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda13.0-llvm15/devcontainer.json b/.devcontainer/cuda13.0-llvm15/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-llvm15",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "15",
diff --git a/.devcontainer/cuda13.0-llvm16/devcontainer.json b/.devcontainer/cuda13.0-llvm16/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-llvm16",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "16",
diff --git a/.devcontainer/cuda13.0-llvm17/devcontainer.json b/.devcontainer/cuda13.0-llvm17/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-llvm17",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "17",
diff --git a/.devcontainer/cuda13.0-llvm18/devcontainer.json b/.devcontainer/cuda13.0-llvm18/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-llvm18",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "18",
diff --git a/.devcontainer/cuda13.0-llvm19/devcontainer.json b/.devcontainer/cuda13.0-llvm19/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-llvm19",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "19",
diff --git a/.devcontainer/cuda13.0-llvm20/devcontainer.json b/.devcontainer/cuda13.0-llvm20/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-llvm20",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "20",
diff --git a/.devcontainer/cuda13.0ext-gcc14/devcontainer.json b/.devcontainer/cuda13.0ext-gcc14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0ext-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda13.0ext-llvm20/devcontainer.json b/.devcontainer/cuda13.0ext-llvm20/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0ext-llvm20",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "20",
diff --git a/.devcontainer/cuda99.8-gcc14/devcontainer.json b/.devcontainer/cuda99.8-gcc14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda99.8-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "99.8",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda99.8-llvm20/devcontainer.json b/.devcontainer/cuda99.8-llvm20/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda99.8-llvm20",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "99.8",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "20",
diff --git a/.devcontainer/cuda99.9-gcc14/devcontainer.json b/.devcontainer/cuda99.9-gcc14/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda99.9-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "99.9",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.devcontainer/cuda99.9-llvm20/devcontainer.json b/.devcontainer/cuda99.9-llvm20/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda99.9-llvm20",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "99.9",
     "CCCL_HOST_COMPILER": "llvm",
     "CCCL_HOST_COMPILER_VERSION": "20",
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -25,6 +25,7 @@
     "AWS_ROLE_ARN": "arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history",
     "DEVCONTAINER_NAME": "cuda13.0-gcc14",
+    "DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST": "1",
     "CCCL_CUDA_VERSION": "13.0",
     "CCCL_HOST_COMPILER": "gcc",
     "CCCL_HOST_COMPILER_VERSION": "14",
diff --git a/.github/actions/workflow-run-job-linux/action.yml b/.github/actions/workflow-run-job-linux/action.yml
diff --git a/.github/actions/workflow-run-job-windows/action.yml b/.github/actions/workflow-run-job-windows/action.yml
diff --git a/ci/build_common.sh b/ci/build_common.sh
diff --git a/ci/nvrtc_libcudacxx.sh b/ci/nvrtc_libcudacxx.sh
diff --git a/ci/sccache_hit_rate.sh b/ci/sccache_hit_rate.sh
diff --git a/ci/sccache_stats.sh b/ci/sccache_stats.sh
diff --git a/ci/test_libcudacxx.sh b/ci/test_libcudacxx.sh
diff --git a/ci/util/workflow/get_job_project.sh b/ci/util/workflow/get_job_project.sh
diff --git a/ci/util/workflow/get_stable_job_hash.sh b/ci/util/workflow/get_stable_job_hash.sh
diff --git a/ci/windows/build_common.psm1 b/ci/windows/build_common.psm1
