Turbopack: only enable nested async availability in production

This avoid the combinations of paths problem in dev since this creates a lot of output files in development (every chunk group has a register chunk group chunk in dev, and we don't have commons chunk optimization in dev)

Author: sokra

crates/next-core/src/next_client/context.rs

@@ -510,6 +510,7 @@ pub async fn get_client_chunking_context(
                 },
             )
             .use_content_hashing(ContentHashing::Direct { length: 16 })
+            .nested_async_availability(true)
             .module_merging(*scope_hoisting.await?);
     }
 

crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_module.rs

@@ -329,9 +329,10 @@ impl EcmascriptChunkItem for EcmascriptClientReferenceProxyChunkItem {
     fn content_with_async_module_info(
         &self,
         async_module_info: Option<Vc<AsyncModuleInfo>>,
+        estimated: bool,
     ) -> Vc<EcmascriptChunkItemContent> {
         self.inner_chunk_item
-            .content_with_async_module_info(async_module_info)
+            .content_with_async_module_info(async_module_info, estimated)
     }
 }
 

turbopack/crates/turbopack-browser/src/chunking_context.rs

@@ -101,6 +101,11 @@ impl BrowserChunkingContextBuilder {
         self
     }
 
+    pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self {
+        self.chunking_context.enable_nested_async_availability = enable_nested_async_availability;
+        self
+    }
+
     pub fn module_merging(mut self, enable_module_merging: bool) -> Self {
         self.chunking_context.enable_module_merging = enable_module_merging;
         self
@@ -261,6 +266,8 @@ pub struct BrowserChunkingContext {
     enable_hot_module_replacement: bool,
     /// Enable tracing for this chunking
     enable_tracing: bool,
+    /// Enable nested async availability for this chunking
+    enable_nested_async_availability: bool,
     /// Enable module merging
     enable_module_merging: bool,
     /// Enable dynamic chunk content loading.
@@ -320,6 +327,7 @@ impl BrowserChunkingContext {
                 asset_base_paths: Default::default(),
                 enable_hot_module_replacement: false,
                 enable_tracing: false,
+                enable_nested_async_availability: false,
                 enable_module_merging: false,
                 enable_dynamic_chunk_content_loading: false,
                 debug_ids: false,
@@ -626,6 +634,11 @@ impl ChunkingContext for BrowserChunkingContext {
         Vc::cell(self.enable_tracing)
     }
 
+    #[turbo_tasks::function]
+    fn is_nested_async_availability_enabled(&self) -> Vc<bool> {
+        Vc::cell(self.enable_nested_async_availability)
+    }
+
     #[turbo_tasks::function]
     fn is_module_merging_enabled(&self) -> Vc<bool> {
         Vc::cell(self.enable_module_merging)

turbopack/crates/turbopack-cli/src/build/mod.rs

@@ -367,6 +367,7 @@ async fn build_internal(
                             },
                         )
                         .use_content_hashing(ContentHashing::Direct { length: 16 })
+                        .nested_async_availability(true)
                         .module_merging(scope_hoist);
                 }
             }

turbopack/crates/turbopack-cli/src/dev/web_entry_source.rs

@@ -55,6 +55,7 @@ pub async fn get_client_chunking_context(
         .hot_module_replacement()
         .source_map_source_type(SourceMapSourceType::AbsoluteFileUri)
         .dynamic_chunk_content_loading(true)
+        .nested_async_availability(true)
         .build(),
     ))
 }

turbopack/crates/turbopack-core/src/chunk/availability_info.rs

@@ -10,6 +10,7 @@ bitfield! {
     #[derive(Clone, Copy, Default, TaskInput, TraceRawVcs, NonLocalValue, Serialize, Deserialize, PartialEq, Eq, Hash)]
     pub struct AvailabilityFlags(u8);
     impl Debug;
+    pub is_in_of_async_module, set_is_in_of_async_module: 0;
 }
 
 #[derive(
@@ -62,6 +63,19 @@ impl AvailabilityInfo {
         })
     }
 
+    pub fn in_async_module(self) -> Self {
+        let mut flags = self.flags;
+        flags.set_is_in_of_async_module(true);
+        Self {
+            flags,
+            available_modules: self.available_modules,
+        }
+    }
+
+    pub fn is_in_async_module(&self) -> bool {
+        self.flags.is_in_of_async_module()
+    }
+
     pub async fn ident(&self) -> Result<Option<RcStr>> {
         Ok(if let Some(available_modules) = self.available_modules {
             Some(available_modules.hash().await?.to_string().into())

turbopack/crates/turbopack-core/src/chunk/chunk_group.rs

@@ -54,6 +54,9 @@ pub async fn make_chunk_group(
         *chunking_context.environment().chunk_loading().await?,
         ChunkLoading::Edge
     );
+    let is_nested_async_availability_enabled = *chunking_context
+        .is_nested_async_availability_enabled()
+        .await?;
     let should_trace = *chunking_context.is_tracing_enabled().await?;
     let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
     let batching_config = chunking_context.batching_config();
@@ -63,15 +66,17 @@ pub async fn make_chunk_group(
         batch_groups,
         async_modules,
         traced_modules,
-        availability_info,
+        availability_info: new_availability_info,
     } = chunk_group_content(
         module_graph,
         chunk_group_entries.clone(),
-        availability_info,
-        can_split_async,
-        should_trace,
-        should_merge_modules,
-        batching_config,
+        ChunkGroupContentOptions {
+            availability_info,
+            can_split_async,
+            should_trace,
+            should_merge_modules,
+            batching_config,
+        },
     )
     .await?;
 
@@ -109,11 +114,17 @@ pub async fn make_chunk_group(
         .await?;
 
     // Insert async chunk loaders for every referenced async module
+    let async_availability_info =
+        if is_nested_async_availability_enabled || !availability_info.is_in_async_module() {
+            new_availability_info.in_async_module()
+        } else {
+            availability_info
+        };
     let async_loaders = async_modules
         .into_iter()
         .map(async |module| {
             chunking_context
-                .async_loader_chunk_item(*module, module_graph, availability_info)
+                .async_loader_chunk_item(*module, module_graph, async_availability_info)
                 .to_resolved()
                 .await
         })
@@ -153,7 +164,7 @@ pub async fn make_chunk_group(
         chunks,
         referenced_output_assets,
         references: ResolvedVc::upcast_vec(async_loaders),
-        availability_info,
+        availability_info: new_availability_info,
     })
 }
 
@@ -180,16 +191,43 @@ pub async fn references_to_output_assets(
     .cell())
 }
 
+pub struct ChunkGroupContentOptions {
+    /// The availability info of the chunk group
+    pub availability_info: AvailabilityInfo,
+    /// Whether async modules can be split into separate chunks
+    pub can_split_async: bool,
+    /// Whether traced modules should be collected
+    pub should_trace: bool,
+    /// Whether module merging is enabled
+    pub should_merge_modules: bool,
+    /// The batching config to use
+    pub batching_config: Vc<BatchingConfig>,
+}
+
+/// Computes the content of a chunk group.
+///
+/// ### Parameters:
+/// - module_graph: the module graph
+/// - chunk_group_entries: the entries of the chunk group
+/// - availability_info: the availability info of the chunk group
+/// - can_split_async: indicates whether async modules can be split into separate chunks
+/// - update_availability_info: indicates whether to return a new availability info which includes
+///   all chunk items.
+/// - should_trace: indicates whether traced modules should be collected
+/// - should_merge_modules: indicates whether module merging is enabled
+/// - batching_config: the batching config to use
 pub async fn chunk_group_content(
     module_graph: Vc<ModuleGraph>,
     chunk_group_entries: impl IntoIterator<
         IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
     > + Send,
-    availability_info: AvailabilityInfo,
-    can_split_async: bool,
-    should_trace: bool,
-    should_merge_modules: bool,
-    batching_config: Vc<BatchingConfig>,
+    ChunkGroupContentOptions {
+        availability_info,
+        can_split_async,
+        should_trace,
+        should_merge_modules,
+        batching_config,
+    }: ChunkGroupContentOptions,
 ) -> Result<ChunkGroupContent> {
     let module_batches_graph = module_graph.module_batches(batching_config).await?;
 

turbopack/crates/turbopack-core/src/chunk/chunking_context.rs

@@ -339,6 +339,15 @@ pub trait ChunkingContext {
         Vc::cell(false)
     }
 
+    /// Whether async modules should create an new availability boundary and therefore nested async
+    /// modules include less modules. Enabling this will lead to better optimized async chunks,
+    /// but it will require to compute all possible paths in the application, which might lead to
+    /// many combinations.
+    #[turbo_tasks::function]
+    fn is_nested_async_availability_enabled(self: Vc<Self>) -> Vc<bool> {
+        Vc::cell(false)
+    }
+
     /// Whether to use `MergeableModule` to merge modules if possible.
     #[turbo_tasks::function]
     fn is_module_merging_enabled(self: Vc<Self>) -> Vc<bool> {

turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs

@@ -3,8 +3,8 @@ use indoc::formatdoc;
 use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
 use turbopack_core::{
     chunk::{
-        ChunkData, ChunkItem, ChunkType, ChunkingContext, ChunkingContextExt, ChunksData,
-        ModuleChunkItemIdExt,
+        AsyncModuleInfo, ChunkData, ChunkItem, ChunkType, ChunkingContext, ChunkingContextExt,
+        ChunksData, ModuleChunkItemIdExt,
     },
     ident::AssetIdent,
     module::Module,
@@ -149,6 +149,30 @@ impl EcmascriptChunkItem for AsyncLoaderChunkItem {
         }
         .cell())
     }
+
+    #[turbo_tasks::function]
+    fn content_with_async_module_info(
+        self: Vc<Self>,
+        _async_module_info: Option<Vc<AsyncModuleInfo>>,
+        estimated: bool,
+    ) -> Vc<EcmascriptChunkItemContent> {
+        if estimated {
+            let code = formatdoc! {
+                r#"
+                    {TURBOPACK_EXPORT_VALUE}((parentImport) => {{
+                        return Promise.all([].map((chunk) => {TURBOPACK_LOAD}(chunk))).then(() => {{}});
+                    }});
+                "#,
+            };
+            EcmascriptChunkItemContent {
+                inner_code: code.into(),
+                ..Default::default()
+            }
+            .cell()
+        } else {
+            self.content()
+        }
+    }
 }
 
 #[turbo_tasks::value_impl]
@@ -168,9 +192,26 @@ impl ChunkItem for AsyncLoaderChunkItem {
 
     #[turbo_tasks::function]
     async fn content_ident(self: Vc<Self>) -> Result<Vc<AssetIdent>> {
-        let ident = self.module().ident();
+        let mut ident = self.module().ident();
+
+        let this = self.await?;
+
+        let nested_async_availability = this
+            .chunking_context
+            .is_nested_async_availability_enabled()
+            .await?;
+
+        let availability_ident = if *nested_async_availability {
+            Some(self.chunks_data().hash().await?.to_string().into())
+        } else {
+            this.module.await?.availability_info.ident().await?
+        };
+
+        if let Some(availability_ident) = availability_ident {
+            ident = ident.with_modifier(availability_ident)
+        }
 
-        Ok(ident.with_modifier(self.chunks_data().hash().await?.to_string().into()))
+        Ok(ident)
     }
 
     #[turbo_tasks::function]

turbopack/crates/turbopack-ecmascript/src/chunk/chunk_type.rs

@@ -67,7 +67,7 @@ impl ChunkType for EcmascriptChunkType {
         };
         Ok(Vc::cell(
             chunk_item
-                .content_with_async_module_info(async_module_info)
+                .content_with_async_module_info(async_module_info, true)
                 .await
                 .map_or(0, |content| round_chunk_item_size(content.inner_code.len())),
         ))