Turbopack: avoid creating another async chunk loader where there is already one available

turbopack/crates/turbopack-core/src/chunk/available_modules.rs

@@ -1,12 +1,67 @@
 use anyhow::Result;
-use turbo_tasks::{FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
+use serde::{Deserialize, Serialize};
+use turbo_tasks::{
+    FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, ValueToString, Vc,
+    trace::TraceRawVcs,
+};
 use turbo_tasks_hash::Xxh3Hash64Hasher;
 
-use crate::module_graph::module_batch::{ChunkableModuleOrBatch, IdentStrings};
+use crate::{
+    chunk::ChunkableModule,
+    module::Module,
+    module_graph::module_batch::{ChunkableModuleOrBatch, IdentStrings, ModuleBatch},
+};
+
+#[derive(
+    Debug,
+    Copy,
+    Clone,
+    Hash,
+    PartialEq,
+    Eq,
+    Serialize,
+    Deserialize,
+    TraceRawVcs,
+    NonLocalValue,
+    TaskInput,
+)]
+pub enum AvailableModuleItem {
+    Module(ResolvedVc<Box<dyn ChunkableModule>>),
+    Batch(ResolvedVc<ModuleBatch>),
+    AsyncLoader(ResolvedVc<Box<dyn ChunkableModule>>),
+}
+
+impl AvailableModuleItem {
+    pub async fn ident_strings(&self) -> Result<IdentStrings> {
+        Ok(match self {
+            AvailableModuleItem::Module(module) => {
+                IdentStrings::Single(module.ident().to_string().owned().await?)
+            }
+            AvailableModuleItem::Batch(batch) => {
+                IdentStrings::Multiple(batch.ident_strings().await?)
+            }
+            AvailableModuleItem::AsyncLoader(module) => IdentStrings::Single(
+                format!("async loader {}", module.ident().to_string().await?).into(),
+            ),
+        })
+    }
+}
+
+impl From<ChunkableModuleOrBatch> for AvailableModuleItem {
+    fn from(value: ChunkableModuleOrBatch) -> Self {
+        match value {
+            ChunkableModuleOrBatch::Module(module) => AvailableModuleItem::Module(module),
+            ChunkableModuleOrBatch::Batch(batch) => AvailableModuleItem::Batch(batch),
+            ChunkableModuleOrBatch::None(id) => {
+                panic!("Cannot create AvailableModuleItem from None({})", id)
+            }
+        }
+    }
+}
 
 #[turbo_tasks::value(transparent)]
 #[derive(Debug, Clone)]
-pub struct AvailableModulesSet(FxIndexSet<ChunkableModuleOrBatch>);
+pub struct AvailableModulesSet(FxIndexSet<AvailableModuleItem>);
 
 /// Allows to gather information about which assets are already available.
 /// Adding more roots will form a linked list like structure to allow caching
@@ -52,7 +107,7 @@ impl AvailableModules {
             .modules
             .await?
             .iter()
-            .map(|&module| module.ident_strings())
+            .map(async |&module| module.ident_strings().await)
             .try_join()
             .await?;
         for idents in item_idents {
@@ -70,12 +125,12 @@ impl AvailableModules {
     }
 
     #[turbo_tasks::function]
-    pub async fn get(&self, module_or_batch: ChunkableModuleOrBatch) -> Result<Vc<bool>> {
-        if self.modules.await?.contains(&module_or_batch) {
+    pub async fn get(&self, item: AvailableModuleItem) -> Result<Vc<bool>> {
+        if self.modules.await?.contains(&item) {
             return Ok(Vc::cell(true));
         };
         if let Some(parent) = self.parent {
-            return Ok(parent.get(module_or_batch));
+            return Ok(parent.get(item));
         }
         Ok(Vc::cell(false))
     }
@@ -101,11 +156,7 @@ pub struct AvailableModulesSnapshot {
 }
 
 impl AvailableModulesSnapshot {
-    pub fn get(&self, module_or_batch: ChunkableModuleOrBatch) -> bool {
-        self.modules.contains(&module_or_batch)
-            || self
-                .parent
-                .as_ref()
-                .is_some_and(|parent| parent.get(module_or_batch))
+    pub fn get(&self, item: AvailableModuleItem) -> bool {
+        self.modules.contains(&item) || self.parent.as_ref().is_some_and(|parent| parent.get(item))
     }
 }

turbopack/crates/turbopack-core/src/chunk/chunk_group.rs

@@ -12,6 +12,7 @@ use super::{
 use crate::{
     chunk::{
         ChunkableModule, ChunkingType,
+        available_modules::AvailableModuleItem,
         chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
     },
     environment::ChunkLoading,
@@ -251,6 +252,9 @@ pub async fn chunk_group_content(
         entries,
         &mut state,
         |parent_info, &node, state| {
+            if matches!(node, ModuleOrBatch::None(_)) {
+                return Ok(GraphTraversalAction::Continue);
+            }
             // Traced modules need to have a special handling
             if let Some((
                 _,
@@ -275,7 +279,7 @@ pub async fn chunk_group_content(
 
             let is_available = available_modules
                 .as_ref()
-                .is_some_and(|available_modules| available_modules.get(chunkable_node));
+                .is_some_and(|available_modules| available_modules.get(chunkable_node.into()));
 
             let Some((_, edge)) = parent_info else {
                 // An entry from the entries list
@@ -310,7 +314,14 @@ pub async fn chunk_group_content(
                     if can_split_async {
                         let chunkable_module = ResolvedVc::try_downcast(edge.module.unwrap())
                             .context("Module in async chunking edge is not chunkable")?;
-                        state.async_modules.insert(chunkable_module);
+                        let is_async_loader_available =
+                            available_modules.as_ref().is_some_and(|available_modules| {
+                                available_modules
+                                    .get(AvailableModuleItem::AsyncLoader(chunkable_module))
+                            });
+                        if !is_async_loader_available {
+                            state.async_modules.insert(chunkable_module);
+                        }
                         GraphTraversalAction::Exclude
                     } else if is_available {
                         GraphTraversalAction::Exclude
@@ -343,8 +354,21 @@ pub async fn chunk_group_content(
     )?;
 
     // This needs to use the unmerged items
+    let available_modules = state
+        .chunkable_items
+        .iter()
+        .copied()
+        .map(Into::into)
+        .chain(
+            state
+                .async_modules
+                .iter()
+                .copied()
+                .map(AvailableModuleItem::AsyncLoader),
+        )
+        .collect();
     let availability_info = availability_info
-        .with_modules(Vc::cell(state.chunkable_items.clone()))
+        .with_modules(Vc::cell(available_modules))
         .await?;
 
     let should_merge_modules = if should_merge_modules {

turbopack/crates/turbopack-core/src/module_graph/module_batch.rs

@@ -35,7 +35,7 @@ impl ModuleOrBatch {
     pub async fn ident_strings(self) -> Result<IdentStrings> {
         Ok(match self {
             ModuleOrBatch::Module(module) => {
-                IdentStrings::Single(module.ident().to_string().await?)
+                IdentStrings::Single(module.ident().to_string().owned().await?)
             }
             ModuleOrBatch::Batch(batch) => IdentStrings::Multiple(batch.ident_strings().await?),
             ModuleOrBatch::None(_) => IdentStrings::None,
@@ -74,7 +74,7 @@ impl ChunkableModuleOrBatch {
     pub async fn ident_strings(self) -> Result<IdentStrings> {
         Ok(match self {
             ChunkableModuleOrBatch::Module(module) => {
-                IdentStrings::Single(module.ident().to_string().await?)
+                IdentStrings::Single(module.ident().to_string().owned().await?)
             }
             ChunkableModuleOrBatch::Batch(batch) => {
                 IdentStrings::Multiple(batch.ident_strings().await?)
@@ -96,7 +96,7 @@ impl From<ChunkableModuleOrBatch> for ModuleOrBatch {
 
 pub enum IdentStrings {
     None,
-    Single(ReadRef<RcStr>),
+    Single(RcStr),
     Multiple(ReadRef<Vec<RcStr>>),
 }
 

turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs

@@ -1,6 +1,7 @@
 use anyhow::Result;
 use indoc::formatdoc;
-use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
+use tracing::Instrument;
+use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueToString, Vc};
 use turbopack_core::{
     chunk::{
         AsyncModuleInfo, ChunkData, ChunkItem, ChunkType, ChunkingContext, ChunkingContextExt,
@@ -45,7 +46,7 @@ impl AsyncLoaderChunkItem {
             let module_or_batch = batches.get_entry(inner_module).await?;
             if let Some(chunkable_module_or_batch) =
                 ChunkableModuleOrBatch::from_module_or_batch(module_or_batch)
-                && *chunk_items.get(chunkable_module_or_batch).await?
+                && *chunk_items.get(chunkable_module_or_batch.into()).await?
             {
                 return Ok(OutputAssetsWithReferenced {
                     assets: ResolvedVc::cell(vec![]),
@@ -66,10 +67,18 @@ impl AsyncLoaderChunkItem {
     #[turbo_tasks::function]
     async fn chunks_data(self: Vc<Self>) -> Result<Vc<ChunksData>> {
         let this = self.await?;
-        Ok(ChunkData::from_assets(
-            this.chunking_context.output_root().owned().await?,
-            *self.chunk_group().await?.assets,
-        ))
+        let span = tracing::info_span!(
+            "compute async chunks",
+            name = this.module.ident().to_string().await?.as_str()
+        );
+        async move {
+            Ok(ChunkData::from_assets(
+                this.chunking_context.output_root().owned().await?,
+                *self.chunk_group().await?.assets,
+            ))
+        }
+        .instrument(span)
+        .await
     }
 }
 

turbopack/crates/turbopack-ecmascript/src/manifest/chunk_asset.rs

@@ -78,7 +78,7 @@ impl ManifestAsyncModule {
             let module_or_batch = batches.get_entry(inner_module).await?;
             if let Some(chunkable_module_or_batch) =
                 ChunkableModuleOrBatch::from_module_or_batch(module_or_batch)
-                && *chunk_items.get(chunkable_module_or_batch).await?
+                && *chunk_items.get(chunkable_module_or_batch.into()).await?
             {
                 return Ok(OutputAssetsWithReferenced {
                     assets: ResolvedVc::cell(vec![]),

turbopack/crates/turbopack-nodejs/src/chunking_context.rs

@@ -75,6 +75,11 @@ impl NodeJsChunkingContextBuilder {
         self
     }
 
+    pub fn nested_async_availability(mut self, enable_nested_async_availability: bool) -> Self {
+        self.chunking_context.enable_nested_async_availability = enable_nested_async_availability;
+        self
+    }
+
     pub fn module_merging(mut self, enable_module_merging: bool) -> Self {
         self.chunking_context.enable_module_merging = enable_module_merging;
         self
@@ -168,6 +173,8 @@ pub struct NodeJsChunkingContext {
     runtime_type: RuntimeType,
     /// Enable tracing for this chunking
     enable_file_tracing: bool,
+    /// Enable nested async availability for this chunking
+    enable_nested_async_availability: bool,
     /// Enable module merging
     enable_module_merging: bool,
     /// Enable dynamic chunk content loading.
@@ -215,6 +222,7 @@ impl NodeJsChunkingContext {
                 asset_prefix: None,
                 asset_prefixes: Default::default(),
                 enable_file_tracing: false,
+                enable_nested_async_availability: false,
                 enable_module_merging: false,
                 enable_dynamic_chunk_content_loading: false,
                 environment,
@@ -306,6 +314,11 @@ impl ChunkingContext for NodeJsChunkingContext {
         Vc::cell(self.enable_file_tracing)
     }
 
+    #[turbo_tasks::function]
+    fn is_nested_async_availability_enabled(&self) -> Vc<bool> {
+        Vc::cell(self.enable_nested_async_availability)
+    }
+
     #[turbo_tasks::function]
     fn is_module_merging_enabled(&self) -> Vc<bool> {
         Vc::cell(self.enable_module_merging)

turbopack/crates/turbopack-tests/tests/execution.rs

@@ -243,6 +243,8 @@ struct TestOptions {
     scope_hoisting: Option<bool>,
     #[serde(default)]
     minify: bool,
+    #[serde(default)]
+    production_chunking: bool,
 }
 
 fn default_tree_shaking_mode() -> Option<TreeShakingMode> {
@@ -256,6 +258,7 @@ impl Default for TestOptions {
             remove_unused_exports: None,
             scope_hoisting: None,
             minify: false,
+            production_chunking: false,
         }
     }
 }
@@ -468,7 +471,7 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
 
     let module_graph = ModuleGraph::from_modules(entries.graph_entries(), false);
 
-    let chunking_context = NodeJsChunkingContext::builder(
+    let mut builder = NodeJsChunkingContext::builder(
         project_root.clone(),
         chunk_root_path.clone(),
         chunk_root_path_in_root_path_offset,
@@ -478,20 +481,6 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
         env,
         RuntimeType::Development,
     )
-    .chunking_config(
-        Vc::<EcmascriptChunkType>::default().to_resolved().await?,
-        ChunkingConfig {
-            min_chunk_size: 10_000,
-            ..Default::default()
-        },
-    )
-    .chunking_config(
-        Vc::<CssChunkType>::default().to_resolved().await?,
-        ChunkingConfig {
-            max_merge_chunk_size: 100_000,
-            ..Default::default()
-        },
-    )
     .module_merging(options.scope_hoisting.unwrap_or(true))
     .minify_type(if options.minify {
         MinifyType::Minify {
@@ -508,8 +497,28 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
         )
     } else {
         None
-    })
-    .build();
+    });
+    if options.production_chunking {
+        builder = builder
+            .chunking_config(
+                Vc::<EcmascriptChunkType>::default().to_resolved().await?,
+                ChunkingConfig {
+                    min_chunk_size: 2_000,
+                    max_chunk_count_per_group: 40,
+                    max_merge_chunk_size: 200_000,
+                    ..Default::default()
+                },
+            )
+            .chunking_config(
+                Vc::<CssChunkType>::default().to_resolved().await?,
+                ChunkingConfig {
+                    max_merge_chunk_size: 100_000,
+                    ..Default::default()
+                },
+            )
+            .nested_async_availability(true);
+    }
+    let chunking_context = builder.build();
 
     let res = evaluate(
         entries,

turbopack/crates/turbopack-tests/tests/execution/turbopack/chunking/combinations/input/a0.js

@@ -0,0 +1,14 @@
+async function all() {
+  await import('./b0.js')
+  await import('./b1.js')
+  await import('./b2.js')
+  await import('./b3.js')
+  await import('./b4.js')
+  await import('./b5.js')
+  await import('./b6.js')
+  await import('./b7.js')
+  await import('./b8.js')
+  await import('./b9.js')
+}
+
+await all()