Improve type length check algorithm

Author: naganohara-yoshino

compiler/rustc_monomorphize/src/collector.rs

@@ -357,6 +357,7 @@ fn collect_items_root<'tcx>(
     starting_item: Spanned<MonoItem<'tcx>>,
     state: &SharedState<'tcx>,
     recursion_limit: Limit,
+    type_length_limit: Limit,
 ) {
     if !state.visited.lock().insert(starting_item.node) {
         // We've been here already, no need to search again.
@@ -370,6 +371,7 @@ fn collect_items_root<'tcx>(
         &mut recursion_depths,
         recursion_limit,
         CollectionMode::UsedItems,
+        type_length_limit,
     );
 }
 
@@ -378,14 +380,18 @@ fn collect_items_root<'tcx>(
 ///
 /// `mode` determined whether we are scanning for [used items][CollectionMode::UsedItems]
 /// or [mentioned items][CollectionMode::MentionedItems].
-#[instrument(skip(tcx, state, recursion_depths, recursion_limit), level = "debug")]
+#[instrument(
+    skip(tcx, state, recursion_depths, recursion_limit, type_length_limit),
+    level = "debug"
+)]
 fn collect_items_rec<'tcx>(
     tcx: TyCtxt<'tcx>,
     starting_item: Spanned<MonoItem<'tcx>>,
     state: &SharedState<'tcx>,
     recursion_depths: &mut DefIdMap<usize>,
     recursion_limit: Limit,
     mode: CollectionMode,
+    type_length_limit: Limit,
 ) {
     let mut used_items = MonoItems::new();
     let mut mentioned_items = MonoItems::new();
@@ -463,13 +469,14 @@ fn collect_items_rec<'tcx>(
             // Sanity check whether this ended up being collected accidentally
             debug_assert!(tcx.should_codegen_locally(instance));
 
-            // Keep track of the monomorphization recursion depth
+            // Check for recursive monomorphization before collecting uses
             recursion_depth_reset = Some(check_recursion_limit(
                 tcx,
                 instance,
                 starting_item.span,
                 recursion_depths,
                 recursion_limit,
+                type_length_limit,
             ));
 
             rustc_data_structures::stack::ensure_sufficient_stack(|| {
@@ -595,6 +602,7 @@ fn collect_items_rec<'tcx>(
                 recursion_depths,
                 recursion_limit,
                 CollectionMode::UsedItems,
+                type_length_limit,
             );
         }
     }
@@ -609,6 +617,7 @@ fn collect_items_rec<'tcx>(
             recursion_depths,
             recursion_limit,
             CollectionMode::MentionedItems,
+            type_length_limit,
         );
     }
 
@@ -652,6 +661,7 @@ fn check_recursion_limit<'tcx>(
     span: Span,
     recursion_depths: &mut DefIdMap<usize>,
     recursion_limit: Limit,
+    type_length_limit: Limit,
 ) -> (DefId, usize) {
     let def_id = instance.def_id();
     let recursion_depth = recursion_depths.get(&def_id).cloned().unwrap_or(0);
@@ -665,17 +675,17 @@ fn check_recursion_limit<'tcx>(
         recursion_depth
     };
 
-    // Rust code can create exponentially-long types using only a
-    // polynomial recursion depth. Start checking the type length before
-    // the depth limit is reached, to avoid hanging on enormous instance
-    // arguments.
-    let type_length_check_depth = (recursion_limit / 8).0.max(4);
+    // Recursive monomorphization can grow instance args exponentially with polynomial
+    // recursion depth. Start checking type lengths around `ilog2(type_length_limit.0)`
+    // to avoid hanging on enormous instance arguments.
+    let type_length_check_depth = type_length_limit.0.checked_ilog2().unwrap_or(0).max(4) as usize;
     let recursive_type_growth_limit_reached = recursion_depth >= type_length_check_depth
-        && !tcx.type_length_limit().value_within_limit(type_length(instance.args));
+        && !type_length_limit.value_within_limit(type_length(instance.args));
 
-    // Code that needs to instantiate the same function recursively
-    // more than the recursion limit, or with type arguments that exceed the
-    // type length limit, is assumed to be causing an infinite expansion.
+    // Code that needs to instantiate the same function recursively more
+    // than the recursion limit is assumed to be causing an infinite
+    // expansion. Bail out earlier if recursive instantiations have already
+    // produced instance args exceeding the type length limit.
     if !recursion_limit.value_within_limit(adjusted_recursion_depth)
         || recursive_type_growth_limit_reached
     {
@@ -1848,9 +1858,17 @@ pub(crate) fn collect_crate_mono_items<'tcx>(
     };
     let recursion_limit = tcx.recursion_limit();
 
+    let type_length_limit = tcx.type_length_limit();
+
     tcx.sess.time("monomorphization_collector_graph_walk", || {
         par_for_each_in(roots, |root| {
-            collect_items_root(tcx, dummy_spanned(*root), &state, recursion_limit);
+            collect_items_root(
+                tcx,
+                dummy_spanned(*root),
+                &state,
+                recursion_limit,
+                type_length_limit,
+            );
         });
     });
 

tests/ui/codegen/overflow-during-mono.rs

@@ -1,8 +1,6 @@
 //@ build-fail
 //@ compile-flags: -Zwrite-long-types-to-disk=yes
 
-//~? ERROR reached the recursion limit while instantiating
-
 #![recursion_limit = "32"]
 
 fn quicksort<It: Clone + Iterator<Item = T>, I: IntoIterator<IntoIter = It>, T: Ord>(
@@ -16,6 +14,7 @@ fn quicksort<It: Clone + Iterator<Item = T>, I: IntoIterator<IntoIter = It>, T:
             let greater = i.filter(|y| &x <= y);
 
             let mut v = quicksort(less);
+            //~^ ERROR reached the recursion limit while instantiating
             let u = quicksort(greater);
             v.push(x);
             v.extend(u);

tests/ui/codegen/overflow-during-mono.stderr

@@ -1,8 +1,16 @@
-error: reached the recursion limit while instantiating `<Filter<Filter<_, _>, _> as Iterator>::try_fold::<(), _, _>`
-  --> $SRC_DIR/core/src/iter/adapters/filter.rs:LL:COL
+error: reached the recursion limit while instantiating `quicksort::<Filter<Filter<Filter<Filter<_, _>, _>, _>, _>, _, i32>`
+  --> $DIR/overflow-during-mono.rs:16:25
    |
-note: `<Filter<I, P> as Iterator>::try_fold` defined here
-  --> $SRC_DIR/core/src/iter/adapters/filter.rs:LL:COL
+LL |             let mut v = quicksort(less);
+   |                         ^^^^^^^^^^^^^^^
+   |
+note: `quicksort` defined here
+  --> $DIR/overflow-during-mono.rs:6:1
+   |
+LL | / fn quicksort<It: Clone + Iterator<Item = T>, I: IntoIterator<IntoIter = It>, T: Ord>(
+LL | |     i: I,
+LL | | ) -> Vec<T> {
+   | |___________^
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/overflow-during-mono.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 

tests/ui/recursion/infinite-instantiation-via-nested-closures.rs

@@ -38,7 +38,6 @@ impl C {
     pub fn matches<F: Fn()>(&self, f: &F) {
         let &C(ref base) = self;
         base.matches(&|| C(base.clone()).matches(f))
-        //~^ ERROR reached the recursion limit while instantiating
     }
 }
 
@@ -49,6 +48,7 @@ impl D {
     pub fn matches<F: Fn()>(&self, f: &F) {
         let &D(ref a) = self;
         a.matches(f)
+        //~^ ERROR reached the recursion limit while instantiating
     }
 }
 

tests/ui/recursion/infinite-instantiation-via-nested-closures.stderr

@@ -1,11 +1,11 @@
-error: reached the recursion limit while instantiating `D::matches::<{closure@$DIR/infinite-instantiation-via-nested-closures.rs:40:23: 40:25}>`
-  --> $DIR/infinite-instantiation-via-nested-closures.rs:40:9
+error: reached the recursion limit while instantiating `A::matches::<{closure@$DIR/infinite-instantiation-via-nested-closures.rs:40:23: 40:25}>`
+  --> $DIR/infinite-instantiation-via-nested-closures.rs:50:9
    |
-LL |         base.matches(&|| C(base.clone()).matches(f))
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |         a.matches(f)
+   |         ^^^^^^^^^^^^
    |
-note: `D::matches` defined here
-  --> $DIR/infinite-instantiation-via-nested-closures.rs:49:5
+note: `A::matches` defined here
+  --> $DIR/infinite-instantiation-via-nested-closures.rs:13:5
    |
 LL |     pub fn matches<F: Fn()>(&self, f: &F) {
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^