[benchmark_harness] Add blackhole

pkgs/benchmark_harness/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.5.0-wip
+
+- Added a compiler-safe, zero-cost `blackhole` utility to protect
+  microbenchmarks from dead-code elimination (DCE) and tree-shaking.
+
 ## 2.4.0
 
 - Added a `bench` command.

pkgs/benchmark_harness/README.md

@@ -23,6 +23,37 @@ to call `run()` once by overriding the `exercise` method:
 
 `AsyncBenchmarkBase` already reports the average time to call `run()` __once__.
 
+## Preventing Dead-Code Elimination (DCE)
+
+Modern optimizing compilers (like the Dart VM, `dart2js`, and `dart2wasm`) are
+highly effective at tree-shaking and optimizing away unused computations. If a
+microbenchmark computes a value but never uses it, the compiler may eliminate
+the entire computation path, leading to deceptively fast but inaccurate
+timings (e.g., `0 us`).
+
+To defeat this, pass the computed value to the `blackhole` shorthand function.
+This registers the computation's result as live and forces the compiler to
+preserve the computation path, while introducing zero runtime execution
+overhead.
+
+```dart
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+class MyBenchmark extends BenchmarkBase {
+  MyBenchmark() : super('MyBenchmark');
+
+  @override
+  void run() {
+    // Without passing 'result' to blackhole(), the compiler could optimize
+    // away the entire heavyCalculation() call.
+    final result = heavyCalculation();
+    blackhole(result);
+  }
+
+  int heavyCalculation() => 42 + 58;
+}
+```
+
 ## Comparing Results
 
 If you are running the same benchmark, on the same machine, running the same OS,

pkgs/benchmark_harness/lib/benchmark_harness.dart

@@ -4,4 +4,5 @@
 
 export 'src/async_benchmark_base.dart';
 export 'src/benchmark_base.dart' show BenchmarkBase;
+export 'src/blackhole.dart' show blackhole;
 export 'src/score_emitter.dart';

pkgs/benchmark_harness/lib/src/async_benchmark_base.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'blackhole.dart';
 import 'score_emitter.dart';
 
 class AsyncBenchmarkBase {
@@ -39,16 +40,20 @@ class AsyncBenchmarkBase {
     Future<void> Function() f,
     int minimumMillis,
   ) async {
-    final minimumMicros = minimumMillis * 1000;
-    final watch = Stopwatch()..start();
-    var iter = 0;
-    var elapsed = 0;
-    while (elapsed < minimumMicros) {
-      await f();
-      elapsed = watch.elapsedMicroseconds;
-      iter++;
+    try {
+      final minimumMicros = minimumMillis * 1000;
+      final watch = Stopwatch()..start();
+      var iter = 0;
+      var elapsed = 0;
+      while (elapsed < minimumMicros) {
+        await f();
+        elapsed = watch.elapsedMicroseconds;
+        iter++;
+      }
+      return elapsed / iter;
+    } finally {
+      Blackhole.preventDCE();
     }
-    return elapsed / iter;
   }
 
   /// Measures the score for the benchmark and returns it.
@@ -61,6 +66,7 @@ class AsyncBenchmarkBase {
       return await measureFor(exercise, 2000);
     } finally {
       await teardown();
+      Blackhole.preventDCE();
     }
   }
 

pkgs/benchmark_harness/lib/src/benchmark_base.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'blackhole.dart';
 import 'measurement.dart';
 import 'score_emitter.dart';
 
@@ -42,18 +43,27 @@ class BenchmarkBase {
 
   /// Measures the score for this benchmark by executing it repeatedly until
   /// time minimum has been reached.
-  static double measureFor(void Function() f, int minimumMillis) =>
-      measureForImpl(f, minimumMillis).score;
+  static double measureFor(void Function() f, int minimumMillis) {
+    try {
+      return measureForImpl(f, minimumMillis).score;
+    } finally {
+      Blackhole.preventDCE();
+    }
+  }
 
   /// Measures the score for the benchmark and returns it.
   double measure() {
     setup();
-    // Warmup for at least 100ms. Discard result.
-    measureForImpl(warmup, 100);
-    // Run the benchmark for at least 2000ms.
-    var result = measureForImpl(exercise, minimumMeasureDurationMillis);
-    teardown();
-    return result.score;
+    try {
+      // Warmup for at least 100ms. Discard result.
+      measureForImpl(warmup, 100);
+      // Run the benchmark for at least 2000ms.
+      var result = measureForImpl(exercise, minimumMeasureDurationMillis);
+      return result.score;
+    } finally {
+      teardown();
+      Blackhole.preventDCE();
+    }
   }
 
   void report() {

pkgs/benchmark_harness/lib/src/blackhole.dart

@@ -0,0 +1,58 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// NOTE: Upgrading to Dart 3.12+
+//
+// When the package's minimum supported SDK version is bumped to Dart 3.12
+// or higher (which fully supports `@pragma('external-effect')` natively), this
+// class can be simplified to a pure zero-cost external function invocation:
+//
+// ```dart
+// @pragma('vm:prefer-inline')
+// @pragma('dart2js:prefer-inline')
+// @pragma('wasm:prefer-inline')
+// void blackhole(Object? value) {
+//   _BlackholeSink._reach(value);
+// }
+//
+// class _BlackholeSink {
+//   @pragma('external-effect')
+//   external static void _reach(Object? value);
+// }
+// ```
+//
+// Until then, we use a highly optimized static dynamic sink with an opaque
+// read guard that is fully compatible with Dart 3.10 and 3.11.
+
+/// A compiler-recognized zero-cost live sink to prevent dead-code elimination.
+///
+/// Passing a value to [blackhole] ensures that the compiler treats the value's
+/// computation as live, preventing tree-shaking and dead-code elimination,
+/// while introducing zero runtime execution overhead.
+class Blackhole {
+  static Object? _sink;
+
+  /// An opaque guard that convinces compiler static analyses (such as TFA)
+  /// that [_sink] is read, preventing it from being tree-shaken as write-only.
+  ///
+  /// Automatically invoked inside benchmark measurement loops.
+  @pragma('vm:never-inline')
+  @pragma('dart2js:never-inline')
+  @pragma('wasm:never-inline')
+  static void preventDCE() {
+    // Opaque condition that is always false at runtime but unresolvable
+    // at compile-time.
+    if (int.tryParse('0') == 1) {
+      print(_sink);
+    }
+  }
+}
+
+/// A zero-cost compiler-safe live sink to prevent dead-code elimination.
+@pragma('vm:prefer-inline')
+@pragma('dart2js:prefer-inline')
+@pragma('wasm:prefer-inline')
+void blackhole(Object? value) {
+  Blackhole._sink = value;
+}

pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart

@@ -7,6 +7,7 @@ import 'dart:convert';
 import 'dart:io';
 
 import 'benchmark_base.dart';
+import 'blackhole.dart';
 import 'measurement.dart';
 import 'score_emitter.dart';
 
@@ -129,6 +130,7 @@ class PerfBenchmarkBase extends BenchmarkBase {
       }
     } finally {
       teardown();
+      Blackhole.preventDCE();
     }
     return result.score;
   }

pkgs/benchmark_harness/pubspec.yaml

@@ -1,5 +1,5 @@
 name: benchmark_harness
-version: 2.4.0
+version: 2.5.0-wip
 description: The official Dart project benchmark harness.
 repository: https://github.com/dart-lang/tools/tree/main/pkgs/benchmark_harness
 issue_tracker: https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Abenchmark_harness

pkgs/benchmark_harness/test/blackhole_test.dart

@@ -0,0 +1,65 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:benchmark_harness/src/blackhole.dart';
+import 'package:test/test.dart';
+
+class SampleBenchmark extends BenchmarkBase {
+  int result = 0;
+  SampleBenchmark() : super('Sample');
+
+  @override
+  void run() {
+    result = 42 + 58;
+    blackhole(result);
+  }
+}
+
+class SampleAsyncBenchmark extends AsyncBenchmarkBase {
+  int result = 0;
+  SampleAsyncBenchmark() : super('SampleAsync');
+
+  @override
+  Future<void> run() async {
+    await Future<void>.delayed(const Duration(microseconds: 1));
+    result = 100;
+    blackhole(result);
+  }
+}
+
+void main() {
+  group('blackhole shorthand', () {
+    test('executes successfully without throwing errors', () {
+      expect(() => blackhole('test-string'), returnsNormally);
+      expect(() => blackhole(null), returnsNormally);
+      expect(() => blackhole(42), returnsNormally);
+    });
+  });
+
+  group('Blackhole class preventDCE', () {
+    test('preventDCE executes successfully without throwing errors', () {
+      expect(Blackhole.preventDCE, returnsNormally);
+    });
+  });
+
+  group('Benchmark Integration', () {
+    test('sync benchmark using blackhole runs and measures successfully', () {
+      final benchmark = SampleBenchmark();
+      final score = benchmark.measure();
+      expect(score, isPositive);
+      expect(benchmark.result, equals(100));
+    });
+
+    test(
+      'async benchmark using blackhole runs and measures successfully',
+      () async {
+        final benchmark = SampleAsyncBenchmark();
+        final score = await benchmark.measure();
+        expect(score, isPositive);
+        expect(benchmark.result, equals(100));
+      },
+    );
+  });
+}