From fc730d483890fd24ceacb56f3a42c0861e170346 Mon Sep 17 00:00:00 2001 From: kevmoo Date: Tue, 19 May 2026 17:55:20 -0700 Subject: [PATCH 1/3] [benchmark_harness] Add `blackhole` Give benchmark writers a place to put results that help avoid compiler optimizations that can skew results --- pkgs/benchmark_harness/CHANGELOG.md | 5 ++ .../lib/benchmark_harness.dart | 1 + .../lib/src/async_benchmark_base.dart | 3 + .../lib/src/benchmark_base.dart | 8 ++- pkgs/benchmark_harness/lib/src/blackhole.dart | 70 ++++++++++++++++++ pkgs/benchmark_harness/pubspec.yaml | 2 +- .../test/blackhole_test.dart | 72 +++++++++++++++++++ 7 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 pkgs/benchmark_harness/lib/src/blackhole.dart create mode 100644 pkgs/benchmark_harness/test/blackhole_test.dart diff --git a/pkgs/benchmark_harness/CHANGELOG.md b/pkgs/benchmark_harness/CHANGELOG.md index f9afe69ae5..ce73bc2bdb 100644 --- a/pkgs/benchmark_harness/CHANGELOG.md +++ b/pkgs/benchmark_harness/CHANGELOG.md @@ -1,3 +1,8 @@ +## 2.5.0-wip + +- Added a compiler-safe, zero-cost `blackhole` utility to protect + microbenchmarks from dead-code elimination (DCE) and tree-shaking. + ## 2.4.0 - Added a `bench` command. diff --git a/pkgs/benchmark_harness/lib/benchmark_harness.dart b/pkgs/benchmark_harness/lib/benchmark_harness.dart index b46a36fb47..1474ccd57b 100644 --- a/pkgs/benchmark_harness/lib/benchmark_harness.dart +++ b/pkgs/benchmark_harness/lib/benchmark_harness.dart @@ -4,4 +4,5 @@ export 'src/async_benchmark_base.dart'; export 'src/benchmark_base.dart' show BenchmarkBase; +export 'src/blackhole.dart' show blackhole; export 'src/score_emitter.dart'; diff --git a/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart b/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart index bd362021fa..5438243f2e 100644 --- a/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'blackhole.dart'; import 'score_emitter.dart'; class AsyncBenchmarkBase { @@ -39,6 +40,7 @@ class AsyncBenchmarkBase { Future Function() f, int minimumMillis, ) async { + Blackhole.preventDCE(); final minimumMicros = minimumMillis * 1000; final watch = Stopwatch()..start(); var iter = 0; @@ -54,6 +56,7 @@ class AsyncBenchmarkBase { /// Measures the score for the benchmark and returns it. Future measure() async { await setup(); + Blackhole.preventDCE(); try { // Warmup for at least 100ms. Discard result. await measureFor(warmup, 100); diff --git a/pkgs/benchmark_harness/lib/src/benchmark_base.dart b/pkgs/benchmark_harness/lib/src/benchmark_base.dart index d119d4da25..e0f0378637 100644 --- a/pkgs/benchmark_harness/lib/src/benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/benchmark_base.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'blackhole.dart'; import 'measurement.dart'; import 'score_emitter.dart'; @@ -42,12 +43,15 @@ class BenchmarkBase { /// Measures the score for this benchmark by executing it repeatedly until /// time minimum has been reached. - static double measureFor(void Function() f, int minimumMillis) => - measureForImpl(f, minimumMillis).score; + static double measureFor(void Function() f, int minimumMillis) { + Blackhole.preventDCE(); + return measureForImpl(f, minimumMillis).score; + } /// Measures the score for the benchmark and returns it. double measure() { setup(); + Blackhole.preventDCE(); // Warmup for at least 100ms. Discard result. measureForImpl(warmup, 100); // Run the benchmark for at least 2000ms. diff --git a/pkgs/benchmark_harness/lib/src/blackhole.dart b/pkgs/benchmark_harness/lib/src/blackhole.dart new file mode 100644 index 0000000000..a2dba2ee6e --- /dev/null +++ b/pkgs/benchmark_harness/lib/src/blackhole.dart @@ -0,0 +1,70 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// NOTE: Upgrading to Dart 3.12+ +// +// When the package's minimum supported SDK version is bumped to Dart 3.12 +// or higher (which fully supports `@pragma('external-effect')` natively), this +// class can be simplified to a pure zero-cost external function invocation: +// +// ```dart +// @pragma('vm:prefer-inline') +// @pragma('dart2js:prefer-inline') +// @pragma('wasm:prefer-inline') +// void blackhole(Object? value) { +// _BlackholeSink._reach(value); +// } +// +// class _BlackholeSink { +// @pragma('external-effect') +// external static void _reach(Object? value); +// } +// ``` +// +// Until then, we use a highly optimized static dynamic sink with an opaque +// read guard that is fully compatible with Dart 3.10 and 3.11. + +/// A compiler-recognized zero-cost live sink to prevent dead-code elimination. +/// +/// Passing a value to [blackhole] ensures that the compiler treats the value's +/// computation as live, preventing tree-shaking and dead-code elimination, +/// while introducing zero runtime execution overhead. +class Blackhole { + static dynamic _sink; + + /// A public static setter to allow the harness runner to implicitly consume + /// benchmark returned values in the timing loops. + static set sink(Object? value) => _sink = value; + + /// Consumes the given [value] to prevent dead-code elimination. + @pragma('vm:prefer-inline') + @pragma('dart2js:prefer-inline') + @pragma('wasm:prefer-inline') + void consume(Object? value) { + _sink = value; + } + + /// An opaque guard that convinces compiler static analyses (such as TFA) + /// that [_sink] is read, preventing it from being tree-shaken as write-only. + /// + /// Automatically invoked inside `BenchmarkRunner` initialization. + @pragma('vm:never-inline') + @pragma('dart2js:never-inline') + @pragma('wasm:never-inline') + static void preventDCE() { + // Opaque condition that is always false at runtime but unresolvable + // at compile-time. + if (int.tryParse('0') == 1) { + print(_sink); + } + } +} + +/// A zero-cost compiler-safe live sink to prevent dead-code elimination. +@pragma('vm:prefer-inline') +@pragma('dart2js:prefer-inline') +@pragma('wasm:prefer-inline') +void blackhole(Object? value) { + Blackhole._sink = value; +} diff --git a/pkgs/benchmark_harness/pubspec.yaml b/pkgs/benchmark_harness/pubspec.yaml index a23b14b763..b0177f8450 100644 --- a/pkgs/benchmark_harness/pubspec.yaml +++ b/pkgs/benchmark_harness/pubspec.yaml @@ -1,5 +1,5 @@ name: benchmark_harness -version: 2.4.0 +version: 2.5.0-wip description: The official Dart project benchmark harness. repository: https://github.com/dart-lang/tools/tree/main/pkgs/benchmark_harness issue_tracker: https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Abenchmark_harness diff --git a/pkgs/benchmark_harness/test/blackhole_test.dart b/pkgs/benchmark_harness/test/blackhole_test.dart new file mode 100644 index 0000000000..8553d15886 --- /dev/null +++ b/pkgs/benchmark_harness/test/blackhole_test.dart @@ -0,0 +1,72 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:benchmark_harness/benchmark_harness.dart'; +import 'package:benchmark_harness/src/blackhole.dart'; +import 'package:test/test.dart'; + +class SampleBenchmark extends BenchmarkBase { + int result = 0; + SampleBenchmark() : super('Sample'); + + @override + void run() { + result = 42 + 58; + blackhole(result); + } +} + +class SampleAsyncBenchmark extends AsyncBenchmarkBase { + int result = 0; + SampleAsyncBenchmark() : super('SampleAsync'); + + @override + Future run() async { + await Future.delayed(const Duration(microseconds: 1)); + result = 100; + blackhole(result); + } +} + +void main() { + group('blackhole shorthand', () { + test('executes successfully without throwing errors', () { + expect(() => blackhole('test-string'), returnsNormally); + expect(() => blackhole(null), returnsNormally); + expect(() => blackhole(42), returnsNormally); + }); + }); + + group('Blackhole class consume and preventDCE', () { + test('consume executes successfully without throwing errors', () { + final bh = Blackhole(); + expect(() => bh.consume('test-string'), returnsNormally); + expect(() => bh.consume(null), returnsNormally); + expect(() => bh.consume(42), returnsNormally); + }); + + test('preventDCE executes successfully without throwing errors', () { + expect(Blackhole.preventDCE, returnsNormally); + }); + }); + + group('Benchmark Integration', () { + test('sync benchmark using blackhole runs and measures successfully', () { + final benchmark = SampleBenchmark(); + final score = benchmark.measure(); + expect(score, isPositive); + expect(benchmark.result, equals(100)); + }); + + test( + 'async benchmark using blackhole runs and measures successfully', + () async { + final benchmark = SampleAsyncBenchmark(); + final score = await benchmark.measure(); + expect(score, isPositive); + expect(benchmark.result, equals(100)); + }, + ); + }); +} From f92580c4e130828c6cc924a255e80053922f1fcc Mon Sep 17 00:00:00 2001 From: kevmoo Date: Tue, 19 May 2026 17:58:41 -0700 Subject: [PATCH 2/3] document black hole a bit more --- pkgs/benchmark_harness/README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pkgs/benchmark_harness/README.md b/pkgs/benchmark_harness/README.md index 8ad873a3cf..00efcbf13a 100644 --- a/pkgs/benchmark_harness/README.md +++ b/pkgs/benchmark_harness/README.md @@ -23,6 +23,37 @@ to call `run()` once by overriding the `exercise` method: `AsyncBenchmarkBase` already reports the average time to call `run()` __once__. +## Preventing Dead-Code Elimination (DCE) + +Modern optimizing compilers (like the Dart VM, `dart2js`, and `dart2wasm`) are +highly effective at tree-shaking and optimizing away unused computations. If a +microbenchmark computes a value but never uses it, the compiler may eliminate +the entire computation path, leading to deceptively fast but inaccurate +timings (e.g., `0 us`). + +To defeat this, pass the computed value to the `blackhole` shorthand function. +This registers the computation's result as live and forces the compiler to +preserve the computation path, while introducing zero runtime execution +overhead. + +```dart +import 'package:benchmark_harness/benchmark_harness.dart'; + +class MyBenchmark extends BenchmarkBase { + MyBenchmark() : super('MyBenchmark'); + + @override + void run() { + // Without passing 'result' to blackhole(), the compiler could optimize + // away the entire heavyCalculation() call. + final result = heavyCalculation(); + blackhole(result); + } + + int heavyCalculation() => 42 + 58; +} +``` + ## Comparing Results If you are running the same benchmark, on the same machine, running the same OS, From 22041b24c380469dededd9d168a4a70a37564e4c Mon Sep 17 00:00:00 2001 From: kevmoo Date: Tue, 19 May 2026 18:13:29 -0700 Subject: [PATCH 3/3] agent review feedback --- .../lib/src/async_benchmark_base.dart | 25 +++++++++++-------- .../lib/src/benchmark_base.dart | 24 +++++++++++------- pkgs/benchmark_harness/lib/src/blackhole.dart | 16 ++---------- .../lib/src/perf_benchmark_base.dart | 2 ++ .../test/blackhole_test.dart | 9 +------ 5 files changed, 34 insertions(+), 42 deletions(-) diff --git a/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart b/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart index 5438243f2e..35a4bf32df 100644 --- a/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/async_benchmark_base.dart @@ -40,23 +40,25 @@ class AsyncBenchmarkBase { Future Function() f, int minimumMillis, ) async { - Blackhole.preventDCE(); - final minimumMicros = minimumMillis * 1000; - final watch = Stopwatch()..start(); - var iter = 0; - var elapsed = 0; - while (elapsed < minimumMicros) { - await f(); - elapsed = watch.elapsedMicroseconds; - iter++; + try { + final minimumMicros = minimumMillis * 1000; + final watch = Stopwatch()..start(); + var iter = 0; + var elapsed = 0; + while (elapsed < minimumMicros) { + await f(); + elapsed = watch.elapsedMicroseconds; + iter++; + } + return elapsed / iter; + } finally { + Blackhole.preventDCE(); } - return elapsed / iter; } /// Measures the score for the benchmark and returns it. Future measure() async { await setup(); - Blackhole.preventDCE(); try { // Warmup for at least 100ms. Discard result. await measureFor(warmup, 100); @@ -64,6 +66,7 @@ class AsyncBenchmarkBase { return await measureFor(exercise, 2000); } finally { await teardown(); + Blackhole.preventDCE(); } } diff --git a/pkgs/benchmark_harness/lib/src/benchmark_base.dart b/pkgs/benchmark_harness/lib/src/benchmark_base.dart index e0f0378637..a432c9fc9a 100644 --- a/pkgs/benchmark_harness/lib/src/benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/benchmark_base.dart @@ -44,20 +44,26 @@ class BenchmarkBase { /// Measures the score for this benchmark by executing it repeatedly until /// time minimum has been reached. static double measureFor(void Function() f, int minimumMillis) { - Blackhole.preventDCE(); - return measureForImpl(f, minimumMillis).score; + try { + return measureForImpl(f, minimumMillis).score; + } finally { + Blackhole.preventDCE(); + } } /// Measures the score for the benchmark and returns it. double measure() { setup(); - Blackhole.preventDCE(); - // Warmup for at least 100ms. Discard result. - measureForImpl(warmup, 100); - // Run the benchmark for at least 2000ms. - var result = measureForImpl(exercise, minimumMeasureDurationMillis); - teardown(); - return result.score; + try { + // Warmup for at least 100ms. Discard result. + measureForImpl(warmup, 100); + // Run the benchmark for at least 2000ms. + var result = measureForImpl(exercise, minimumMeasureDurationMillis); + return result.score; + } finally { + teardown(); + Blackhole.preventDCE(); + } } void report() { diff --git a/pkgs/benchmark_harness/lib/src/blackhole.dart b/pkgs/benchmark_harness/lib/src/blackhole.dart index a2dba2ee6e..715c2ca2fb 100644 --- a/pkgs/benchmark_harness/lib/src/blackhole.dart +++ b/pkgs/benchmark_harness/lib/src/blackhole.dart @@ -31,24 +31,12 @@ /// computation as live, preventing tree-shaking and dead-code elimination, /// while introducing zero runtime execution overhead. class Blackhole { - static dynamic _sink; - - /// A public static setter to allow the harness runner to implicitly consume - /// benchmark returned values in the timing loops. - static set sink(Object? value) => _sink = value; - - /// Consumes the given [value] to prevent dead-code elimination. - @pragma('vm:prefer-inline') - @pragma('dart2js:prefer-inline') - @pragma('wasm:prefer-inline') - void consume(Object? value) { - _sink = value; - } + static Object? _sink; /// An opaque guard that convinces compiler static analyses (such as TFA) /// that [_sink] is read, preventing it from being tree-shaken as write-only. /// - /// Automatically invoked inside `BenchmarkRunner` initialization. + /// Automatically invoked inside benchmark measurement loops. @pragma('vm:never-inline') @pragma('dart2js:never-inline') @pragma('wasm:never-inline') diff --git a/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart b/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart index 46f1b4f6b9..e1fc41f4ec 100644 --- a/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart +++ b/pkgs/benchmark_harness/lib/src/perf_benchmark_base.dart @@ -7,6 +7,7 @@ import 'dart:convert'; import 'dart:io'; import 'benchmark_base.dart'; +import 'blackhole.dart'; import 'measurement.dart'; import 'score_emitter.dart'; @@ -129,6 +130,7 @@ class PerfBenchmarkBase extends BenchmarkBase { } } finally { teardown(); + Blackhole.preventDCE(); } return result.score; } diff --git a/pkgs/benchmark_harness/test/blackhole_test.dart b/pkgs/benchmark_harness/test/blackhole_test.dart index 8553d15886..41b5b7eff3 100644 --- a/pkgs/benchmark_harness/test/blackhole_test.dart +++ b/pkgs/benchmark_harness/test/blackhole_test.dart @@ -38,14 +38,7 @@ void main() { }); }); - group('Blackhole class consume and preventDCE', () { - test('consume executes successfully without throwing errors', () { - final bh = Blackhole(); - expect(() => bh.consume('test-string'), returnsNormally); - expect(() => bh.consume(null), returnsNormally); - expect(() => bh.consume(42), returnsNormally); - }); - + group('Blackhole class preventDCE', () { test('preventDCE executes successfully without throwing errors', () { expect(Blackhole.preventDCE, returnsNormally); });