From b6aa2976dca9523c28120731533c8bbd191c90ee Mon Sep 17 00:00:00 2001 From: Jonas Termansen Date: Thu, 22 Feb 2024 14:38:19 +0000 Subject: [PATCH] [benchmarks] Fix benchmarks warming up incorrectly. The benchmarks were using a range of antipatterns that did not do what the authors thought they did. It seems that the authors thought the warmup method has to run for a while and do the full warmup, but the truth is that the harness will do that for you by running the warmup function in a timed loop. Instead these patterns just wasted time by making the warmup more expensive and complex than needed. This change just removes the warmup overrides since none of them do anything positive. This change prepares us for future improvements to the benchmark harness. Fixes: b/324874055 Change-Id: Ib7c282123a2151614bc95a105a30e67221f11315 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/352022 Reviewed-by: William Hesse Commit-Queue: Jonas Termansen --- DEPS | 2 +- benchmarks/Iterators/dart/Iterators.dart | 2 +- benchmarks/LongStringCompare/dart/LongStringCompare.dart | 7 ------- benchmarks/MD5/dart/md5.dart | 7 ------- benchmarks/MD5/dart2/md5.dart | 7 ------- benchmarks/SHA1/dart/sha1.dart | 7 ------- benchmarks/SHA1/dart2/sha1.dart | 7 ------- benchmarks/SHA256/dart/sha256.dart | 7 ------- benchmarks/SHA256/dart2/sha256.dart | 7 ------- .../TypedDataDuplicate/dart/TypedDataDuplicate.dart | 7 ------- .../TypedDataDuplicate/dart2/TypedDataDuplicate.dart | 7 ------- benchmarks/Utf8Decode/dart/Utf8Decode.dart | 9 +++------ benchmarks/Utf8Decode/dart2/Utf8Decode.dart | 9 +++------ benchmarks/Utf8DecodeComplex/dart/benchmarks.dart | 5 ----- benchmarks/Utf8Encode/dart/Utf8Encode.dart | 5 ----- benchmarks/Utf8Encode/dart2/Utf8Encode.dart | 5 ----- 16 files changed, 8 insertions(+), 92 deletions(-) diff --git a/DEPS b/DEPS index fe437414e72..008863ea46e 100644 --- a/DEPS +++ b/DEPS @@ -57,7 +57,7 @@ vars = { "co19_rev": "867d139b3169fc131488e893ec1133dc98cc3aa0", # The internal benchmarks to use. See go/dart-benchmarks-internal - "benchmarks_internal_rev": "3e58fcfa1464ffd7b1d3edc7a8d5d9a24a321c5d", + "benchmarks_internal_rev": "175974d2b43dae39c380fc5f7f583a377cd3c11d", "checkout_benchmarks_internal": False, # Checkout the flute benchmark only when benchmarking. diff --git a/benchmarks/Iterators/dart/Iterators.dart b/benchmarks/Iterators/dart/Iterators.dart index 3b9cfc97aff..83621da50c5 100644 --- a/benchmarks/Iterators/dart/Iterators.dart +++ b/benchmarks/Iterators/dart/Iterators.dart @@ -508,7 +508,7 @@ void pollute() { /// /// Run benchmarks with name containing all the dot-separated words in the /// selector, so `--Set.const` will run benchmark -/// 'Iterators.const.Set.int.N`, and `--2.UpTo` will select +/// `Iterators.const.Set.int.N`, and `--2.UpTo` will select /// `Iterators.UpTo.2`. Each selector is matched independently, and if /// selectors are used, only benchmarks matching some selector are run. /// diff --git a/benchmarks/LongStringCompare/dart/LongStringCompare.dart b/benchmarks/LongStringCompare/dart/LongStringCompare.dart index b6845858d4b..6d5aff26713 100644 --- a/benchmarks/LongStringCompare/dart/LongStringCompare.dart +++ b/benchmarks/LongStringCompare/dart/LongStringCompare.dart @@ -26,13 +26,6 @@ class LongStringCompare extends BenchmarkBase { s.add(single + '!' + single); } - @override - void warmup() { - for (int i = 0; i < reps / 2; i++) { - run(); - } - } - @override void run() { for (int i = 0; i < reps; i++) { diff --git a/benchmarks/MD5/dart/md5.dart b/benchmarks/MD5/dart/md5.dart index 30dd6d435c3..718534fb9d0 100644 --- a/benchmarks/MD5/dart/md5.dart +++ b/benchmarks/MD5/dart/md5.dart @@ -16,13 +16,6 @@ class MD5Bench extends BenchmarkBase { : data = List.generate(size, (i) => i % 256, growable: false), super('MD5'); - @override - void warmup() { - for (int i = 0; i < 4; i++) { - run(); - } - } - @override void run() { final hash = md5.convert(data); diff --git a/benchmarks/MD5/dart2/md5.dart b/benchmarks/MD5/dart2/md5.dart index e73b901c62d..8ecd0095cc2 100644 --- a/benchmarks/MD5/dart2/md5.dart +++ b/benchmarks/MD5/dart2/md5.dart @@ -21,13 +21,6 @@ class MD5Bench extends BenchmarkBase { } } - @override - void warmup() { - for (int i = 0; i < 4; i++) { - run(); - } - } - @override void run() { final hash = md5.convert(data); diff --git a/benchmarks/SHA1/dart/sha1.dart b/benchmarks/SHA1/dart/sha1.dart index 713cb87e9d9..1b9f7a0339b 100644 --- a/benchmarks/SHA1/dart/sha1.dart +++ b/benchmarks/SHA1/dart/sha1.dart @@ -16,13 +16,6 @@ class SHA1Bench extends BenchmarkBase { : data = List.generate(size, (i) => i % 256, growable: false), super('SHA1'); - @override - void warmup() { - for (int i = 0; i < 4; i++) { - run(); - } - } - @override void run() { final hash = sha1.convert(data); diff --git a/benchmarks/SHA1/dart2/sha1.dart b/benchmarks/SHA1/dart2/sha1.dart index 0b7128faf78..24ffef63712 100644 --- a/benchmarks/SHA1/dart2/sha1.dart +++ b/benchmarks/SHA1/dart2/sha1.dart @@ -21,13 +21,6 @@ class SHA1Bench extends BenchmarkBase { } } - @override - void warmup() { - for (int i = 0; i < 4; i++) { - run(); - } - } - @override void run() { final hash = sha1.convert(data); diff --git a/benchmarks/SHA256/dart/sha256.dart b/benchmarks/SHA256/dart/sha256.dart index 83ae27d9860..cb580b34b0c 100644 --- a/benchmarks/SHA256/dart/sha256.dart +++ b/benchmarks/SHA256/dart/sha256.dart @@ -17,13 +17,6 @@ class SHA256Bench extends BenchmarkBase { : data = List.generate(size, (i) => i % 256, growable: false), super('SHA256'); - @override - void warmup() { - for (int i = 0; i < 4; i++) { - run(); - } - } - @override void run() { final hash = sha256.convert(data); diff --git a/benchmarks/SHA256/dart2/sha256.dart b/benchmarks/SHA256/dart2/sha256.dart index e381dc0cf82..9be21a94bd7 100644 --- a/benchmarks/SHA256/dart2/sha256.dart +++ b/benchmarks/SHA256/dart2/sha256.dart @@ -22,13 +22,6 @@ class SHA256Bench extends BenchmarkBase { } } - @override - void warmup() { - for (int i = 0; i < 4; i++) { - run(); - } - } - @override void run() { final hash = sha256.convert(data); diff --git a/benchmarks/TypedDataDuplicate/dart/TypedDataDuplicate.dart b/benchmarks/TypedDataDuplicate/dart/TypedDataDuplicate.dart index c21c575f0a7..f5086f8633d 100644 --- a/benchmarks/TypedDataDuplicate/dart/TypedDataDuplicate.dart +++ b/benchmarks/TypedDataDuplicate/dart/TypedDataDuplicate.dart @@ -24,13 +24,6 @@ abstract class Uint8ListCopyBenchmark extends BenchmarkBase { } } - @override - void warmup() { - for (var i = 0; i < 100; ++i) { - run(); - } - } - @override void teardown() { for (var i = 0; i < size; ++i) { diff --git a/benchmarks/TypedDataDuplicate/dart2/TypedDataDuplicate.dart b/benchmarks/TypedDataDuplicate/dart2/TypedDataDuplicate.dart index 6fc20687888..022a04eb09d 100644 --- a/benchmarks/TypedDataDuplicate/dart2/TypedDataDuplicate.dart +++ b/benchmarks/TypedDataDuplicate/dart2/TypedDataDuplicate.dart @@ -26,13 +26,6 @@ abstract class Uint8ListCopyBenchmark extends BenchmarkBase { } } - @override - void warmup() { - for (var i = 0; i < 100; ++i) { - run(); - } - } - @override void teardown() { for (var i = 0; i < size; ++i) { diff --git a/benchmarks/Utf8Decode/dart/Utf8Decode.dart b/benchmarks/Utf8Decode/dart/Utf8Decode.dart index e37eadf00a1..5224bbc4413 100644 --- a/benchmarks/Utf8Decode/dart/Utf8Decode.dart +++ b/benchmarks/Utf8Decode/dart/Utf8Decode.dart @@ -29,7 +29,9 @@ class Utf8Decode extends BenchmarkBase { String name = 'Utf8Decode.$language.'; name += size >= 1000000 ? '${size ~/ 1000000}M' - : size >= 1000 ? '${size ~/ 1000}k' : '$size'; + : size >= 1000 + ? '${size ~/ 1000}k' + : '$size'; if (allowMalformed) name += '.malformed'; return name; } @@ -93,11 +95,6 @@ class Utf8Decode extends BenchmarkBase { run(); } - @override - void warmup() { - BenchmarkBase.measureFor(run, 1000); - } - @override double measure() { // Report time per input byte. diff --git a/benchmarks/Utf8Decode/dart2/Utf8Decode.dart b/benchmarks/Utf8Decode/dart2/Utf8Decode.dart index 98dd260f769..4c8ad9ede62 100644 --- a/benchmarks/Utf8Decode/dart2/Utf8Decode.dart +++ b/benchmarks/Utf8Decode/dart2/Utf8Decode.dart @@ -31,7 +31,9 @@ class Utf8Decode extends BenchmarkBase { String name = 'Utf8Decode.$language.'; name += size >= 1000000 ? '${size ~/ 1000000}M' - : size >= 1000 ? '${size ~/ 1000}k' : '$size'; + : size >= 1000 + ? '${size ~/ 1000}k' + : '$size'; if (allowMalformed) name += '.malformed'; return name; } @@ -95,11 +97,6 @@ class Utf8Decode extends BenchmarkBase { run(); } - @override - void warmup() { - BenchmarkBase.measureFor(run, 1000); - } - @override double measure() { // Report time per input byte. diff --git a/benchmarks/Utf8DecodeComplex/dart/benchmarks.dart b/benchmarks/Utf8DecodeComplex/dart/benchmarks.dart index b1bd61279a8..90c0150aed3 100644 --- a/benchmarks/Utf8DecodeComplex/dart/benchmarks.dart +++ b/benchmarks/Utf8DecodeComplex/dart/benchmarks.dart @@ -164,11 +164,6 @@ class Utf8DecodeBenchmarkBase extends BenchmarkBase { run(); } - @override - void warmup() { - BenchmarkBase.measureFor(run, 100); - } - @override double measure() { // Report time per input byte. diff --git a/benchmarks/Utf8Encode/dart/Utf8Encode.dart b/benchmarks/Utf8Encode/dart/Utf8Encode.dart index 5d50bf8a263..9686a9356ec 100644 --- a/benchmarks/Utf8Encode/dart/Utf8Encode.dart +++ b/benchmarks/Utf8Encode/dart/Utf8Encode.dart @@ -66,11 +66,6 @@ class Utf8Encode extends BenchmarkBase { run(); } - @override - void warmup() { - BenchmarkBase.measureFor(run, 1000); - } - @override double measure() { // Report time per input rune. diff --git a/benchmarks/Utf8Encode/dart2/Utf8Encode.dart b/benchmarks/Utf8Encode/dart2/Utf8Encode.dart index 422756fe28e..61851ffa32e 100644 --- a/benchmarks/Utf8Encode/dart2/Utf8Encode.dart +++ b/benchmarks/Utf8Encode/dart2/Utf8Encode.dart @@ -69,11 +69,6 @@ class Utf8Encode extends BenchmarkBase { run(); } - @override - void warmup() { - BenchmarkBase.measureFor(run, 1000); - } - @override double measure() { // Report time per input rune.