// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'dart:async'; import 'dart:io'; import 'dart:math'; import 'package:path/path.dart'; import '../../integration_test/support/integration_test_methods.dart'; import '../../integration_test/support/integration_tests.dart'; /// Instances of the class [TimingResult] represent the timing information /// gathered while executing a given timing test. class TimingResult { /// The number of nanoseconds in a millisecond. static int NANOSECONDS_PER_MILLISECOND = 1000000; /// The amount of time spent executing each test, in nanoseconds. List times; /// Initialize a newly created timing result. new(this.times); /// The average amount of time spent executing a single iteration, in /// milliseconds. int get averageTime { return totalTime ~/ times.length; } /// The maximum amount of time spent executing a single iteration, in /// milliseconds. int get maxTime { var maxTime = 0; var count = times.length; for (var i = 0; i < count; i++) { maxTime = max(maxTime, times[i]); } return maxTime ~/ NANOSECONDS_PER_MILLISECOND; } /// The minimum amount of time spent executing a single iteration, in /// milliseconds. int get minTime { var minTime = times[0]; var count = times.length; for (var i = 1; i < count; i++) { minTime = min(minTime, times[i]); } return minTime ~/ NANOSECONDS_PER_MILLISECOND; } /// The standard deviation of the times. double get standardDeviation { return computeStandardDeviation(toMilliseconds(times)); } /// The total amount of time spent executing the test, in milliseconds. int get totalTime { var totalTime = 0; var count = times.length; for (var i = 0; i < count; i++) { totalTime += times[i]; } return totalTime ~/ NANOSECONDS_PER_MILLISECOND; } /// Compute the standard deviation of the given set of [values]. double computeStandardDeviation(List values) { var count = values.length; var sumOfValues = 0; for (var i = 0; i < count; i++) { sumOfValues += values[i]; } var average = sumOfValues / count; var sumOfDiffSquared = 0.0; for (var i = 0; i < count; i++) { var diff = values[i] - average; sumOfDiffSquared += diff * diff; } return sqrt(sumOfDiffSquared / (count - 1)); } /// Convert the given [times], expressed in nanoseconds, to times expressed in /// milliseconds. List toMilliseconds(List times) { var count = times.length; var convertedValues = []; for (var i = 0; i < count; i++) { convertedValues.add(times[i] ~/ NANOSECONDS_PER_MILLISECOND); } return convertedValues; } } /// The abstract class [TimingTest] defines the behavior of objects that measure /// the time required to perform some sequence of server operations. abstract class TimingTest extends IntegrationTest { /// The number of times the test will be performed in order to warm up the VM. static const int _defaultWarmupCount = 10; /// The number of times the test will be performed in order to compute a time. static const int _defaultTimingCount = 10; /// The amount of time to give the server to respond to a shutdown request /// before forcibly terminating it. static const Duration _shutdownTimeout = Duration(seconds: 5); /// The connection to the analysis server. @override late Server server; /// The temporary directory in which source files can be stored. late Directory sourceDirectory; /// A flag indicating whether the teardown process should skip sending a /// "server.shutdown" request because the server is known to have already /// shutdown. bool skipShutdown = false; /// Return the number of iterations that should be performed in order to /// compute a time. int get timingCount => _defaultTimingCount; /// Return the number of iterations that should be performed in order to warm /// up the VM. int get warmupCount => _defaultWarmupCount; /// Perform any operations that need to be performed once before any /// iterations. Future oneTimeSetUp() { server = Server(); sourceDirectory = Directory.systemTemp.createTempSync( 'analysisServer_test_timing', ); var serverConnected = Completer(); onServerConnected.listen((_) { serverConnected.complete(); }); skipShutdown = true; var dartSdkPath = dirname(dirname(Platform.resolvedExecutable)); return server.start(dartSdkPath: dartSdkPath).then((_) { server.listenToOutput(dispatchNotification); server.exitCode.then((_) { skipShutdown = true; }); return serverConnected.future; }); } /// Perform any operations that need to be performed once after all /// iterations. Future oneTimeTearDown() { return _shutdownIfNeeded().then((_) { sourceDirectory.deleteSync(recursive: true); }); } /// Perform any operations that part of a single iteration. It is the /// execution of this method that will be measured. Future perform(); /// Return a future that will complete with a timing result representing the /// number of milliseconds required to perform the operation the specified /// number of times. Future run() async { var times = []; await oneTimeSetUp(); await _repeat(warmupCount, null); await _repeat(timingCount, times); await oneTimeTearDown(); return TimingResult(times); } /// Perform any operations that need to be performed before each iteration. Future setUp(); /// Convert the given [relativePath] to an absolute path, by interpreting it /// relative to [sourceDirectory]. On Windows any forward slashes in /// [relativePath] are converted to backslashes. String sourcePath(String relativePath) { return join(sourceDirectory.path, relativePath.replaceAll('/', separator)); } /// Perform any operations that need to be performed after each iteration. Future tearDown(); /// Write a source file with the given absolute [pathname] and [contents]. /// /// If the file didn't previously exist, it is created. If it did, it is /// overwritten. /// /// Parent directories are created as necessary. void writeFile(String pathname, String contents) { Directory(dirname(pathname)).createSync(recursive: true); File(pathname).writeAsStringSync(contents); } /// Return the number of nanoseconds that have elapsed since the given /// [stopwatch] was last stopped. int _elapsedNanoseconds(Stopwatch stopwatch) { return (stopwatch.elapsedTicks * 1000000000) ~/ stopwatch.frequency; } /// Repeatedly execute this test [count] times, adding timing information to /// the given list of [times] if it is non-`null`. Future _repeat(int count, List? times) { var stopwatch = Stopwatch(); return setUp().then((_) { stopwatch.start(); return perform().then((_) { stopwatch.stop(); if (times != null) { times.add(_elapsedNanoseconds(stopwatch)); } return tearDown().then((_) { if (count > 0) { return _repeat(count - 1, times); } else { return Future.value(); } }); }); }); } /// Shut the server down unless [skipShutdown] is `true`. Future _shutdownIfNeeded() { if (skipShutdown) { return Future.value(); } // Give the server a short time to comply with the shutdown request; if it // doesn't exit, then forcibly terminate it. sendServerShutdown(); return server.exitCode.timeout( _shutdownTimeout, onTimeout: () { return server.kill('server failed to exit'); }, ); } }