efortuna@google.com
4abe74e1e4
Review URL: http://codereview.chromium.org//8670013 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@1914 260f80e4-7a28-3924-810f-c04153c831b5
623 lines
24 KiB
Python
Executable File
623 lines
24 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
|
|
# for details. All rights reserved. Use of this source code is governed by a
|
|
# BSD-style license that can be found in the LICENSE file.
|
|
|
|
import datetime
|
|
import math
|
|
from matplotlib.font_manager import FontProperties
|
|
import matplotlib.pyplot as plt
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import traceback
|
|
|
|
"""This script runs to track performance and correctness progress of
|
|
different svn revisions. It tests to see if there a newer version of the code on
|
|
the server, and will sync and run the performance tests if so."""
|
|
|
|
DART_INSTALL_LOCATION = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
'..', '..', '..')
|
|
V8_MEAN = 'V8 Mean'
|
|
FROG_MEAN = 'frog Mean'
|
|
COMMAND_LINE = 'commandline'
|
|
V8 = 'v8'
|
|
FROG = 'frog'
|
|
V8_AND_FROG = [V8, FROG]
|
|
CORRECTNESS = 'Percent passing'
|
|
BENCHMARKS = ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody',
|
|
'BinaryTrees', 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci',
|
|
'Loop', 'Permute', 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum',
|
|
'Tak', 'Takl', 'Towers', 'TreeSort']
|
|
COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black']
|
|
GRAPH_OUT_DIR = 'graphs'
|
|
|
|
"""First, some utility methods."""
|
|
|
|
def RunCmd(string):
|
|
"""Run the specified command and print out any output to stdout.
|
|
Args:
|
|
string the command to run"""
|
|
p = subprocess.Popen(string, stdout = subprocess.PIPE,
|
|
stderr = subprocess.STDOUT, close_fds=True)
|
|
output, not_used = p.communicate();
|
|
print output:
|
|
return lines
|
|
|
|
def TimeCmd(self, cmd):
|
|
start = time.time()
|
|
RunCmd(cmd)
|
|
return time.time() - start
|
|
|
|
def SyncAndBuild(failed_once=False):
|
|
"""Make sure we have the latest version of of the repo, and build it. We
|
|
begin and end standing in DART_INSTALL_LOCATION.
|
|
Args:
|
|
failed_once True if we have attempted to build this once before, and we've
|
|
failed, indicating ."""
|
|
os.chdir(DART_INSTALL_LOCATION)
|
|
#Revert our newly built frogsh to prevent conflicts when we update
|
|
RunCmd('svn revert ' + os.path.join(os.getcwd(), 'frog', 'frogsh'))
|
|
|
|
RunCmd('gclient sync')
|
|
lines = RunCmd('%s -m release' % os.path.join('.', 'tools', 'build.py'))
|
|
os.chdir('frog')
|
|
lines += RunCmd('%s -m debug,release' % os.path.join('..', 'tools',
|
|
'build.py'))
|
|
os.chdir('..')
|
|
|
|
for line in lines:
|
|
if '** BUILD FAILED **' in lines:
|
|
if TestRunner.failed_once:
|
|
# Someone checked in a broken build! Just stop trying to make it work
|
|
# and wait for the next hour to try again.
|
|
print 'Broken Build'
|
|
sys.exit(0)
|
|
#Remove the xcode directory and attempt to build again. If it still
|
|
#fails, abort, and try again next hour.
|
|
out_dir = 'out'
|
|
if platform.system() == 'Darwin':
|
|
out_dir = 'xcodebuild'
|
|
os.removedirs(os.getcwd() + os.path.join('dart', out_dir, 'Release_ia32'))
|
|
os.removedirs(os.getcwd() + os.path.join('dart', 'frog', out_dir,
|
|
'Release_ia32'))
|
|
os.removedirs(os.getcwd() + os.path.join('dart', 'frog', out_dir,
|
|
'Debug_ia32'))
|
|
TestRunner.failed_once = True
|
|
SyncAndBuild()
|
|
|
|
def EnsureOutputDirectory(dir_name):
|
|
"""Test that the listed directory name exists, and if not, create one for
|
|
our output to be placed."""
|
|
dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
|
|
'perf_testing', dir_name)
|
|
if not os.path.exists(dir_path):
|
|
os.mkdir(dir_path)
|
|
print 'Creating output directory ', dir_path
|
|
|
|
def HasNewCode():
|
|
"""Tests if there are any newer versions of files on the server."""
|
|
os.chdir(DART_INSTALL_LOCATION)
|
|
results = RunCmd('svn st -u')
|
|
for line in results:
|
|
if '*' in line:
|
|
return True
|
|
return False
|
|
|
|
def GetBrowsers():
|
|
browsers = ['ff', 'chrome']
|
|
if platform.system() == 'Windows':
|
|
browsers += ['ie']
|
|
return browsers
|
|
|
|
class TestRunner(object):
|
|
"""The base clas to provide shared code for different tests we will run and
|
|
graph."""
|
|
|
|
def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list,
|
|
values_list):
|
|
"""Args:
|
|
result_folder_name the name of the folder where a tracefile of
|
|
performance results will be stored.
|
|
platform_list a list containing the platform(s) that our data has been
|
|
run on. (command line, firefox, chrome, etc)
|
|
v8_and_or_frog_list a list specifying whether we hold data about Frog
|
|
generated code, plain JS code (v8), or a combination of both.
|
|
values_list a list containing the type of data we will be graphing
|
|
(benchmarks, percentage passing, etc)"""
|
|
self.result_folder_name = result_folder_name
|
|
# cur_time is used as a timestamp of when this performance test was run.
|
|
self.cur_time = str(time.mktime(datetime.datetime.now().timetuple()))
|
|
self.browser_color = {'chrome': 'green', 'ie': 'blue', 'ff': 'red'}
|
|
self.values_list = values_list
|
|
self.platform_list = platform_list
|
|
self.revision_dict = dict()
|
|
self.values_dict = dict()
|
|
self.color_index = 0
|
|
for platform in platform_list:
|
|
self.revision_dict[platform] = dict()
|
|
self.values_dict[platform] = dict()
|
|
for f in v8_and_or_frog_list:
|
|
self.revision_dict[platform][f] = dict()
|
|
self.values_dict[platform][f] = dict()
|
|
for val in values_list:
|
|
self.revision_dict[platform][f][val] = []
|
|
self.values_dict[platform][f][val] = []
|
|
if V8 in v8_and_or_frog_list:
|
|
self.revision_dict[platform][V8][V8_MEAN] = []
|
|
self.values_dict[platform][V8][V8_MEAN] = []
|
|
if FROG in v8_and_or_frog_list:
|
|
self.revision_dict[platform][FROG][FROG_MEAN] = []
|
|
self.values_dict[platform][FROG][FROG_MEAN] = []
|
|
|
|
def GetColor(self):
|
|
color = COLORS[self.color_index]
|
|
self.color_index = (self.color_index + 1) % len(COLORS)
|
|
return color
|
|
|
|
def StyleAndSavePerfPlot(self, chart_title, y_axis_label, size_x, size_y,
|
|
legend_loc, filename, platform_list, v8_and_or_frog_list, values_list,
|
|
should_clear_axes=True):
|
|
"""Sets style preferences for chart boilerplate that is consistent across
|
|
all charts, and saves the chart as a png.
|
|
Args:
|
|
size_x the size of the printed chart, in inches, in the horizontal
|
|
direction
|
|
size_y the size of the printed chart, in inches in the vertical direction
|
|
legend_loc the location of the legend in on the chart. See suitable
|
|
arguments for the loc argument in matplotlib
|
|
filename the filename that we want to save the resulting chart as
|
|
platform_list a list containing the platform(s) that our data has been run
|
|
on. (command line, firefox, chrome, etc)
|
|
values_list a list containing the type of data we will be graphing
|
|
(performance, percentage passing, etc)
|
|
should_clear_axes True if we want to create a fresh graph, instead of
|
|
plotting additional lines on the current graph."""
|
|
if should_clear_axes:
|
|
plt.cla() # cla = clear current axes
|
|
for platform in platform_list:
|
|
for f in v8_and_or_frog_list:
|
|
for val in values_list:
|
|
plt.plot(self.revision_dict[platform][f][val],
|
|
self.values_dict[platform][f][val],
|
|
color=self.GetColor(), label='%s-%s-%s' % (platform, f, val))
|
|
|
|
plt.xlabel('Revision Number')
|
|
plt.ylabel(y_axis_label)
|
|
plt.title(chart_title)
|
|
fontP = FontProperties()
|
|
fontP.set_size('small')
|
|
plt.legend(loc=legend_loc, prop = fontP)
|
|
|
|
fig = plt.gcf()
|
|
fig.set_size_inches(size_x, size_y)
|
|
fig.savefig(os.path.join(GRAPH_OUT_DIR, filename))
|
|
|
|
def AddSvnRevisionToTrace(self, outfile):
|
|
"""Add the svn version number to the provided tracefile."""
|
|
p = subprocess.Popen('svn info ', stdout = subprocess.PIPE,
|
|
stderr = subprocess.STDOUT, close_fds=True)
|
|
output, not_used = p.communicate()
|
|
for line in output.split('\n'):
|
|
if 'Revision' in line:
|
|
RunCmd('echo "%s" > %s' % (line.strip(), outfile))
|
|
|
|
def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2,
|
|
cleanFile=False):
|
|
"""Adds an html table to the webpage to display the data values. This method
|
|
will be removed when we have a nicer way to display data values."""
|
|
#TODO(efortuna): fix this.
|
|
return
|
|
#TODO(efortuna): Take this method out when have finalized where the data is
|
|
# going to be displayed.
|
|
f = ''
|
|
out = ''
|
|
if cleanFile:
|
|
f = open('template.html')
|
|
else:
|
|
shutil.copy('index.html', 'temp.html')
|
|
f = open('temp.html')
|
|
out = open('index.html', 'w')
|
|
inTable = False
|
|
for line in f.readlines():
|
|
if not inTable:
|
|
out.write(line)
|
|
if delimiter in line:
|
|
inTable = not inTable
|
|
if inTable:
|
|
out.write('<table border="1"> <tr> <td> svn revision </td>')
|
|
for revision in rev_nums:
|
|
out.write('<td>%d</td>' % revision)
|
|
out.write('</tr>\n<tr><td> %s</td>' % label_1)
|
|
for perf in dict_1:
|
|
out.write('<td>%f</td>' % perf)
|
|
out.write('</tr>\n<tr><td> %s</td>' % label_2)
|
|
for perf in dict_2:
|
|
out.write('<td>%f</td>' % perf)
|
|
out.write('</tr> </table>')
|
|
|
|
def CalculateGeometricMean(self, platform, frog_or_v8, svn_revision):
|
|
"""Calculate the aggregate geometric mean for V8 and frog benchmark sets,
|
|
given two benchmark dictionaries."""
|
|
geo_mean = 0
|
|
for benchmark in BENCHMARKS:
|
|
geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][
|
|
len(self.values_dict[platform][frog_or_v8][benchmark]) - 1])
|
|
|
|
mean = V8_MEAN
|
|
if frog_or_v8 == FROG:
|
|
mean = FROG_MEAN
|
|
self.values_dict[platform][frog_or_v8][mean] += \
|
|
[math.pow(math.e, geo_mean / len(BENCHMARKS))]
|
|
self.revision_dict[platform][frog_or_v8][mean] += [svn_revision]
|
|
|
|
def Run(self):
|
|
"""Run the benchmarks/tests from the command line and plot the
|
|
results."""
|
|
plt.cla() # cla = clear current axes
|
|
os.chdir(DART_INSTALL_LOCATION)
|
|
EnsureOutputDirectory(self.result_folder_name)
|
|
EnsureOutputDirectory(GRAPH_OUT_DIR)
|
|
self.RunTests()
|
|
os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
|
|
|
|
# TODO(efortuna): You will want to make this only use a subset of the files
|
|
# eventually.
|
|
files = os.listdir(self.result_folder_name)
|
|
|
|
for afile in files:
|
|
if not afile.startswith('.'):
|
|
self.ProcessFile(afile)
|
|
self.PlotResults('%s.png' % self.result_folder_name)
|
|
|
|
class PerformanceTestRunner(TestRunner):
|
|
"""Super class for all performance testing."""
|
|
def __init__(self, result_folder_name, platform_list, platform_type):
|
|
super(PerformanceTestRunner, self).__init__(result_folder_name,
|
|
platform_list, V8_AND_FROG, BENCHMARKS)
|
|
self.platform_list = platform_list
|
|
self.platform_type = platform_type
|
|
|
|
def PlotAllPerf(self, png_filename):
|
|
"""Create a plot that shows the performance changes of individual benchmarks
|
|
run by V8 and generated by frog, over svn history."""
|
|
for benchmark in BENCHMARKS:
|
|
self.StyleAndSavePerfPlot(
|
|
'Performance of %s over time on the %s' % (benchmark,
|
|
self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left',
|
|
benchmark + png_filename, self.platform_list, V8_AND_FROG,
|
|
[benchmark])
|
|
|
|
def PlotAvgPerf(self, png_filename):
|
|
"""Generate a plot that shows the performance changes of the geomentric mean
|
|
of V8 and frog benchmark performance over svn history."""
|
|
(title, y_axis, size_x, size_y, loc, filename) = \
|
|
('Geometric Mean of benchmark %s performance' % self.platform_type,
|
|
'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename)
|
|
for platform in self.platform_list:
|
|
self.StyleAndSavePerfPlot(title, y_axis, size_x, size_y, loc, filename,
|
|
[platform], [V8], [V8_MEAN], True)
|
|
self.StyleAndSavePerfPlot(title, y_axis, size_x, size_y, loc, filename,
|
|
[platform], [FROG], [FROG_MEAN], False)
|
|
self.WriteHtml('table',
|
|
self.revision_dict[platform][V8],
|
|
'V8 mean', self.values_dict[platform][V8][V8_MEAN],
|
|
'Frog mean', self.values_dict[platform][FROG][FROG_MEAN],
|
|
True)
|
|
|
|
def PlotResults(self, png_filename):
|
|
self.PlotAllPerf(png_filename)
|
|
self.PlotAvgPerf('2' + png_filename)
|
|
|
|
class CommandLinePerformanceTestRunner(PerformanceTestRunner):
|
|
"""Run performance tests from the command line."""
|
|
|
|
def __init__(self, result_folder_name):
|
|
super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name,
|
|
[COMMAND_LINE], 'command line')
|
|
|
|
def ProcessFile(self, afile):
|
|
"""Pull all the relevant information out of a given tracefile.
|
|
|
|
Args:
|
|
afile is the filename string we will be processing."""
|
|
f = open(os.path.join(self.result_folder_name, afile))
|
|
tabulate_data = False
|
|
revision_num = 0
|
|
for line in f.readlines():
|
|
if 'Revision' in line:
|
|
revision_num = int(line.split()[1])
|
|
elif 'Benchmark' in line:
|
|
tabulate_data = True
|
|
elif tabulate_data:
|
|
tokens = line.split()
|
|
if len(tokens) < 4 or tokens[0] not in BENCHMARKS:
|
|
#Done tabulating data.
|
|
break
|
|
v8_value = float(tokens[1])
|
|
frog_value = float(tokens[3])
|
|
if v8_value == 0 or frog_value == 0:
|
|
#Then there was an error when this performance test was run. Do not
|
|
#count it in our numbers.
|
|
return
|
|
benchmark = tokens[0]
|
|
self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num]
|
|
self.values_dict[COMMAND_LINE][V8][benchmark] += [v8_value]
|
|
self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num]
|
|
self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value]
|
|
f.close()
|
|
|
|
self.CalculateGeometricMean(COMMAND_LINE, FROG, revision_num)
|
|
self.CalculateGeometricMean(COMMAND_LINE, V8, revision_num)
|
|
|
|
def RunTests(self):
|
|
"""Run a performance test on our updated system."""
|
|
os.chdir('frog')
|
|
file_path = os.path.join('..', 'tools', 'testing', 'perf_testing',
|
|
self.result_folder_name, 'result')
|
|
RunCmd('python %s > %s%s' % (os.path.join('benchmarks',
|
|
'perf_tests.py'), file_path, self.cur_time))
|
|
os.chdir('..')
|
|
|
|
|
|
class BrowserPerformanceTestRunner(PerformanceTestRunner):
|
|
"""Runs performance tests, in the browser."""
|
|
|
|
def __init__(self, result_folder_name):
|
|
super(BrowserPerformanceTestRunner, self).__init__(
|
|
result_folder_name, GetBrowsers(), 'browser')
|
|
|
|
def RunTests(self):
|
|
"""Run a performance test in the browser."""
|
|
os.chdir('frog')
|
|
RunCmd('python benchmarks/make_web_benchmarks.py')
|
|
os.chdir('..')
|
|
|
|
for browser in GetBrowsers():
|
|
for version in V8_AND_FROG:
|
|
self.AddSvnRevisionToTrace(os.path.join('tools', 'testing',
|
|
'perf_testing', self.result_folder_name,
|
|
'perf-%s-%s-%s' % (self.cur_time, browser, version)))
|
|
RunCmd('python %s --out %s --browser %s --timeout 1000 --perf >> %s' %
|
|
(os.path.join('tools', 'testing', 'run_selenium.py'),
|
|
os.path.join(os.getcwd(), 'internal', 'browserBenchmarks',
|
|
'benchmark_page_%s.html' % version),
|
|
browser,
|
|
os.path.join('tools', 'testing', 'perf_testing',
|
|
self.result_folder_name, 'perf-%s-%s-%s' % (self.cur_time, browser,
|
|
version))))
|
|
|
|
def ProcessFile(self, afile):
|
|
"""Comb through the html to find the performance results."""
|
|
parts = afile.split('-')
|
|
browser = parts[2]
|
|
version = parts[3]
|
|
f = open(os.path.join(self.result_folder_name, afile))
|
|
lines = f.readlines()
|
|
line = ''
|
|
i = 0
|
|
revision_num = 0
|
|
while '<div id="results">' not in line and i < len(lines):
|
|
if 'Revision' in line:
|
|
revision_num = int(line.split()[1])
|
|
line = lines[i]
|
|
i += 1
|
|
|
|
if i >= len(lines) or revision_num == 0:
|
|
# Then this run did not complete. Ignore this tracefile.
|
|
return
|
|
|
|
line = lines[i]
|
|
i += 1
|
|
results = []
|
|
if line.find('<br>') > -1:
|
|
results = line.split('<br>')
|
|
else:
|
|
results = line.split('<br />')
|
|
for result in results:
|
|
name_and_score = result.split(':')
|
|
if len(name_and_score) < 2:
|
|
break
|
|
name = name_and_score[0].strip()
|
|
score = name_and_score[1].strip()
|
|
if version == V8:
|
|
bench_dict = self.values_dict[browser][V8]
|
|
else:
|
|
bench_dict = self.values_dict[browser][FROG]
|
|
bench_dict[name] += [float(score)]
|
|
self.revision_dict[browser][version][name] += [revision_num]
|
|
|
|
f.close()
|
|
self.CalculateGeometricMean(browser, version, revision_num)
|
|
|
|
def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2,
|
|
cleanFile=False):
|
|
#TODO(efortuna)
|
|
pass
|
|
|
|
class BrowserCorrectnessTestRunner(TestRunner):
|
|
def __init__(self, test_type, result_folder_name):
|
|
super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name,
|
|
GetBrowsers(), [FROG], [CORRECTNESS])
|
|
self.test_type = test_type
|
|
|
|
def RunTests(self):
|
|
"""Run a test of the latest svn revision."""
|
|
for browser in GetBrowsers():
|
|
current_file = 'correctness%s-%s' % (self.cur_time, browser)
|
|
current_file_path = os.path.join('tools', 'testing',
|
|
'perf_testing', self.result_folder_name, current_file)
|
|
self.AddSvnRevisionToTrace(current_file_path)
|
|
RunCmd(os.path.join('.', 'tools', 'test.py') +
|
|
' --component=webdriver --flag=%s --report --timeout=20 ' % browser +
|
|
'--progress=color --mode=release -j1 %s >>' % self.test_type +
|
|
current_file_path)
|
|
|
|
def ProcessFile(self, afile):
|
|
"""Given a trace file, extract all the relevant information out of it to
|
|
determine the number of correctly passing tests.
|
|
|
|
Arguments:
|
|
afile the filename string"""
|
|
browser = afile.rpartition('-')[2]
|
|
f = open(os.path.join(self.result_folder_name, afile))
|
|
revision_num = 0
|
|
lines = f.readlines()
|
|
total_tests = 0
|
|
num_failed = 0
|
|
expect_fail = 0
|
|
for line in lines:
|
|
if 'Total:' in line:
|
|
total_tests = int(line.split()[1])
|
|
if 'will be skipped' in line:
|
|
total_tests -= int(line.split()[1])
|
|
if 'we should fix' in line:
|
|
expect_fail += int(line.split()[1])
|
|
if 'Revision' in line:
|
|
revision_num = int(line.split()[1])
|
|
if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line:
|
|
# (A printed out 'PASS' indicates we incorrectly passed a negative
|
|
# test.)
|
|
num_failed += 1
|
|
|
|
self.revision_dict[browser][FROG][CORRECTNESS] += [revision_num]
|
|
self.values_dict[browser][FROG][CORRECTNESS] += [100.0 *
|
|
(((float)(total_tests - (expect_fail + num_failed))) /total_tests)]
|
|
f.close()
|
|
|
|
def PlotResults(self, png_filename):
|
|
first_time = True
|
|
for browser in GetBrowsers():
|
|
self.StyleAndSavePerfPlot('Percentage of language tests passing in '
|
|
'different browsers', '% of tests passed', 8, 8, 'lower left',
|
|
png_filename, [browser], [FROG], [CORRECTNESS], first_time)
|
|
first_time = False
|
|
|
|
|
|
class CompileTimeAndSizeTestRunner(TestRunner):
|
|
"""Run tests to determine how long frogsh takes to compile, and the compiled
|
|
file output size of some benchmarking files."""
|
|
def __init__(self, result_folder_name):
|
|
super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name,
|
|
[COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping',
|
|
'frogsh', 'swarm', 'total'])
|
|
self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5,
|
|
'frogsh' : 100, 'swarm' : 100, 'total' : 100}
|
|
|
|
def RunTests(self):
|
|
os.chdir('frog')
|
|
current_file_path = os.path.join('..', 'tools', 'testing', 'perf_testing',
|
|
self.result_folder_name, self.result_folder_name + self.cur_time)
|
|
|
|
self.AddSvnRevisionToTrace(current_file_path)
|
|
|
|
elapsed = TimeCmd(os.path.join('.', 'frog.py') +
|
|
' --vm_flags="--compile_all --enable_type_checks --enable_asserts" --'
|
|
' --compile_all --enable_type_checks --out=frogsh frog.dart')
|
|
RunCmd('echo "%f Compiling on Dart VM in checked mode in' % elapsed +
|
|
' seconds" >> %s' % current_file_path)
|
|
RunCmd('chmod +x frogsh')
|
|
elapsed = TimeCmd(os.path.join('.', 'frogsh') + ' --out=frogsh '
|
|
'--enable_type_checks frog.dart --enable_type_checks ' +
|
|
os.path.join('tests', 'hello.dart'))
|
|
if elapsed < self.failure_threshold['Bootstrapping']:
|
|
#frogsh didn't compile correctly. Stop testing now, because subsequent
|
|
#numbers will be meaningless.
|
|
return
|
|
size = os.path.getsize('frogsh')
|
|
RunCmd('echo "%f Bootstrapping time in seconds in checked mode"' %
|
|
elapsed + ' >> %s' % current_file_path)
|
|
RunCmd('echo "%d Generated checked frogsh size "' % size +
|
|
' >> %s' % current_file_path)
|
|
|
|
RunCmd(os.path.join('.', 'frogsh') + ' --out=swarm-result '
|
|
'--compile-only ' + os.path.join('..', 'client', 'samples', 'swarm',
|
|
'swarm.dart'))
|
|
swarm_size = 0
|
|
try:
|
|
swarm_size = os.path.getsize('swarm-result')
|
|
except OSError:
|
|
pass #If compilation failed, continue on running other tests.
|
|
|
|
RunCmd(os.path.join('.', 'frogsh') + ' --out=total-result '
|
|
'--compile-only ' + os.path.join('..', 'client', 'samples', 'total',
|
|
'src', 'Total.dart'))
|
|
total_size = 0
|
|
try:
|
|
total_size = os.path.getsize('total-result')
|
|
except OSError:
|
|
pass #If compilation failed, continue on running other tests.
|
|
|
|
RunCmd('echo "%d Generated checked swarm size "' % swarm_size +
|
|
' >> %s' % current_file_path)
|
|
|
|
RunCmd('echo "%d Generated checked total size "' % total_size +
|
|
' >> %s' % current_file_path)
|
|
os.chdir('..')
|
|
|
|
def ProcessFile(self, afile):
|
|
"""Pull all the relevant information out of a given tracefile.
|
|
Args:
|
|
afile is the filename string we will be processing."""
|
|
f = open(os.path.join(self.result_folder_name, afile))
|
|
tabulate_data = False
|
|
revision_num = 0
|
|
for line in f.readlines():
|
|
tokens = line.split()
|
|
if 'Revision' in line:
|
|
revision_num = int(line.split()[1])
|
|
else:
|
|
for metric in self.values_list:
|
|
if metric in line:
|
|
num = tokens[0]
|
|
if num.find('.') == -1:
|
|
num = int(num)
|
|
else:
|
|
num = float(num)
|
|
self.values_dict[COMMAND_LINE][FROG][metric] += [num]
|
|
self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
|
|
|
|
if revision_num != 0:
|
|
for metric in self.values_list:
|
|
self.revision_dict[COMMAND_LINE][FROG][metric].pop()
|
|
self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
|
|
# Fill 0 if compilation failed.
|
|
if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \
|
|
self.failure_threshold[metric]:
|
|
self.values_dict[COMMAND_LINE][FROG][metric] += [0]
|
|
self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
|
|
|
|
f.close()
|
|
|
|
def PlotResults(self, png_filename):
|
|
self.StyleAndSavePerfPlot('Compiled frogsh Sizes',
|
|
'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE],
|
|
[FROG], ['swarm', 'total', 'frogsh'])
|
|
self.WriteHtml('bar', self.revision_dict[COMMAND_LINE][FROG]['frogsh'],
|
|
'frogsh size', self.values_dict[COMMAND_LINE][FROG]['frogsh'], '', [])
|
|
|
|
self.StyleAndSavePerfPlot('Time to compile and bootstrap',
|
|
'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG],
|
|
['Bootstrapping', 'Compiling on Dart VM'])
|
|
self.WriteHtml('baz',
|
|
self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'],
|
|
'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'],
|
|
'Compiling on Dart VM',
|
|
self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM'])
|
|
|
|
|
|
def main():
|
|
if HasNewCode():
|
|
SyncAndBuild()
|
|
CommandLinePerformanceTestRunner('cl-results').Run()
|
|
CompileTimeAndSizeTestRunner('code-time-size').Run()
|
|
BrowserCorrectnessTestRunner('language', 'browser-correctness').Run()
|
|
BrowserPerformanceTestRunner('browser-perf').Run()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
|