sigmund@google.com
396f798124
Review URL: http://codereview.chromium.org//8999030 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@2792 260f80e4-7a28-3924-810f-c04153c831b5
606 lines
19 KiB
Python
Executable File
606 lines
19 KiB
Python
Executable File
# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
|
|
# for details. All rights reserved. Use of this source code is governed by a
|
|
# BSD-style license that can be found in the LICENSE file.
|
|
#
|
|
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
import utils
|
|
|
|
OS_GUESS = utils.GuessOS()
|
|
|
|
HTML_CONTENTS = """
|
|
<html>
|
|
<head>
|
|
<title> Test %(title)s </title>
|
|
<style>
|
|
.unittest-table { font-family:monospace; border:1px; }
|
|
.unittest-pass { background: #6b3;}
|
|
.unittest-fail { background: #d55;}
|
|
.unittest-error { background: #a11;}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1> Running %(title)s </h1>
|
|
<script type="text/javascript" src="%(controller_script)s"></script>
|
|
<script type="text/javascript">
|
|
// If nobody intercepts the error, finish the test.
|
|
onerror = function() { window.layoutTestController.notifyDone() };
|
|
|
|
document.onreadystatechange = function() {
|
|
if (document.readyState != "loaded") return;
|
|
// If 'startedDartTest' is not set, that means that the test did not have
|
|
// a chance to load. This will happen when a load error occurs in the VM.
|
|
// Give the machine time to start up.
|
|
setTimeout(function() {
|
|
// A window.postMessage might have been enqueued after this timeout.
|
|
// Just sleep another time to give the browser the time to process the
|
|
// posted message.
|
|
setTimeout(function() {
|
|
if (layoutTestController && !layoutTestController.startedDartTest) {
|
|
layoutTestController.notifyDone();
|
|
}
|
|
}, 0);
|
|
}, 50);
|
|
};
|
|
</script>
|
|
<script type="%(script_type)s" src="%(source_script)s"></script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
DART_TEST_AS_LIBRARY = """
|
|
#library('test');
|
|
#source('%(test)s');
|
|
"""
|
|
|
|
DART_CONTENTS = """
|
|
#library('test');
|
|
|
|
#import('%(dom_library)s');
|
|
#import('%(test_framework)s');
|
|
|
|
#import('%(library)s', prefix: "Test");
|
|
|
|
waitForDone() {
|
|
window.postMessage('unittest-suite-wait-for-done', '*');
|
|
}
|
|
|
|
pass() {
|
|
document.body.innerHTML = 'PASS';
|
|
window.postMessage('unittest-suite-done', '*');
|
|
}
|
|
|
|
fail(e, trace) {
|
|
document.body.innerHTML = 'FAIL: $e, $trace';
|
|
window.postMessage('unittest-suite-done', '*');
|
|
}
|
|
|
|
main() {
|
|
bool needsToWait = false;
|
|
bool mainIsFinished = false;
|
|
TestRunner.waitForDoneCallback = () { needsToWait = true; };
|
|
TestRunner.doneCallback = () {
|
|
if (mainIsFinished) {
|
|
pass();
|
|
} else {
|
|
needsToWait = false;
|
|
}
|
|
};
|
|
try {
|
|
Test.main();
|
|
if (needsToWait) {
|
|
waitForDone();
|
|
} else {
|
|
pass();
|
|
}
|
|
mainIsFinished = true;
|
|
} catch(var e, var trace) {
|
|
fail(e, trace);
|
|
}
|
|
}
|
|
"""
|
|
|
|
|
|
# Patterns for matching test options in .dart files.
|
|
DART_OPTIONS_PATTERN = re.compile(r'// DartOptions=(.*)')
|
|
|
|
# Pattern for checking if the test is a web test.
|
|
DOM_IMPORT_PATTERN = re.compile(r'#import.*(dart:(dom|html)|html\.dart).*\);',
|
|
re.MULTILINE)
|
|
|
|
# Pattern for matching the output of a browser test.
|
|
BROWSER_OUTPUT_PASS_PATTERN = re.compile(r'^Content-Type: text/plain\nPASS$',
|
|
re.MULTILINE)
|
|
|
|
# Pattern for matching flaky errors of browser tests. xvfb-run by default uses
|
|
# DISPLAY=:99, we keep that in the error pattern to avoid matching real
|
|
# errors when DISPLAY is set incorrectly.
|
|
BROWSER_FLAKY_DISPLAY_ERR_PATTERN = re.compile(
|
|
r'Gtk-WARNING \*\*: cannot open display: :99', re.MULTILINE)
|
|
|
|
# Pattern for checking if the test is a library in itself.
|
|
LIBRARY_DEFINITION_PATTERN = re.compile(r'^#library\(.*\);',
|
|
re.MULTILINE)
|
|
SOURCE_OR_IMPORT_PATTERN = re.compile(r'^#(source|import)\(.*\);',
|
|
re.MULTILINE)
|
|
|
|
|
|
class Error(Exception):
|
|
"""Base class for exceptions in this module."""
|
|
pass
|
|
|
|
|
|
def _IsWebTest(source):
|
|
"""Returns True if the source includes a dart dom library #import."""
|
|
return DOM_IMPORT_PATTERN.search(source)
|
|
|
|
|
|
def IsLibraryDefinition(test, source):
|
|
"""Returns True if the source has a #library statement."""
|
|
if LIBRARY_DEFINITION_PATTERN.search(source):
|
|
return True
|
|
if SOURCE_OR_IMPORT_PATTERN.search(source):
|
|
print ('WARNING for %s: Browser tests need a #library '
|
|
'for a file that #import or #source' % test)
|
|
return False
|
|
|
|
|
|
class Architecture(object):
|
|
"""Definitions for different ways to test based on the component flag."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
self.root_path = root_path
|
|
self.arch = arch
|
|
self.mode = mode
|
|
self.component = component
|
|
self.test = test
|
|
self.build_root = utils.GetBuildRoot(OS_GUESS, self.mode, self.arch)
|
|
source = file(test).read()
|
|
self.vm_options = []
|
|
self.dart_options = utils.ParseTestOptions(DART_OPTIONS_PATTERN,
|
|
source,
|
|
root_path)
|
|
self.is_web_test = _IsWebTest(source)
|
|
self.temp_dir = None
|
|
|
|
def GetVMOption(self, option):
|
|
for flag in self.vm_options:
|
|
if flag.startswith('--%s=' % option):
|
|
return flag.split('=')[1]
|
|
return None
|
|
|
|
def HasFatalTypeErrors(self):
|
|
"""Returns True if this type of component supports --fatal-type-errors."""
|
|
return False
|
|
|
|
def GetTestFrameworkPath(self):
|
|
"""Path to dart source (TestFramework.dart) for testing framework."""
|
|
return os.path.join(self.root_path, 'tests', 'isolate', 'src',
|
|
'TestFramework.dart')
|
|
|
|
|
|
class BrowserArchitecture(Architecture):
|
|
"""Architecture that runs compiled dart->JS through a browser."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(BrowserArchitecture, self).__init__(root_path, arch, mode, component,
|
|
test)
|
|
self.temp_dir = tempfile.mkdtemp()
|
|
if not self.is_web_test: self.GenerateWebTestScript()
|
|
|
|
def GetTestScriptFile(self):
|
|
"""Returns the name of the .dart file to compile."""
|
|
if self.is_web_test: return os.path.abspath(self.test)
|
|
return os.path.join(self.temp_dir, 'test.dart')
|
|
|
|
def GetHtmlContents(self):
|
|
"""Fills in the HTML_CONTENTS template with info for this architecture."""
|
|
script_type = self.GetScriptType()
|
|
controller_path = os.path.join(self.root_path, 'client', 'testing',
|
|
'unittest', 'test_controller.js')
|
|
return HTML_CONTENTS % {
|
|
'title': self.test,
|
|
'controller_script': controller_path,
|
|
'script_type': script_type,
|
|
'source_script': self.GetScriptPath()
|
|
}
|
|
|
|
def GetHtmlPath(self):
|
|
"""Creates a path for the generated .html file.
|
|
|
|
Resources for web tests are relative to the 'html' file. We
|
|
output the 'html' file in the 'out' directory instead of the temporary
|
|
directory because we can easily go the the resources in 'client' through
|
|
'out'.
|
|
|
|
Returns:
|
|
Created path for the generated .html file.
|
|
"""
|
|
if self.is_web_test:
|
|
html_path = os.path.join(self.root_path, 'client', self.build_root)
|
|
if not os.path.exists(html_path):
|
|
os.makedirs(html_path)
|
|
return html_path
|
|
|
|
return self.temp_dir
|
|
|
|
def GetTestContents(self, library_file):
|
|
"""Pastes a preamble on the front of the .dart file before testing."""
|
|
unittest_path = os.path.join(self.root_path, 'client', 'testing',
|
|
'unittest', 'unittest.dart')
|
|
|
|
if self.component == 'chromium':
|
|
dom_path = os.path.join(self.root_path, 'client', 'testing',
|
|
'unittest', 'dom_for_unittest.dart')
|
|
else:
|
|
dom_path = os.path.join('dart:dom')
|
|
|
|
test_framework_path = self.GetTestFrameworkPath()
|
|
test_path = os.path.abspath(self.test)
|
|
|
|
inputs = {
|
|
'unittest': unittest_path,
|
|
'test': test_path,
|
|
'dom_library': dom_path,
|
|
'test_framework': test_framework_path,
|
|
'library': library_file
|
|
}
|
|
return DART_CONTENTS % inputs
|
|
|
|
def GenerateWebTestScript(self):
|
|
"""Creates a .dart file to run in the test."""
|
|
if IsLibraryDefinition(self.test, file(self.test).read()):
|
|
library_file = os.path.abspath(self.test)
|
|
else:
|
|
library_file = 'test_as_library.dart'
|
|
test_as_library = DART_TEST_AS_LIBRARY % {
|
|
'test': os.path.abspath(self.test)
|
|
}
|
|
test_as_library_file = os.path.join(self.temp_dir, library_file)
|
|
f = open(test_as_library_file, 'w')
|
|
f.write(test_as_library)
|
|
f.close()
|
|
|
|
app_output_file = self.GetTestScriptFile()
|
|
f = open(app_output_file, 'w')
|
|
f.write(self.GetTestContents(library_file))
|
|
f.close()
|
|
|
|
def GetRunCommand(self, fatal_static_type_errors=False):
|
|
"""Returns a command line to execute for the test."""
|
|
fatal_static_type_errors = fatal_static_type_errors # shutup lint!
|
|
# Find DRT
|
|
# For some reason, DRT needs to be called via an absolute path
|
|
drt_location = self.GetVMOption('browser')
|
|
if drt_location is not None:
|
|
drt_location = os.path.abspath(drt_location)
|
|
else:
|
|
drt_location = os.path.join(self.root_path, 'client', 'tests', 'drt',
|
|
'DumpRenderTree')
|
|
|
|
# On Mac DumpRenderTree is a .app folder
|
|
if platform.system() == 'Darwin':
|
|
drt_location += '.app/Contents/MacOS/DumpRenderTree'
|
|
|
|
drt_flags = ['--no-timeout']
|
|
dart_flags = '--dart-flags=--enable_asserts --enable_type_checks '
|
|
dart_flags += ' '.join(self.vm_options)
|
|
|
|
drt_flags.append(dart_flags)
|
|
|
|
html_output_file = os.path.join(self.GetHtmlPath(), self.GetHtmlName())
|
|
f = open(html_output_file, 'w')
|
|
f.write(self.GetHtmlContents())
|
|
f.close()
|
|
|
|
drt_flags.append(html_output_file)
|
|
|
|
return [drt_location] + drt_flags
|
|
|
|
def HasFailed(self, output):
|
|
"""Return True if the 'PASS' result string isn't in the output."""
|
|
return not BROWSER_OUTPUT_PASS_PATTERN.search(output)
|
|
|
|
def WasFlakyDrt(self, error):
|
|
"""Return whether the error indicates a flaky error from running.
|
|
DumpRenderTree within xvfb-run.
|
|
"""
|
|
return BROWSER_FLAKY_DISPLAY_ERR_PATTERN.search(error)
|
|
|
|
def RunTest(self, verbose):
|
|
"""Calls GetRunCommand() and executes the returned commandline.
|
|
|
|
Args:
|
|
verbose: if True, print additional diagnostics to stdout.
|
|
|
|
Returns:
|
|
Return code from executable. 0 == PASS, 253 = CRASH, anything
|
|
else is treated as FAIL
|
|
"""
|
|
retcode = self.Compile()
|
|
if retcode != 0: return 1
|
|
|
|
command = self.GetRunCommand()
|
|
|
|
unused_status, output, err = ExecutePipedCommand(command, verbose)
|
|
if not self.HasFailed(output):
|
|
return 0
|
|
|
|
# TODO(sigmund): print better error message, including how to run test
|
|
# locally, and translate error traces using source map info.
|
|
print '(FAIL) test page:\033[31m %s \033[0m' % command[2]
|
|
if verbose:
|
|
print 'Additional info: '
|
|
print output
|
|
print err
|
|
return 1
|
|
|
|
def Cleanup(self):
|
|
"""Removes temporary files created for the test."""
|
|
if self.temp_dir:
|
|
shutil.rmtree(self.temp_dir)
|
|
self.temp_dir = None
|
|
|
|
|
|
class ChromiumArchitecture(BrowserArchitecture):
|
|
"""Architecture that runs compiled dart->JS through a chromium DRT."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(ChromiumArchitecture, self).__init__(root_path, arch, mode, component, test)
|
|
|
|
def GetScriptType(self):
|
|
return 'text/javascript'
|
|
|
|
def GetScriptPath(self):
|
|
"""Returns the name of the output .js file to create."""
|
|
path = self.GetTestScriptFile()
|
|
return os.path.abspath(os.path.join(self.temp_dir,
|
|
os.path.basename(path) + '.js'))
|
|
|
|
def GetHtmlName(self):
|
|
"""Returns the name of the output .html file to create."""
|
|
relpath = os.path.relpath(self.test, self.root_path)
|
|
return relpath.replace(os.sep, '_') + '.html'
|
|
|
|
def Compile(self):
|
|
return ExecuteCommand(self.GetCompileCommand())
|
|
|
|
class DartcChromiumArchitecture(ChromiumArchitecture):
|
|
"""ChromiumArchitecture that compiles code using dartc."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(DartcChromiumArchitecture, self).__init__(
|
|
root_path, arch, mode, component, test)
|
|
|
|
def GetCompileCommand(self, fatal_static_type_errors=False):
|
|
"""Returns cmdline as an array to invoke the compiler on this test."""
|
|
|
|
# We need an absolute path because the compilation will run
|
|
# in a temporary directory.
|
|
build_root = utils.GetBuildRoot(OS_GUESS, self.mode, 'ia32')
|
|
dartc = os.path.abspath(os.path.join(build_root, 'compiler', 'bin',
|
|
'dartc'))
|
|
if utils.IsWindows(): dartc += '.exe'
|
|
cmd = [dartc, '--work', self.temp_dir]
|
|
if self.mode == 'release':
|
|
cmd += ['--optimize']
|
|
cmd += self.vm_options
|
|
cmd += ['--out', self.GetScriptPath()]
|
|
if fatal_static_type_errors:
|
|
# TODO(zundel): update to --fatal_type_errors for both VM and Compiler
|
|
cmd.append('-fatal-type-errors')
|
|
cmd.append(self.GetTestScriptFile())
|
|
return cmd
|
|
|
|
|
|
class FrogChromiumArchitecture(ChromiumArchitecture):
|
|
"""ChromiumArchitecture that compiles code using frog."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(FrogChromiumArchitecture, self).__init__(
|
|
root_path, arch, mode, component, test)
|
|
|
|
def GetCompileCommand(self, fatal_static_type_errors=False):
|
|
"""Returns cmdline as an array to invoke the compiler on this test."""
|
|
|
|
# Get a frog executable from the command line. Default to frogsh.
|
|
# We need an absolute path because the compilation will run
|
|
# in a temporary directory.
|
|
frog = self.GetVMOption('frog')
|
|
if frog is not None:
|
|
frog = os.path.abspath(frog)
|
|
else:
|
|
frog = os.path.abspath(utils.GetDartRunner(self.mode, self.arch, 'frogsh'))
|
|
frog_libdir = self.GetVMOption('froglib')
|
|
if frog_libdir is not None:
|
|
frog_libdir = os.path.abspath(frog_libdir)
|
|
else:
|
|
frog_libdir = os.path.abspath(os.path.join(self.root_path, 'frog', 'lib'))
|
|
cmd = [frog,
|
|
'--libdir=%s' % frog_libdir,
|
|
'--compile-only',
|
|
'--out=%s' % self.GetScriptPath()]
|
|
cmd.extend(self.vm_options)
|
|
cmd.append(self.GetTestScriptFile())
|
|
return cmd
|
|
|
|
|
|
class DartiumArchitecture(BrowserArchitecture):
|
|
"""Architecture that runs dart in an VM embedded in DumpRenderTree."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(DartiumArchitecture, self).__init__(root_path, arch, mode, component, test)
|
|
|
|
def GetScriptType(self):
|
|
return 'application/dart'
|
|
|
|
def GetScriptPath(self):
|
|
return 'file:///' + self.GetTestScriptFile()
|
|
|
|
def GetHtmlName(self):
|
|
path = os.path.relpath(self.test, self.root_path).replace(os.sep, '_')
|
|
return path + '.dartium.html'
|
|
|
|
def GetCompileCommand(self, fatal_static_type_errors=False):
|
|
fatal_static_type_errors = fatal_static_type_errors # shutup lint!
|
|
return None
|
|
|
|
def Compile(self):
|
|
return 0
|
|
|
|
|
|
class WebDriverArchitecture(FrogChromiumArchitecture):
|
|
"""Architecture that runs compiled dart->JS (via frog) through a variety of
|
|
real browsers using WebDriver."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(WebDriverArchitecture, self).__init__(root_path, arch, mode,
|
|
component, test)
|
|
|
|
def GetRunCommand(self, fatal_static_type_errors=False):
|
|
"""Returns a command line to execute for the test."""
|
|
flags = self.vm_options
|
|
browser_flag = 'chrome'
|
|
if 'ff' in flags or 'firefox' in flags:
|
|
browser_flag = 'ff'
|
|
elif 'ie' in flags or 'explorer' in flags or 'internet-explorer' in flags:
|
|
browser_flag = 'ie'
|
|
elif 'safari' in flags:
|
|
browser_flag = 'safari'
|
|
|
|
selenium_location = os.path.join(self.root_path, 'tools', 'testing',
|
|
'run_selenium.py')
|
|
|
|
html_output_file = os.path.join(self.GetHtmlPath(), self.GetHtmlName())
|
|
f = open(html_output_file, 'w')
|
|
f.write(self.GetHtmlContents())
|
|
f.close()
|
|
return [selenium_location, '--out', html_output_file, '--browser',
|
|
browser_flag]
|
|
|
|
|
|
class StandaloneArchitecture(Architecture):
|
|
"""Base class for architectures that run tests without a browser."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(StandaloneArchitecture, self).__init__(root_path, arch, mode, component,
|
|
test)
|
|
|
|
def GetExecutable(self):
|
|
"""Returns the path to the Dart test runner (executes the .dart file)."""
|
|
return utils.GetDartRunner(self.mode, self.arch, self.component)
|
|
|
|
|
|
def GetCompileCommand(self, fatal_static_type_errors=False):
|
|
fatal_static_type_errors = fatal_static_type_errors # shutup lint!
|
|
return None
|
|
|
|
def GetOptions(self):
|
|
return []
|
|
|
|
def GetRunCommand(self, fatal_static_type_errors=False):
|
|
"""Returns a command line to execute for the test."""
|
|
dart = self.GetExecutable()
|
|
command = [dart] + self.GetOptions() + self.vm_options
|
|
if fatal_static_type_errors:
|
|
command += self.GetFatalTypeErrorsFlags()
|
|
|
|
if self.dart_options:
|
|
command += self.dart_options
|
|
else:
|
|
command += [self.test]
|
|
|
|
return command
|
|
|
|
def GetFatalTypeErrorsFlags(self):
|
|
return []
|
|
|
|
def RunTest(self, verbose):
|
|
command = self.GetRunCommand()
|
|
return ExecuteCommand(command, verbose)
|
|
|
|
def Cleanup(self):
|
|
return
|
|
|
|
|
|
class LegArchitecture(StandaloneArchitecture):
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(LegArchitecture, self).__init__(root_path, arch, mode, component,
|
|
test)
|
|
def GetOptions(self):
|
|
return ['--leg_only']
|
|
|
|
def GetExecutable(self):
|
|
"""Returns the path to the Dart test runner (executes the .dart file)."""
|
|
return utils.GetDartRunner(self.mode, self.arch, 'frog')
|
|
|
|
|
|
# Long term, we should do the running machinery that is currently in
|
|
# DartRunner.java
|
|
class DartcArchitecture(StandaloneArchitecture):
|
|
"""Runs the Dart ->JS compiler then runs the result in a standalone JS VM."""
|
|
|
|
def __init__(self, root_path, arch, mode, component, test):
|
|
super(DartcArchitecture, self).__init__(root_path, arch, mode, component, test)
|
|
|
|
def GetOptions(self):
|
|
if self.mode == 'release': return ['--optimize']
|
|
return []
|
|
|
|
def GetFatalTypeErrorsFlags(self):
|
|
return ['--fatal-type-errors']
|
|
|
|
def HasFatalTypeErrors(self):
|
|
return True
|
|
|
|
|
|
def ExecutePipedCommand(cmd, verbose):
|
|
"""Execute a command in a subprocess."""
|
|
if verbose:
|
|
print 'Executing: ' + ' '.join(cmd)
|
|
pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
(output, err) = pipe.communicate()
|
|
if pipe.returncode != 0 and verbose:
|
|
print 'Execution failed: ' + output + '\n' + err
|
|
print output
|
|
print err
|
|
return pipe.returncode, output, err
|
|
|
|
|
|
def ExecuteCommand(cmd, verbose=False):
|
|
"""Execute a command in a subprocess."""
|
|
if verbose: print 'Executing: ' + ' '.join(cmd)
|
|
return subprocess.call(cmd)
|
|
|
|
|
|
def GetArchitecture(arch, mode, component, test):
|
|
root_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
|
|
if component == 'chromium':
|
|
return DartcChromiumArchitecture(root_path, arch, mode, component, test)
|
|
|
|
elif component == 'dartium':
|
|
return DartiumArchitecture(root_path, arch, mode, component, test)
|
|
|
|
elif component == 'frogium':
|
|
return FrogChromiumArchitecture(root_path, arch, mode, component, test)
|
|
|
|
elif component == 'webdriver':
|
|
return WebDriverArchitecture(root_path, arch, mode, component, test)
|
|
|
|
elif component in ['vm', 'frog', 'frogsh']:
|
|
return StandaloneArchitecture(root_path, arch, mode, component, test)
|
|
|
|
elif component == 'leg':
|
|
return LegArchitecture(root_path, arch, mode, component, test)
|
|
|
|
elif component == 'dartc':
|
|
return DartcArchitecture(root_path, arch, mode, component, test)
|