diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 00000000000..0e9640c29a2 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,2 @@ +[style] +based_on_style = google diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 0a3948320b3..26354c47aee 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -1,7 +1,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """Top-level presubmit script for Dart. See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts @@ -16,209 +15,219 @@ import subprocess import tempfile -def _CheckFormat(input_api, identification, extension, windows, - hasFormatErrors, should_skip = lambda path : False): - local_root = input_api.change.RepositoryRoot() - upstream = input_api.change._upstream - unformatted_files = [] - for git_file in input_api.AffectedTextFiles(): - if git_file.LocalPath().startswith("pkg/front_end/testcases/"): - continue - if should_skip(git_file.LocalPath()): - continue - filename = git_file.AbsoluteLocalPath() - if filename.endswith(extension) and hasFormatErrors(filename=filename): - old_version_has_errors = False - try: - path = git_file.LocalPath() - if windows: - # Git expects a linux style path. - path = path.replace(os.sep, '/') - old_contents = scm.GIT.Capture( - ['show', upstream + ':' + path], - cwd=local_root, - strip_out=False) - if hasFormatErrors(contents=old_contents): - old_version_has_errors = True - except subprocess.CalledProcessError as e: - old_version_has_errors = False +def _CheckFormat(input_api, + identification, + extension, + windows, + hasFormatErrors, + should_skip=lambda path: False): + local_root = input_api.change.RepositoryRoot() + upstream = input_api.change._upstream + unformatted_files = [] + for git_file in input_api.AffectedTextFiles(): + if git_file.LocalPath().startswith("pkg/front_end/testcases/"): + continue + if should_skip(git_file.LocalPath()): + continue + filename = git_file.AbsoluteLocalPath() + if filename.endswith(extension) and hasFormatErrors(filename=filename): + old_version_has_errors = False + try: + path = git_file.LocalPath() + if windows: + # Git expects a linux style path. + path = path.replace(os.sep, '/') + old_contents = scm.GIT.Capture(['show', upstream + ':' + path], + cwd=local_root, + strip_out=False) + if hasFormatErrors(contents=old_contents): + old_version_has_errors = True + except subprocess.CalledProcessError as e: + old_version_has_errors = False - if old_version_has_errors: - print("WARNING: %s has existing and possibly new %s issues" % - (git_file.LocalPath(), identification)) - else: - unformatted_files.append(filename) + if old_version_has_errors: + print("WARNING: %s has existing and possibly new %s issues" % + (git_file.LocalPath(), identification)) + else: + unformatted_files.append(filename) - return unformatted_files + return unformatted_files def _CheckBuildStatus(input_api, output_api): - results = [] - status_check = input_api.canned_checks.CheckTreeIsOpen( - input_api, - output_api, - json_url='http://dart-status.appspot.com/current?format=json') - results.extend(status_check) - return results + results = [] + status_check = input_api.canned_checks.CheckTreeIsOpen( + input_api, + output_api, + json_url='http://dart-status.appspot.com/current?format=json') + results.extend(status_check) + return results def _CheckDartFormat(input_api, output_api): - local_root = input_api.change.RepositoryRoot() - upstream = input_api.change._upstream - utils = imp.load_source('utils', - os.path.join(local_root, 'tools', 'utils.py')) + local_root = input_api.change.RepositoryRoot() + upstream = input_api.change._upstream + utils = imp.load_source('utils', + os.path.join(local_root, 'tools', 'utils.py')) - prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt') + prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt') - windows = utils.GuessOS() == 'win32' - if windows: - prebuilt_dartfmt += '.bat' - - if not os.path.isfile(prebuilt_dartfmt): - print('WARNING: dartfmt not found: %s' % (prebuilt_dartfmt)) - return [] - - def HasFormatErrors(filename=None, contents=None): - # Don't look for formatting errors in multitests. Since those are very - # sensitive to whitespace, many cannot be formatted with dartfmt without - # breaking them. - if filename and filename.endswith('_test.dart'): - with open(filename) as f: - contents = f.read() - if '//#' in contents: - return False - - args = [prebuilt_dartfmt, '--set-exit-if-changed'] - if not contents: - args += [filename, '-n'] - - process = subprocess.Popen(args, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - process.communicate(input=contents) - - # Check for exit code 1 explicitly to distinguish it from a syntax error - # in the file (exit code 65). The repo contains many Dart files that are - # known to have syntax errors for testing purposes and which can't be - # parsed and formatted. Don't treat those as errors. - return process.returncode == 1 - - unformatted_files = _CheckFormat(input_api, "dartfmt", ".dart", windows, - HasFormatErrors) - - if unformatted_files: - lineSep = " \\\n" + windows = utils.GuessOS() == 'win32' if windows: - lineSep = " ^\n"; - return [output_api.PresubmitError( - 'File output does not match dartfmt.\n' - 'Fix these issues with:\n' - '%s -w%s%s' % (prebuilt_dartfmt, lineSep, - lineSep.join(unformatted_files)))] + prebuilt_dartfmt += '.bat' - return [] + if not os.path.isfile(prebuilt_dartfmt): + print('WARNING: dartfmt not found: %s' % (prebuilt_dartfmt)) + return [] + + def HasFormatErrors(filename=None, contents=None): + # Don't look for formatting errors in multitests. Since those are very + # sensitive to whitespace, many cannot be formatted with dartfmt without + # breaking them. + if filename and filename.endswith('_test.dart'): + with open(filename) as f: + contents = f.read() + if '//#' in contents: + return False + + args = [prebuilt_dartfmt, '--set-exit-if-changed'] + if not contents: + args += [filename, '-n'] + + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) + process.communicate(input=contents) + + # Check for exit code 1 explicitly to distinguish it from a syntax error + # in the file (exit code 65). The repo contains many Dart files that are + # known to have syntax errors for testing purposes and which can't be + # parsed and formatted. Don't treat those as errors. + return process.returncode == 1 + + unformatted_files = _CheckFormat(input_api, "dartfmt", ".dart", windows, + HasFormatErrors) + + if unformatted_files: + lineSep = " \\\n" + if windows: + lineSep = " ^\n" + return [ + output_api.PresubmitError( + 'File output does not match dartfmt.\n' + 'Fix these issues with:\n' + '%s -w%s%s' % (prebuilt_dartfmt, lineSep, + lineSep.join(unformatted_files))) + ] + + return [] def _CheckStatusFiles(input_api, output_api): - local_root = input_api.change.RepositoryRoot() - upstream = input_api.change._upstream - utils = imp.load_source('utils', - os.path.join(local_root, 'tools', 'utils.py')) + local_root = input_api.change.RepositoryRoot() + upstream = input_api.change._upstream + utils = imp.load_source('utils', + os.path.join(local_root, 'tools', 'utils.py')) - dart = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dart') - lint = os.path.join(local_root, 'pkg', 'status_file', 'bin', 'lint.dart') + dart = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dart') + lint = os.path.join(local_root, 'pkg', 'status_file', 'bin', 'lint.dart') - windows = utils.GuessOS() == 'win32' - if windows: - dart += '.exe' - - if not os.path.isfile(dart): - print('WARNING: dart not found: %s' % dart) - return [] - - if not os.path.isfile(lint): - print('WARNING: Status file linter not found: %s' % lint) - return [] - - def HasFormatErrors(filename=None, contents=None): - args = [dart, lint] + (['-t'] if contents else [filename]) - process = subprocess.Popen(args, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - process.communicate(input=contents) - return process.returncode != 0 - - def should_skip(path): - return (path.startswith("pkg/status_file/test/data/") - or path.startswith("pkg/front_end/")) - - unformatted_files = _CheckFormat(input_api, "status file", ".status", - windows, HasFormatErrors, should_skip) - - if unformatted_files: - normalize = os.path.join(local_root, 'pkg', 'status_file', 'bin', - 'normalize.dart') - lineSep = " \\\n" + windows = utils.GuessOS() == 'win32' if windows: - lineSep = " ^\n"; - return [output_api.PresubmitError( - 'Status files are not normalized.\n' - 'Fix these issues with:\n' - '%s %s -w%s%s' % (dart, normalize, lineSep, - lineSep.join(unformatted_files)))] + dart += '.exe' - return [] + if not os.path.isfile(dart): + print('WARNING: dart not found: %s' % dart) + return [] + + if not os.path.isfile(lint): + print('WARNING: Status file linter not found: %s' % lint) + return [] + + def HasFormatErrors(filename=None, contents=None): + args = [dart, lint] + (['-t'] if contents else [filename]) + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) + process.communicate(input=contents) + return process.returncode != 0 + + def should_skip(path): + return (path.startswith("pkg/status_file/test/data/") or + path.startswith("pkg/front_end/")) + + unformatted_files = _CheckFormat(input_api, "status file", ".status", + windows, HasFormatErrors, should_skip) + + if unformatted_files: + normalize = os.path.join(local_root, 'pkg', 'status_file', 'bin', + 'normalize.dart') + lineSep = " \\\n" + if windows: + lineSep = " ^\n" + return [ + output_api.PresubmitError( + 'Status files are not normalized.\n' + 'Fix these issues with:\n' + '%s %s -w%s%s' % (dart, normalize, lineSep, + lineSep.join(unformatted_files))) + ] + + return [] def _CheckValidHostsInDEPS(input_api, output_api): - """Checks that DEPS file deps are from allowed_hosts.""" - # Run only if DEPS file has been modified to annoy fewer bystanders. - if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()): - return [] - # Outsource work to gclient verify - try: - input_api.subprocess.check_output(['gclient', 'verify']) - return [] - except input_api.subprocess.CalledProcessError, error: - return [output_api.PresubmitError( - 'DEPS file must have only dependencies from allowed hosts.', - long_text=error.output)] + """Checks that DEPS file deps are from allowed_hosts.""" + # Run only if DEPS file has been modified to annoy fewer bystanders. + if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()): + return [] + # Outsource work to gclient verify + try: + input_api.subprocess.check_output(['gclient', 'verify']) + return [] + except input_api.subprocess.CalledProcessError, error: + return [ + output_api.PresubmitError( + 'DEPS file must have only dependencies from allowed hosts.', + long_text=error.output) + ] + def _CheckLayering(input_api, output_api): - """Run VM layering check. + """Run VM layering check. This check validates that sources from one layer do not reference sources from another layer accidentally. """ - # Run only if .cc or .h file was modified. - def is_cpp_file(path): - return path.endswith('.cc') or path.endswith('.h') - if all(not is_cpp_file(f.LocalPath()) for f in input_api.AffectedFiles()): - return [] - local_root = input_api.change.RepositoryRoot() - layering_check = imp.load_source('layering_check', - os.path.join(local_root, 'runtime', 'tools', 'layering_check.py')) - errors = layering_check.DoCheck(local_root) - if errors: - return [output_api.PresubmitError( - 'Layering check violation for C++ sources.', - long_text='\n'.join(errors))] - else: - return [] + # Run only if .cc or .h file was modified. + def is_cpp_file(path): + return path.endswith('.cc') or path.endswith('.h') + + if all(not is_cpp_file(f.LocalPath()) for f in input_api.AffectedFiles()): + return [] + + local_root = input_api.change.RepositoryRoot() + layering_check = imp.load_source( + 'layering_check', + os.path.join(local_root, 'runtime', 'tools', 'layering_check.py')) + errors = layering_check.DoCheck(local_root) + if errors: + return [ + output_api.PresubmitError( + 'Layering check violation for C++ sources.', + long_text='\n'.join(errors)) + ] + else: + return [] def CheckChangeOnCommit(input_api, output_api): - return (_CheckValidHostsInDEPS(input_api, output_api) + - _CheckBuildStatus(input_api, output_api) + - _CheckDartFormat(input_api, output_api) + - _CheckStatusFiles(input_api, output_api) + - _CheckLayering(input_api, output_api)) + return (_CheckValidHostsInDEPS(input_api, output_api) + _CheckBuildStatus( + input_api, output_api) + _CheckDartFormat(input_api, output_api) + + _CheckStatusFiles(input_api, output_api) + _CheckLayering( + input_api, output_api)) def CheckChangeOnUpload(input_api, output_api): - return (_CheckValidHostsInDEPS(input_api, output_api) + - _CheckDartFormat(input_api, output_api) + - _CheckStatusFiles(input_api, output_api) + - _CheckLayering(input_api, output_api)) + return (_CheckValidHostsInDEPS(input_api, output_api) + _CheckDartFormat( + input_api, output_api) + _CheckStatusFiles(input_api, output_api) + + _CheckLayering(input_api, output_api)) diff --git a/build/config/linux/pkg-config.py b/build/config/linux/pkg-config.py index fadcc0bbcf9..70cc608cbea 100644 --- a/build/config/linux/pkg-config.py +++ b/build/config/linux/pkg-config.py @@ -34,139 +34,141 @@ from optparse import OptionParser # success. This allows us to "kind of emulate" a Linux build from other # platforms. if sys.platform.find("linux") == -1: - print "[[],[],[],[],[]]" - sys.exit(0) + print "[[],[],[],[],[]]" + sys.exit(0) def SetConfigPath(options): - """Set the PKG_CONFIG_PATH environment variable. + """Set the PKG_CONFIG_PATH environment variable. This takes into account any sysroot and architecture specification from the options on the given command line.""" - sysroot = options.sysroot - if not sysroot: - sysroot = "" + sysroot = options.sysroot + if not sysroot: + sysroot = "" - # Compute the library path name based on the architecture. - arch = options.arch - if sysroot and not arch: - print "You must specify an architecture via -a if using a sysroot." - sys.exit(1) - if arch == 'x64': - libpath = 'lib64' - else: - libpath = 'lib' + # Compute the library path name based on the architecture. + arch = options.arch + if sysroot and not arch: + print "You must specify an architecture via -a if using a sysroot." + sys.exit(1) + if arch == 'x64': + libpath = 'lib64' + else: + libpath = 'lib' - # Add the sysroot path to the environment's PKG_CONFIG_PATH - config_path = sysroot + '/usr/' + libpath + '/pkgconfig' - config_path += ':' + sysroot + '/usr/share/pkgconfig' - if 'PKG_CONFIG_PATH' in os.environ: - os.environ['PKG_CONFIG_PATH'] += ':' + config_path - else: - os.environ['PKG_CONFIG_PATH'] = config_path + # Add the sysroot path to the environment's PKG_CONFIG_PATH + config_path = sysroot + '/usr/' + libpath + '/pkgconfig' + config_path += ':' + sysroot + '/usr/share/pkgconfig' + if 'PKG_CONFIG_PATH' in os.environ: + os.environ['PKG_CONFIG_PATH'] += ':' + config_path + else: + os.environ['PKG_CONFIG_PATH'] = config_path def GetPkgConfigPrefixToStrip(args): - """Returns the prefix from pkg-config where packages are installed. + """Returns the prefix from pkg-config where packages are installed. This returned prefix is the one that should be stripped from the beginning of directory names to take into account sysroots.""" - # Some sysroots, like the Chromium OS ones, may generate paths that are not - # relative to the sysroot. For example, - # /path/to/chroot/build/x86-generic/usr/lib/pkgconfig/pkg.pc may have all - # paths relative to /path/to/chroot (i.e. prefix=/build/x86-generic/usr) - # instead of relative to /path/to/chroot/build/x86-generic (i.e prefix=/usr). - # To support this correctly, it's necessary to extract the prefix to strip - # from pkg-config's |prefix| variable. - prefix = subprocess.check_output(["pkg-config", "--variable=prefix"] + args, - env=os.environ) - if prefix[-4] == '/usr': - return prefix[4:] - return prefix + # Some sysroots, like the Chromium OS ones, may generate paths that are not + # relative to the sysroot. For example, + # /path/to/chroot/build/x86-generic/usr/lib/pkgconfig/pkg.pc may have all + # paths relative to /path/to/chroot (i.e. prefix=/build/x86-generic/usr) + # instead of relative to /path/to/chroot/build/x86-generic (i.e prefix=/usr). + # To support this correctly, it's necessary to extract the prefix to strip + # from pkg-config's |prefix| variable. + prefix = subprocess.check_output( + ["pkg-config", "--variable=prefix"] + args, env=os.environ) + if prefix[-4] == '/usr': + return prefix[4:] + return prefix def MatchesAnyRegexp(flag, list_of_regexps): - """Returns true if the first argument matches any regular expression in the + """Returns true if the first argument matches any regular expression in the given list.""" - for regexp in list_of_regexps: - if regexp.search(flag) != None: - return True - return False + for regexp in list_of_regexps: + if regexp.search(flag) != None: + return True + return False def RewritePath(path, strip_prefix, sysroot): - """Rewrites a path by stripping the prefix and prepending the sysroot.""" - if os.path.isabs(path) and not path.startswith(sysroot): - if path.startswith(strip_prefix): - path = path[len(strip_prefix):] - path = path.lstrip('/') - return os.path.join(sysroot, path) - else: - return path + """Rewrites a path by stripping the prefix and prepending the sysroot.""" + if os.path.isabs(path) and not path.startswith(sysroot): + if path.startswith(strip_prefix): + path = path[len(strip_prefix):] + path = path.lstrip('/') + return os.path.join(sysroot, path) + else: + return path parser = OptionParser() -parser.add_option('-p', action='store', dest='pkg_config', type='string', - default='pkg-config') +parser.add_option( + '-p', + action='store', + dest='pkg_config', + type='string', + default='pkg-config') parser.add_option('-v', action='append', dest='strip_out', type='string') parser.add_option('-s', action='store', dest='sysroot', type='string') parser.add_option('-a', action='store', dest='arch', type='string') -parser.add_option('--atleast-version', action='store', - dest='atleast_version', type='string') +parser.add_option( + '--atleast-version', action='store', dest='atleast_version', type='string') parser.add_option('--libdir', action='store_true', dest='libdir') (options, args) = parser.parse_args() # Make a list of regular expressions to strip out. strip_out = [] if options.strip_out != None: - for regexp in options.strip_out: - strip_out.append(re.compile(regexp)) + for regexp in options.strip_out: + strip_out.append(re.compile(regexp)) SetConfigPath(options) if options.sysroot: - prefix = GetPkgConfigPrefixToStrip(args) + prefix = GetPkgConfigPrefixToStrip(args) else: - prefix = '' + prefix = '' if options.atleast_version: - # When asking for the return value, just run pkg-config and print the return - # value, no need to do other work. - if not subprocess.call([options.pkg_config, - "--atleast-version=" + options.atleast_version] + - args, - env=os.environ): - print "true" - else: - print "false" - sys.exit(0) + # When asking for the return value, just run pkg-config and print the return + # value, no need to do other work. + if not subprocess.call( + [options.pkg_config, "--atleast-version=" + options.atleast_version] + + args, + env=os.environ): + print "true" + else: + print "false" + sys.exit(0) if options.libdir: - try: - libdir = subprocess.check_output([options.pkg_config, - "--variable=libdir"] + - args, - env=os.environ) - except: - print "Error from pkg-config." - sys.exit(1) - sys.stdout.write(libdir.strip()) - sys.exit(0) + try: + libdir = subprocess.check_output( + [options.pkg_config, "--variable=libdir"] + args, env=os.environ) + except: + print "Error from pkg-config." + sys.exit(1) + sys.stdout.write(libdir.strip()) + sys.exit(0) try: - flag_string = subprocess.check_output( - [ options.pkg_config, "--cflags", "--libs-only-l", "--libs-only-L" ] + - args, env=os.environ) - # For now just split on spaces to get the args out. This will break if - # pkgconfig returns quoted things with spaces in them, but that doesn't seem - # to happen in practice. - all_flags = flag_string.strip().split(' ') + flag_string = subprocess.check_output( + [options.pkg_config, "--cflags", "--libs-only-l", "--libs-only-L"] + + args, + env=os.environ) + # For now just split on spaces to get the args out. This will break if + # pkgconfig returns quoted things with spaces in them, but that doesn't seem + # to happen in practice. + all_flags = flag_string.strip().split(' ') except: - print "Could not run pkg-config." - sys.exit(1) - + print "Could not run pkg-config." + sys.exit(1) sysroot = options.sysroot if not sysroot: - sysroot = '' + sysroot = '' includes = [] cflags = [] @@ -175,24 +177,24 @@ lib_dirs = [] ldflags = [] for flag in all_flags[:]: - if len(flag) == 0 or MatchesAnyRegexp(flag, strip_out): - continue; + if len(flag) == 0 or MatchesAnyRegexp(flag, strip_out): + continue - if flag[:2] == '-l': - libs.append(RewritePath(flag[2:], prefix, sysroot)) - elif flag[:2] == '-L': - lib_dirs.append(RewritePath(flag[2:], prefix, sysroot)) - elif flag[:2] == '-I': - includes.append(RewritePath(flag[2:], prefix, sysroot)) - elif flag[:3] == '-Wl': - ldflags.append(flag) - elif flag == '-pthread': - # Many libs specify "-pthread" which we don't need since we always include - # this anyway. Removing it here prevents a bunch of duplicate inclusions on - # the command line. - pass - else: - cflags.append(flag) + if flag[:2] == '-l': + libs.append(RewritePath(flag[2:], prefix, sysroot)) + elif flag[:2] == '-L': + lib_dirs.append(RewritePath(flag[2:], prefix, sysroot)) + elif flag[:2] == '-I': + includes.append(RewritePath(flag[2:], prefix, sysroot)) + elif flag[:3] == '-Wl': + ldflags.append(flag) + elif flag == '-pthread': + # Many libs specify "-pthread" which we don't need since we always include + # this anyway. Removing it here prevents a bunch of duplicate inclusions on + # the command line. + pass + else: + cflags.append(flag) # Output a GN array, the first one is the cflags, the second are the libs. The # JSON formatter prints GN compatible lists when everything is a list of diff --git a/build/config/linux/sysroot_ld_path.py b/build/config/linux/sysroot_ld_path.py index 4bce7ee3e2b..53b094d6fa7 100644 --- a/build/config/linux/sysroot_ld_path.py +++ b/build/config/linux/sysroot_ld_path.py @@ -12,8 +12,8 @@ import subprocess import sys if len(sys.argv) != 3: - print "Need two arguments" - sys.exit(1) + print "Need two arguments" + sys.exit(1) result = subprocess.check_output([sys.argv[1], sys.argv[2]]).strip() diff --git a/build/config/mac/mac_app.py b/build/config/mac/mac_app.py index 909fa583e64..82459678d36 100644 --- a/build/config/mac/mac_app.py +++ b/build/config/mac/mac_app.py @@ -9,105 +9,100 @@ import errno import subprocess import sys -PLUTIL = [ - '/usr/bin/env', - 'xcrun', - 'plutil' -] +PLUTIL = ['/usr/bin/env', 'xcrun', 'plutil'] IBTOOL = [ - '/usr/bin/env', - 'xcrun', - 'ibtool', + '/usr/bin/env', + 'xcrun', + 'ibtool', ] def MakeDirectories(path): - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - return 0 - else: - return -1 + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + return 0 + else: + return -1 - return 0 + return 0 def ProcessInfoPlist(args): - output_plist_file = os.path.abspath(os.path.join(args.output, 'Info.plist')) - return subprocess.check_call( PLUTIL + [ - '-convert', - 'binary1', - '-o', - output_plist_file, - '--', - args.input, - ]) + output_plist_file = os.path.abspath(os.path.join(args.output, 'Info.plist')) + return subprocess.check_call(PLUTIL + [ + '-convert', + 'binary1', + '-o', + output_plist_file, + '--', + args.input, + ]) def ProcessNIB(args): - output_nib_file = os.path.join(os.path.abspath(args.output), - "%s.nib" % os.path.splitext(os.path.basename(args.input))[0]) + output_nib_file = os.path.join( + os.path.abspath(args.output), + "%s.nib" % os.path.splitext(os.path.basename(args.input))[0]) - return subprocess.check_call(IBTOOL + [ - '--module', - args.module, - '--auto-activate-custom-fonts', - '--target-device', - 'mac', - '--compile', - output_nib_file, - os.path.abspath(args.input), - ]) + return subprocess.check_call(IBTOOL + [ + '--module', + args.module, + '--auto-activate-custom-fonts', + '--target-device', + 'mac', + '--compile', + output_nib_file, + os.path.abspath(args.input), + ]) def GenerateProjectStructure(args): - application_path = os.path.join( args.dir, args.name + ".app", "Contents" ) - return MakeDirectories( application_path ) + application_path = os.path.join(args.dir, args.name + ".app", "Contents") + return MakeDirectories(application_path) def Main(): - parser = argparse.ArgumentParser(description='A script that aids in ' - 'the creation of an Mac application') + parser = argparse.ArgumentParser(description='A script that aids in ' + 'the creation of an Mac application') - subparsers = parser.add_subparsers() + subparsers = parser.add_subparsers() - # Plist Parser + # Plist Parser - plist_parser = subparsers.add_parser('plist', - help='Process the Info.plist') - plist_parser.set_defaults(func=ProcessInfoPlist) - - plist_parser.add_argument('-i', dest='input', help='The input plist path') - plist_parser.add_argument('-o', dest='output', help='The output plist dir') + plist_parser = subparsers.add_parser('plist', help='Process the Info.plist') + plist_parser.set_defaults(func=ProcessInfoPlist) - # NIB Parser + plist_parser.add_argument('-i', dest='input', help='The input plist path') + plist_parser.add_argument('-o', dest='output', help='The output plist dir') - plist_parser = subparsers.add_parser('nib', - help='Process a NIB file') - plist_parser.set_defaults(func=ProcessNIB) - - plist_parser.add_argument('-i', dest='input', help='The input nib path') - plist_parser.add_argument('-o', dest='output', help='The output nib dir') - plist_parser.add_argument('-m', dest='module', help='The module name') + # NIB Parser - # Directory Structure Parser + plist_parser = subparsers.add_parser('nib', help='Process a NIB file') + plist_parser.set_defaults(func=ProcessNIB) - dir_struct_parser = subparsers.add_parser('structure', - help='Creates the directory of an Mac application') + plist_parser.add_argument('-i', dest='input', help='The input nib path') + plist_parser.add_argument('-o', dest='output', help='The output nib dir') + plist_parser.add_argument('-m', dest='module', help='The module name') - dir_struct_parser.set_defaults(func=GenerateProjectStructure) + # Directory Structure Parser - dir_struct_parser.add_argument('-d', dest='dir', help='Out directory') - dir_struct_parser.add_argument('-n', dest='name', help='App name') + dir_struct_parser = subparsers.add_parser( + 'structure', help='Creates the directory of an Mac application') - # Engage! + dir_struct_parser.set_defaults(func=GenerateProjectStructure) - args = parser.parse_args() + dir_struct_parser.add_argument('-d', dest='dir', help='Out directory') + dir_struct_parser.add_argument('-n', dest='name', help='App name') - return args.func(args) + # Engage! + + args = parser.parse_args() + + return args.func(args) if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/build/detect_host_arch.py b/build/detect_host_arch.py index cb61ec7ac2d..1b179cbfac7 100755 --- a/build/detect_host_arch.py +++ b/build/detect_host_arch.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Outputs host CPU architecture in format recognized by gyp.""" import platform @@ -11,41 +10,42 @@ import sys def HostArch(): - """Returns the host architecture with a predictable string.""" - host_arch = platform.machine() + """Returns the host architecture with a predictable string.""" + host_arch = platform.machine() - # Convert machine type to format recognized by gyp. - if re.match(r'i.86', host_arch) or host_arch == 'i86pc': - host_arch = 'ia32' - elif host_arch in ['x86_64', 'amd64']: - host_arch = 'x64' - elif host_arch.startswith('arm'): - host_arch = 'arm' - elif host_arch.startswith('aarch64'): - host_arch = 'arm64' - elif host_arch.startswith('mips'): - host_arch = 'mips' - elif host_arch.startswith('ppc'): - host_arch = 'ppc' - elif host_arch.startswith('s390'): - host_arch = 's390' + # Convert machine type to format recognized by gyp. + if re.match(r'i.86', host_arch) or host_arch == 'i86pc': + host_arch = 'ia32' + elif host_arch in ['x86_64', 'amd64']: + host_arch = 'x64' + elif host_arch.startswith('arm'): + host_arch = 'arm' + elif host_arch.startswith('aarch64'): + host_arch = 'arm64' + elif host_arch.startswith('mips'): + host_arch = 'mips' + elif host_arch.startswith('ppc'): + host_arch = 'ppc' + elif host_arch.startswith('s390'): + host_arch = 's390' + # platform.machine is based on running kernel. It's possible to use 64-bit + # kernel with 32-bit userland, e.g. to give linker slightly more memory. + # Distinguish between different userland bitness by querying + # the python binary. + if host_arch == 'x64' and platform.architecture()[0] == '32bit': + host_arch = 'ia32' + if host_arch == 'arm64' and platform.architecture()[0] == '32bit': + host_arch = 'arm' - # platform.machine is based on running kernel. It's possible to use 64-bit - # kernel with 32-bit userland, e.g. to give linker slightly more memory. - # Distinguish between different userland bitness by querying - # the python binary. - if host_arch == 'x64' and platform.architecture()[0] == '32bit': - host_arch = 'ia32' - if host_arch == 'arm64' and platform.architecture()[0] == '32bit': - host_arch = 'arm' - - return host_arch + return host_arch def DoMain(_): - """Hook to be called from gyp without starting a separate python + """Hook to be called from gyp without starting a separate python interpreter.""" - return HostArch() + return HostArch() + + if __name__ == '__main__': - print DoMain([]) + print DoMain([]) diff --git a/build/gn_helpers.py b/build/gn_helpers.py index 9a94abff6ca..3acb32990ba 100644 --- a/build/gn_helpers.py +++ b/build/gn_helpers.py @@ -1,39 +1,40 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Helper functions useful when writing scripts that are run from GN's exec_script function.""" + class GNException(Exception): - pass + pass -def ToGNString(value, allow_dicts = True): - """Prints the given value to stdout. +def ToGNString(value, allow_dicts=True): + """Prints the given value to stdout. allow_dicts indicates if this function will allow converting dictionaries to GN scopes. This is only possible at the top level, you can't nest a GN scope in a list, so this should be set to False for recursive calls.""" - if isinstance(value, str) or isinstance(value, unicode): - if value.find('\n') >= 0: - raise GNException("Trying to print a string with a newline in it.") - return '"' + value.replace('"', '\\"') + '"' + if isinstance(value, str) or isinstance(value, unicode): + if value.find('\n') >= 0: + raise GNException("Trying to print a string with a newline in it.") + return '"' + value.replace('"', '\\"') + '"' - if isinstance(value, list): - return '[ %s ]' % ', '.join(ToGNString(v) for v in value) + if isinstance(value, list): + return '[ %s ]' % ', '.join(ToGNString(v) for v in value) - if isinstance(value, dict): - if not allow_dicts: - raise GNException("Attempting to recursively print a dictionary.") - result = "" - for key in value: - if not isinstance(key, str): - raise GNException("Dictionary key is not a string.") - result += "%s = %s\n" % (key, ToGNString(value[key], False)) - return result + if isinstance(value, dict): + if not allow_dicts: + raise GNException("Attempting to recursively print a dictionary.") + result = "" + for key in value: + if not isinstance(key, str): + raise GNException("Dictionary key is not a string.") + result += "%s = %s\n" % (key, ToGNString(value[key], False)) + return result - if isinstance(value, int): - return str(value) + if isinstance(value, int): + return str(value) - raise GNException("Unsupported type %s (value %s) when printing to GN." % (type(value), value)) + raise GNException("Unsupported type %s (value %s) when printing to GN." % + (type(value), value)) diff --git a/build/gn_run_binary.py b/build/gn_run_binary.py index 718bf240974..0c2287dc5e7 100755 --- a/build/gn_run_binary.py +++ b/build/gn_run_binary.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Helper script for GN to run an arbitrary binary. See compiled_action.gni. Run with: @@ -18,47 +17,50 @@ import os import sys import subprocess + # Run a command, swallowing the output unless there is an error. def run_command(command): - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - return 0 - except subprocess.CalledProcessError as e: - return ("Command failed: " + ' '.join(command) + "\n" + - "output: " + e.output) - except OSError as e: - return ("Command failed: " + ' '.join(command) + "\n" + - "output: " + e.strerror) + try: + subprocess.check_output(command, stderr=subprocess.STDOUT) + return 0 + except subprocess.CalledProcessError as e: + return ("Command failed: " + ' '.join(command) + "\n" + "output: " + + e.output) + except OSError as e: + return ("Command failed: " + ' '.join(command) + "\n" + "output: " + + e.strerror) + def main(argv): - error_exit = 0 - if argv[1] == "compiled_action": - error_exit = 1 - elif argv[1] != "exec_script": - print ("The first argument should be either " - "'compiled_action' or 'exec_script") - return 1 + error_exit = 0 + if argv[1] == "compiled_action": + error_exit = 1 + elif argv[1] != "exec_script": + print("The first argument should be either " + "'compiled_action' or 'exec_script") + return 1 - # Unless the path is absolute, this script is designed to run binaries - # produced by the current build. We always prefix it with "./" to avoid - # picking up system versions that might also be on the path. - if os.path.isabs(argv[2]): - path = argv[2] - else: - path = './' + argv[2] + # Unless the path is absolute, this script is designed to run binaries + # produced by the current build. We always prefix it with "./" to avoid + # picking up system versions that might also be on the path. + if os.path.isabs(argv[2]): + path = argv[2] + else: + path = './' + argv[2] - if not os.path.isfile(path): - print ("Binary not found: " + path) - return error_exit + if not os.path.isfile(path): + print("Binary not found: " + path) + return error_exit - # The rest of the arguments are passed directly to the executable. - args = [path] + argv[3:] + # The rest of the arguments are passed directly to the executable. + args = [path] + argv[3:] + + result = run_command(args) + if result != 0: + print(result) + return error_exit + return 0 - result = run_command(args) - if result != 0: - print (result) - return error_exit - return 0 if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/build/linux/sysroot_scripts/install-sysroot.py b/build/linux/sysroot_scripts/install-sysroot.py index 304824d91bc..1fe69987afe 100755 --- a/build/linux/sysroot_scripts/install-sysroot.py +++ b/build/linux/sysroot_scripts/install-sysroot.py @@ -2,7 +2,6 @@ # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Install Debian sysroots for building chromium. """ @@ -32,7 +31,6 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.dirname(os.path.dirname(SCRIPT_DIR))) import detect_host_arch - URL_PREFIX = 'https://commondatastorage.googleapis.com' URL_PATH = 'chrome-linux-sysroot/toolchain' @@ -40,132 +38,136 @@ VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64', 'mips') class Error(Exception): - pass + pass def GetSha1(filename): - sha1 = hashlib.sha1() - with open(filename, 'rb') as f: - while True: - # Read in 1mb chunks, so it doesn't all have to be loaded into memory. - chunk = f.read(1024*1024) - if not chunk: - break - sha1.update(chunk) - return sha1.hexdigest() + sha1 = hashlib.sha1() + with open(filename, 'rb') as f: + while True: + # Read in 1mb chunks, so it doesn't all have to be loaded into memory. + chunk = f.read(1024 * 1024) + if not chunk: + break + sha1.update(chunk) + return sha1.hexdigest() def DetectHostArch(): - # Figure out host arch using build/detect_host_arch.py and - # set target_arch to host arch - detected_host_arch = detect_host_arch.HostArch() - if detected_host_arch == 'x64': - return 'amd64' - elif detected_host_arch == 'ia32': - return 'i386' - elif detected_host_arch == 'arm': - return 'arm' - elif detected_host_arch == 'arm64': - return 'arm64' - elif detected_host_arch == 'mips': - return 'mips' - elif detected_host_arch == 'ppc': - return 'ppc' - elif detected_host_arch == 's390': - return 's390' + # Figure out host arch using build/detect_host_arch.py and + # set target_arch to host arch + detected_host_arch = detect_host_arch.HostArch() + if detected_host_arch == 'x64': + return 'amd64' + elif detected_host_arch == 'ia32': + return 'i386' + elif detected_host_arch == 'arm': + return 'arm' + elif detected_host_arch == 'arm64': + return 'arm64' + elif detected_host_arch == 'mips': + return 'mips' + elif detected_host_arch == 'ppc': + return 'ppc' + elif detected_host_arch == 's390': + return 's390' - raise Error('Unrecognized host arch: %s' % detected_host_arch) + raise Error('Unrecognized host arch: %s' % detected_host_arch) def main(args): - parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__) - parser.add_option('--arch', type='choice', choices=VALID_ARCHS, - help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS)) - options, _ = parser.parse_args(args) - if not sys.platform.startswith('linux'): + parser = optparse.OptionParser( + 'usage: %prog [OPTIONS]', description=__doc__) + parser.add_option( + '--arch', + type='choice', + choices=VALID_ARCHS, + help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS)) + options, _ = parser.parse_args(args) + if not sys.platform.startswith('linux'): + return 0 + + if not options.arch: + print 'You much specify either --arch or --running-as-hook' + return 1 + InstallDefaultSysrootForArch(options.arch) + return 0 - if not options.arch: - print 'You much specify either --arch or --running-as-hook' - return 1 - InstallDefaultSysrootForArch(options.arch) - - return 0 - def InstallDefaultSysrootForArch(target_arch): - if target_arch == 'amd64': - InstallSysroot('Jessie', 'amd64') - elif target_arch == 'arm': - InstallSysroot('Jessie', 'arm') - elif target_arch == 'arm64': - InstallSysroot('Jessie', 'arm64') - elif target_arch == 'i386': - InstallSysroot('Jessie', 'i386') - elif target_arch == 'mips': - InstallSysroot('Jessie', 'mips') - else: - raise Error('Unknown architecture: %s' % target_arch) + if target_arch == 'amd64': + InstallSysroot('Jessie', 'amd64') + elif target_arch == 'arm': + InstallSysroot('Jessie', 'arm') + elif target_arch == 'arm64': + InstallSysroot('Jessie', 'arm64') + elif target_arch == 'i386': + InstallSysroot('Jessie', 'i386') + elif target_arch == 'mips': + InstallSysroot('Jessie', 'mips') + else: + raise Error('Unknown architecture: %s' % target_arch) def InstallSysroot(target_platform, target_arch): - # The sysroot directory should match the one specified in build/common.gypi. - # TODO(thestig) Consider putting this elsewhere to avoid having to recreate - # it on every build. - linux_dir = os.path.dirname(SCRIPT_DIR) + # The sysroot directory should match the one specified in build/common.gypi. + # TODO(thestig) Consider putting this elsewhere to avoid having to recreate + # it on every build. + linux_dir = os.path.dirname(SCRIPT_DIR) - sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json') - sysroots = json.load(open(sysroots_file)) - sysroot_key = '%s_%s' % (target_platform.lower(), target_arch) - if sysroot_key not in sysroots: - raise Error('No sysroot for: %s %s' % (target_platform, target_arch)) - sysroot_dict = sysroots[sysroot_key] - revision = sysroot_dict['Revision'] - tarball_filename = sysroot_dict['Tarball'] - tarball_sha1sum = sysroot_dict['Sha1Sum'] - sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir']) + sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json') + sysroots = json.load(open(sysroots_file)) + sysroot_key = '%s_%s' % (target_platform.lower(), target_arch) + if sysroot_key not in sysroots: + raise Error('No sysroot for: %s %s' % (target_platform, target_arch)) + sysroot_dict = sysroots[sysroot_key] + revision = sysroot_dict['Revision'] + tarball_filename = sysroot_dict['Tarball'] + tarball_sha1sum = sysroot_dict['Sha1Sum'] + sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir']) - url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename) + url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename) - stamp = os.path.join(sysroot, '.stamp') - if os.path.exists(stamp): - with open(stamp) as s: - if s.read() == url: - return + stamp = os.path.join(sysroot, '.stamp') + if os.path.exists(stamp): + with open(stamp) as s: + if s.read() == url: + return - print 'Installing Debian %s %s root image: %s' % \ - (target_platform, target_arch, sysroot) - if os.path.isdir(sysroot): - shutil.rmtree(sysroot) - os.mkdir(sysroot) - tarball = os.path.join(sysroot, tarball_filename) - print 'Downloading %s' % url - sys.stdout.flush() - sys.stderr.flush() - for _ in range(3): - try: - response = urllib2.urlopen(url) - with open(tarball, "wb") as f: - f.write(response.read()) - break - except: - pass - else: - raise Error('Failed to download %s' % url) - sha1sum = GetSha1(tarball) - if sha1sum != tarball_sha1sum: - raise Error('Tarball sha1sum is wrong.' - 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)) - subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot]) - os.remove(tarball) + print 'Installing Debian %s %s root image: %s' % \ + (target_platform, target_arch, sysroot) + if os.path.isdir(sysroot): + shutil.rmtree(sysroot) + os.mkdir(sysroot) + tarball = os.path.join(sysroot, tarball_filename) + print 'Downloading %s' % url + sys.stdout.flush() + sys.stderr.flush() + for _ in range(3): + try: + response = urllib2.urlopen(url) + with open(tarball, "wb") as f: + f.write(response.read()) + break + except: + pass + else: + raise Error('Failed to download %s' % url) + sha1sum = GetSha1(tarball) + if sha1sum != tarball_sha1sum: + raise Error('Tarball sha1sum is wrong.' + 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)) + subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot]) + os.remove(tarball) - with open(stamp, 'w') as s: - s.write(url) + with open(stamp, 'w') as s: + s.write(url) if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except Error as e: - sys.stderr.write(str(e) + '\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except Error as e: + sys.stderr.write(str(e) + '\n') + sys.exit(1) diff --git a/build/mac/change_mach_o_flags.py b/build/mac/change_mach_o_flags.py index c2aeaec9b10..427117ce72e 100755 --- a/build/mac/change_mach_o_flags.py +++ b/build/mac/change_mach_o_flags.py @@ -2,7 +2,6 @@ # Copyright (c) 2011 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Usage: change_mach_o_flags.py [--executable-heap] [--no-pie] Arranges for the executable at |executable_path| to have its data (heap) @@ -78,7 +77,6 @@ import os import struct import sys - # FAT_MAGIC = 0xcafebabe FAT_CIGAM = 0xbebafeca @@ -94,86 +92,86 @@ MH_NO_HEAP_EXECUTION = 0x01000000 class MachOError(Exception): - """A class for exceptions thrown by this module.""" + """A class for exceptions thrown by this module.""" - pass + pass def CheckedSeek(file, offset): - """Seeks the file-like object at |file| to offset |offset| and raises a + """Seeks the file-like object at |file| to offset |offset| and raises a MachOError if anything funny happens.""" - file.seek(offset, os.SEEK_SET) - new_offset = file.tell() - if new_offset != offset: - raise MachOError, \ - 'seek: expected offset %d, observed %d' % (offset, new_offset) + file.seek(offset, os.SEEK_SET) + new_offset = file.tell() + if new_offset != offset: + raise MachOError, \ + 'seek: expected offset %d, observed %d' % (offset, new_offset) def CheckedRead(file, count): - """Reads |count| bytes from the file-like |file| object, raising a + """Reads |count| bytes from the file-like |file| object, raising a MachOError if any other number of bytes is read.""" - bytes = file.read(count) - if len(bytes) != count: - raise MachOError, \ - 'read: expected length %d, observed %d' % (count, len(bytes)) + bytes = file.read(count) + if len(bytes) != count: + raise MachOError, \ + 'read: expected length %d, observed %d' % (count, len(bytes)) - return bytes + return bytes def ReadUInt32(file, endian): - """Reads an unsinged 32-bit integer from the file-like |file| object, + """Reads an unsinged 32-bit integer from the file-like |file| object, treating it as having endianness specified by |endian| (per the |struct| module), and returns it as a number. Raises a MachOError if the proper length of data can't be read from |file|.""" - bytes = CheckedRead(file, 4) + bytes = CheckedRead(file, 4) - (uint32,) = struct.unpack(endian + 'I', bytes) - return uint32 + (uint32,) = struct.unpack(endian + 'I', bytes) + return uint32 def ReadMachHeader(file, endian): - """Reads an entire |mach_header| structure () from the + """Reads an entire |mach_header| structure () from the file-like |file| object, treating it as having endianness specified by |endian| (per the |struct| module), and returns a 7-tuple of its members as numbers. Raises a MachOError if the proper length of data can't be read from |file|.""" - bytes = CheckedRead(file, 28) + bytes = CheckedRead(file, 28) - magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = \ - struct.unpack(endian + '7I', bytes) - return magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags + magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = \ + struct.unpack(endian + '7I', bytes) + return magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags def ReadFatArch(file): - """Reads an entire |fat_arch| structure () from the file-like + """Reads an entire |fat_arch| structure () from the file-like |file| object, treating it as having endianness specified by |endian| (per the |struct| module), and returns a 5-tuple of its members as numbers. Raises a MachOError if the proper length of data can't be read from |file|.""" - bytes = CheckedRead(file, 20) + bytes = CheckedRead(file, 20) - cputype, cpusubtype, offset, size, align = struct.unpack('>5I', bytes) - return cputype, cpusubtype, offset, size, align + cputype, cpusubtype, offset, size, align = struct.unpack('>5I', bytes) + return cputype, cpusubtype, offset, size, align def WriteUInt32(file, uint32, endian): - """Writes |uint32| as an unsinged 32-bit integer to the file-like |file| + """Writes |uint32| as an unsinged 32-bit integer to the file-like |file| object, treating it as having endianness specified by |endian| (per the |struct| module).""" - bytes = struct.pack(endian + 'I', uint32) - assert len(bytes) == 4 + bytes = struct.pack(endian + 'I', uint32) + assert len(bytes) == 4 - file.write(bytes) + file.write(bytes) def HandleMachOFile(file, options, offset=0): - """Seeks the file-like |file| object to |offset|, reads its |mach_header|, + """Seeks the file-like |file| object to |offset|, reads its |mach_header|, and rewrites the header's |flags| field if appropriate. The header's endianness is detected. Both 32-bit and 64-bit Mach-O headers are supported (mach_header and mach_header_64). Raises MachOError if used on a header that @@ -182,92 +180,98 @@ def HandleMachOFile(file, options, offset=0): according to |options| and written to |file| if any changes need to be made. If already set or clear as specified by |options|, nothing is written.""" - CheckedSeek(file, offset) - magic = ReadUInt32(file, '<') - if magic == MH_MAGIC or magic == MH_MAGIC_64: - endian = '<' - elif magic == MH_CIGAM or magic == MH_CIGAM_64: - endian = '>' - else: - raise MachOError, \ - 'Mach-O file at offset %d has illusion of magic' % offset + CheckedSeek(file, offset) + magic = ReadUInt32(file, '<') + if magic == MH_MAGIC or magic == MH_MAGIC_64: + endian = '<' + elif magic == MH_CIGAM or magic == MH_CIGAM_64: + endian = '>' + else: + raise MachOError, \ + 'Mach-O file at offset %d has illusion of magic' % offset - CheckedSeek(file, offset) - magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = \ - ReadMachHeader(file, endian) - assert magic == MH_MAGIC or magic == MH_MAGIC_64 - if filetype != MH_EXECUTE: - raise MachOError, \ - 'Mach-O file at offset %d is type 0x%x, expected MH_EXECUTE' % \ - (offset, filetype) + CheckedSeek(file, offset) + magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = \ + ReadMachHeader(file, endian) + assert magic == MH_MAGIC or magic == MH_MAGIC_64 + if filetype != MH_EXECUTE: + raise MachOError, \ + 'Mach-O file at offset %d is type 0x%x, expected MH_EXECUTE' % \ + (offset, filetype) - original_flags = flags + original_flags = flags - if options.no_heap_execution: - flags |= MH_NO_HEAP_EXECUTION - else: - flags &= ~MH_NO_HEAP_EXECUTION + if options.no_heap_execution: + flags |= MH_NO_HEAP_EXECUTION + else: + flags &= ~MH_NO_HEAP_EXECUTION - if options.pie: - flags |= MH_PIE - else: - flags &= ~MH_PIE + if options.pie: + flags |= MH_PIE + else: + flags &= ~MH_PIE - if flags != original_flags: - CheckedSeek(file, offset + 24) - WriteUInt32(file, flags, endian) + if flags != original_flags: + CheckedSeek(file, offset + 24) + WriteUInt32(file, flags, endian) def HandleFatFile(file, options, fat_offset=0): - """Seeks the file-like |file| object to |offset| and loops over its + """Seeks the file-like |file| object to |offset| and loops over its |fat_header| entries, calling HandleMachOFile for each.""" - CheckedSeek(file, fat_offset) - magic = ReadUInt32(file, '>') - assert magic == FAT_MAGIC + CheckedSeek(file, fat_offset) + magic = ReadUInt32(file, '>') + assert magic == FAT_MAGIC - nfat_arch = ReadUInt32(file, '>') + nfat_arch = ReadUInt32(file, '>') - for index in xrange(0, nfat_arch): - cputype, cpusubtype, offset, size, align = ReadFatArch(file) - assert size >= 28 + for index in xrange(0, nfat_arch): + cputype, cpusubtype, offset, size, align = ReadFatArch(file) + assert size >= 28 - # HandleMachOFile will seek around. Come back here after calling it, in - # case it sought. - fat_arch_offset = file.tell() - HandleMachOFile(file, options, offset) - CheckedSeek(file, fat_arch_offset) + # HandleMachOFile will seek around. Come back here after calling it, in + # case it sought. + fat_arch_offset = file.tell() + HandleMachOFile(file, options, offset) + CheckedSeek(file, fat_arch_offset) def main(me, args): - parser = optparse.OptionParser('%prog [options] ') - parser.add_option('--executable-heap', action='store_false', - dest='no_heap_execution', default=True, - help='Clear the MH_NO_HEAP_EXECUTION bit') - parser.add_option('--no-pie', action='store_false', - dest='pie', default=True, - help='Clear the MH_PIE bit') - (options, loose_args) = parser.parse_args(args) - if len(loose_args) != 1: - parser.print_usage() - return 1 + parser = optparse.OptionParser('%prog [options] ') + parser.add_option( + '--executable-heap', + action='store_false', + dest='no_heap_execution', + default=True, + help='Clear the MH_NO_HEAP_EXECUTION bit') + parser.add_option( + '--no-pie', + action='store_false', + dest='pie', + default=True, + help='Clear the MH_PIE bit') + (options, loose_args) = parser.parse_args(args) + if len(loose_args) != 1: + parser.print_usage() + return 1 - executable_path = loose_args[0] - executable_file = open(executable_path, 'rb+') + executable_path = loose_args[0] + executable_file = open(executable_path, 'rb+') - magic = ReadUInt32(executable_file, '<') - if magic == FAT_CIGAM: - # Check FAT_CIGAM and not FAT_MAGIC because the read was little-endian. - HandleFatFile(executable_file, options) - elif magic == MH_MAGIC or magic == MH_CIGAM or \ - magic == MH_MAGIC_64 or magic == MH_CIGAM_64: - HandleMachOFile(executable_file, options) - else: - raise MachOError, '%s is not a Mach-O or fat file' % executable_file + magic = ReadUInt32(executable_file, '<') + if magic == FAT_CIGAM: + # Check FAT_CIGAM and not FAT_MAGIC because the read was little-endian. + HandleFatFile(executable_file, options) + elif magic == MH_MAGIC or magic == MH_CIGAM or \ + magic == MH_MAGIC_64 or magic == MH_CIGAM_64: + HandleMachOFile(executable_file, options) + else: + raise MachOError, '%s is not a Mach-O or fat file' % executable_file - executable_file.close() - return 0 + executable_file.close() + return 0 if __name__ == '__main__': - sys.exit(main(sys.argv[0], sys.argv[1:])) + sys.exit(main(sys.argv[0], sys.argv[1:])) diff --git a/build/mac/find_sdk.py b/build/mac/find_sdk.py index 0534766e807..3e7bbc16969 100755 --- a/build/mac/find_sdk.py +++ b/build/mac/find_sdk.py @@ -2,7 +2,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Prints the lowest locally available SDK version greater than or equal to a given minimum sdk version to standard output. @@ -15,76 +14,91 @@ import re import subprocess import sys - from optparse import OptionParser def parse_version(version_str): - """'10.6' => [10, 6]""" - return map(int, re.findall(r'(\d+)', version_str)) + """'10.6' => [10, 6]""" + return map(int, re.findall(r'(\d+)', version_str)) def main(): - parser = OptionParser() - parser.add_option("--verify", - action="store_true", dest="verify", default=False, - help="return the sdk argument and warn if it doesn't exist") - parser.add_option("--sdk_path", - action="store", type="string", dest="sdk_path", default="", - help="user-specified SDK path; bypasses verification") - parser.add_option("--print_sdk_path", - action="store_true", dest="print_sdk_path", default=False, - help="Additionaly print the path the SDK (appears first).") - (options, args) = parser.parse_args() - min_sdk_version = args[0] + parser = OptionParser() + parser.add_option( + "--verify", + action="store_true", + dest="verify", + default=False, + help="return the sdk argument and warn if it doesn't exist") + parser.add_option( + "--sdk_path", + action="store", + type="string", + dest="sdk_path", + default="", + help="user-specified SDK path; bypasses verification") + parser.add_option( + "--print_sdk_path", + action="store_true", + dest="print_sdk_path", + default=False, + help="Additionaly print the path the SDK (appears first).") + (options, args) = parser.parse_args() + min_sdk_version = args[0] - job = subprocess.Popen(['xcode-select', '-print-path'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out, err = job.communicate() - if job.returncode != 0: - print >> sys.stderr, out - print >> sys.stderr, err - raise Exception(('Error %d running xcode-select, you might have to run ' - '|sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer| ' - 'if you are using Xcode 4.') % job.returncode) - # The Developer folder moved in Xcode 4.3. - xcode43_sdk_path = os.path.join( - out.rstrip(), 'Platforms/MacOSX.platform/Developer/SDKs') - if os.path.isdir(xcode43_sdk_path): - sdk_dir = xcode43_sdk_path - else: - sdk_dir = os.path.join(out.rstrip(), 'SDKs') - sdks = [re.findall('^MacOSX(10\.\d+)\.sdk$', s) for s in os.listdir(sdk_dir)] - sdks = [s[0] for s in sdks if s] # [['10.5'], ['10.6']] => ['10.5', '10.6'] - sdks = [s for s in sdks # ['10.5', '10.6'] => ['10.6'] - if parse_version(s) >= parse_version(min_sdk_version)] - if not sdks: - raise Exception('No %s+ SDK found' % min_sdk_version) - best_sdk = sorted(sdks, key=parse_version)[0] + job = subprocess.Popen(['xcode-select', '-print-path'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, err = job.communicate() + if job.returncode != 0: + print >> sys.stderr, out + print >> sys.stderr, err + raise Exception(( + 'Error %d running xcode-select, you might have to run ' + '|sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer| ' + 'if you are using Xcode 4.') % job.returncode) + # The Developer folder moved in Xcode 4.3. + xcode43_sdk_path = os.path.join(out.rstrip(), + 'Platforms/MacOSX.platform/Developer/SDKs') + if os.path.isdir(xcode43_sdk_path): + sdk_dir = xcode43_sdk_path + else: + sdk_dir = os.path.join(out.rstrip(), 'SDKs') + sdks = [ + re.findall('^MacOSX(10\.\d+)\.sdk$', s) for s in os.listdir(sdk_dir) + ] + sdks = [s[0] for s in sdks if s] # [['10.5'], ['10.6']] => ['10.5', '10.6'] + sdks = [ + s for s in sdks # ['10.5', '10.6'] => ['10.6'] + if parse_version(s) >= parse_version(min_sdk_version) + ] + if not sdks: + raise Exception('No %s+ SDK found' % min_sdk_version) + best_sdk = sorted(sdks, key=parse_version)[0] - if options.verify and best_sdk != min_sdk_version and not options.sdk_path: - print >> sys.stderr, '' - print >> sys.stderr, ' vvvvvvv' - print >> sys.stderr, '' - print >> sys.stderr, \ - 'This build requires the %s SDK, but it was not found on your system.' \ - % min_sdk_version - print >> sys.stderr, \ - 'Either install it, or explicitly set mac_sdk in your GYP_DEFINES.' - print >> sys.stderr, '' - print >> sys.stderr, ' ^^^^^^^' - print >> sys.stderr, '' - return min_sdk_version + if options.verify and best_sdk != min_sdk_version and not options.sdk_path: + print >> sys.stderr, '' + print >> sys.stderr, ' vvvvvvv' + print >> sys.stderr, '' + print >> sys.stderr, \ + 'This build requires the %s SDK, but it was not found on your system.' \ + % min_sdk_version + print >> sys.stderr, \ + 'Either install it, or explicitly set mac_sdk in your GYP_DEFINES.' + print >> sys.stderr, '' + print >> sys.stderr, ' ^^^^^^^' + print >> sys.stderr, '' + return min_sdk_version - if options.print_sdk_path: - print subprocess.check_output(['xcodebuild', '-version', '-sdk', - 'macosx' + best_sdk, 'Path']).strip() + if options.print_sdk_path: + print subprocess.check_output( + ['xcodebuild', '-version', '-sdk', 'macosx' + best_sdk, + 'Path']).strip() - return best_sdk + return best_sdk if __name__ == '__main__': - if sys.platform != 'darwin': - raise Exception("This script only runs on Mac") - print main() + if sys.platform != 'darwin': + raise Exception("This script only runs on Mac") + print main() diff --git a/build/mac/tweak_info_plist.py b/build/mac/tweak_info_plist.py index 2057bac8386..b2a0f0c8225 100755 --- a/build/mac/tweak_info_plist.py +++ b/build/mac/tweak_info_plist.py @@ -33,248 +33,278 @@ TOP = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) def _GetOutput(args): - """Runs a subprocess and waits for termination. Returns (stdout, returncode) + """Runs a subprocess and waits for termination. Returns (stdout, returncode) of the process. stderr is attached to the parent.""" - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - (stdout, stderr) = proc.communicate() - return (stdout, proc.returncode) + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + return (stdout, proc.returncode) def _GetOutputNoError(args): - """Similar to _GetOutput() but ignores stderr. If there's an error launching + """Similar to _GetOutput() but ignores stderr. If there's an error launching the child (like file not found), the exception will be caught and (None, 1) will be returned to mimic quiet failure.""" - try: - proc = subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError: - return (None, 1) - (stdout, stderr) = proc.communicate() - return (stdout, proc.returncode) + try: + proc = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + return (None, 1) + (stdout, stderr) = proc.communicate() + return (stdout, proc.returncode) def _RemoveKeys(plist, *keys): - """Removes a varargs of keys from the plist.""" - for key in keys: - try: - del plist[key] - except KeyError: - pass + """Removes a varargs of keys from the plist.""" + for key in keys: + try: + del plist[key] + except KeyError: + pass def _AddVersionKeys(plist, version=None): - """Adds the product version number into the plist. Returns True on success and + """Adds the product version number into the plist. Returns True on success and False on error. The error will be printed to stderr.""" - if version: - match = re.match('\d+\.\d+\.(\d+\.\d+)$', version) - if not match: - print >>sys.stderr, 'Invalid version string specified: "%s"' % version - return False + if version: + match = re.match('\d+\.\d+\.(\d+\.\d+)$', version) + if not match: + print >> sys.stderr, 'Invalid version string specified: "%s"' % version + return False - full_version = match.group(0) - bundle_version = match.group(1) + full_version = match.group(0) + bundle_version = match.group(1) - else: - # Pull in the Chrome version number. - VERSION_TOOL = os.path.join(TOP, 'build/util/version.py') - VERSION_FILE = os.path.join(TOP, 'chrome/VERSION') + else: + # Pull in the Chrome version number. + VERSION_TOOL = os.path.join(TOP, 'build/util/version.py') + VERSION_FILE = os.path.join(TOP, 'chrome/VERSION') - (stdout, retval1) = _GetOutput([VERSION_TOOL, '-f', VERSION_FILE, '-t', - '@MAJOR@.@MINOR@.@BUILD@.@PATCH@']) - full_version = stdout.rstrip() + (stdout, retval1) = _GetOutput([ + VERSION_TOOL, '-f', VERSION_FILE, '-t', + '@MAJOR@.@MINOR@.@BUILD@.@PATCH@' + ]) + full_version = stdout.rstrip() - (stdout, retval2) = _GetOutput([VERSION_TOOL, '-f', VERSION_FILE, '-t', - '@BUILD@.@PATCH@']) - bundle_version = stdout.rstrip() + (stdout, retval2) = _GetOutput( + [VERSION_TOOL, '-f', VERSION_FILE, '-t', '@BUILD@.@PATCH@']) + bundle_version = stdout.rstrip() - # If either of the two version commands finished with non-zero returncode, - # report the error up. - if retval1 or retval2: - return False + # If either of the two version commands finished with non-zero returncode, + # report the error up. + if retval1 or retval2: + return False - # Add public version info so "Get Info" works. - plist['CFBundleShortVersionString'] = full_version + # Add public version info so "Get Info" works. + plist['CFBundleShortVersionString'] = full_version - # Honor the 429496.72.95 limit. The maximum comes from splitting 2^32 - 1 - # into 6, 2, 2 digits. The limitation was present in Tiger, but it could - # have been fixed in later OS release, but hasn't been tested (it's easy - # enough to find out with "lsregister -dump). - # http://lists.apple.com/archives/carbon-dev/2006/Jun/msg00139.html - # BUILD will always be an increasing value, so BUILD_PATH gives us something - # unique that meetings what LS wants. - plist['CFBundleVersion'] = bundle_version + # Honor the 429496.72.95 limit. The maximum comes from splitting 2^32 - 1 + # into 6, 2, 2 digits. The limitation was present in Tiger, but it could + # have been fixed in later OS release, but hasn't been tested (it's easy + # enough to find out with "lsregister -dump). + # http://lists.apple.com/archives/carbon-dev/2006/Jun/msg00139.html + # BUILD will always be an increasing value, so BUILD_PATH gives us something + # unique that meetings what LS wants. + plist['CFBundleVersion'] = bundle_version - # Return with no error. - return True + # Return with no error. + return True def _DoSCMKeys(plist, add_keys): - """Adds the SCM information, visible in about:version, to property list. If + """Adds the SCM information, visible in about:version, to property list. If |add_keys| is True, it will insert the keys, otherwise it will remove them.""" - scm_revision = None - if add_keys: - # Pull in the Chrome revision number. - VERSION_TOOL = os.path.join(TOP, 'build/util/version.py') - LASTCHANGE_FILE = os.path.join(TOP, 'build/util/LASTCHANGE') - (stdout, retval) = _GetOutput([VERSION_TOOL, '-f', LASTCHANGE_FILE, '-t', - '@LASTCHANGE@']) - if retval: - return False - scm_revision = stdout.rstrip() + scm_revision = None + if add_keys: + # Pull in the Chrome revision number. + VERSION_TOOL = os.path.join(TOP, 'build/util/version.py') + LASTCHANGE_FILE = os.path.join(TOP, 'build/util/LASTCHANGE') + (stdout, retval) = _GetOutput( + [VERSION_TOOL, '-f', LASTCHANGE_FILE, '-t', '@LASTCHANGE@']) + if retval: + return False + scm_revision = stdout.rstrip() - # See if the operation failed. - _RemoveKeys(plist, 'SCMRevision') - if scm_revision != None: - plist['SCMRevision'] = scm_revision - elif add_keys: - print >>sys.stderr, 'Could not determine SCM revision. This may be OK.' + # See if the operation failed. + _RemoveKeys(plist, 'SCMRevision') + if scm_revision != None: + plist['SCMRevision'] = scm_revision + elif add_keys: + print >> sys.stderr, 'Could not determine SCM revision. This may be OK.' - return True + return True def _AddBreakpadKeys(plist, branding): - """Adds the Breakpad keys. This must be called AFTER _AddVersionKeys() and + """Adds the Breakpad keys. This must be called AFTER _AddVersionKeys() and also requires the |branding| argument.""" - plist['BreakpadReportInterval'] = '3600' # Deliberately a string. - plist['BreakpadProduct'] = '%s_Mac' % branding - plist['BreakpadProductDisplay'] = branding - plist['BreakpadVersion'] = plist['CFBundleShortVersionString'] - # These are both deliberately strings and not boolean. - plist['BreakpadSendAndExit'] = 'YES' - plist['BreakpadSkipConfirm'] = 'YES' + plist['BreakpadReportInterval'] = '3600' # Deliberately a string. + plist['BreakpadProduct'] = '%s_Mac' % branding + plist['BreakpadProductDisplay'] = branding + plist['BreakpadVersion'] = plist['CFBundleShortVersionString'] + # These are both deliberately strings and not boolean. + plist['BreakpadSendAndExit'] = 'YES' + plist['BreakpadSkipConfirm'] = 'YES' def _RemoveBreakpadKeys(plist): - """Removes any set Breakpad keys.""" - _RemoveKeys(plist, - 'BreakpadURL', - 'BreakpadReportInterval', - 'BreakpadProduct', - 'BreakpadProductDisplay', - 'BreakpadVersion', - 'BreakpadSendAndExit', - 'BreakpadSkipConfirm') + """Removes any set Breakpad keys.""" + _RemoveKeys(plist, 'BreakpadURL', 'BreakpadReportInterval', + 'BreakpadProduct', 'BreakpadProductDisplay', 'BreakpadVersion', + 'BreakpadSendAndExit', 'BreakpadSkipConfirm') def _TagSuffixes(): - # Keep this list sorted in the order that tag suffix components are to - # appear in a tag value. That is to say, it should be sorted per ASCII. - components = ('32bit', 'full') - assert tuple(sorted(components)) == components + # Keep this list sorted in the order that tag suffix components are to + # appear in a tag value. That is to say, it should be sorted per ASCII. + components = ('32bit', 'full') + assert tuple(sorted(components)) == components - components_len = len(components) - combinations = 1 << components_len - tag_suffixes = [] - for combination in xrange(0, combinations): - tag_suffix = '' - for component_index in xrange(0, components_len): - if combination & (1 << component_index): - tag_suffix += '-' + components[component_index] - tag_suffixes.append(tag_suffix) - return tag_suffixes + components_len = len(components) + combinations = 1 << components_len + tag_suffixes = [] + for combination in xrange(0, combinations): + tag_suffix = '' + for component_index in xrange(0, components_len): + if combination & (1 << component_index): + tag_suffix += '-' + components[component_index] + tag_suffixes.append(tag_suffix) + return tag_suffixes def _AddKeystoneKeys(plist, bundle_identifier): - """Adds the Keystone keys. This must be called AFTER _AddVersionKeys() and + """Adds the Keystone keys. This must be called AFTER _AddVersionKeys() and also requires the |bundle_identifier| argument (com.example.product).""" - plist['KSVersion'] = plist['CFBundleShortVersionString'] - plist['KSProductID'] = bundle_identifier - plist['KSUpdateURL'] = 'https://tools.google.com/service/update2' + plist['KSVersion'] = plist['CFBundleShortVersionString'] + plist['KSProductID'] = bundle_identifier + plist['KSUpdateURL'] = 'https://tools.google.com/service/update2' - _RemoveKeys(plist, 'KSChannelID') - for tag_suffix in _TagSuffixes(): - if tag_suffix: - plist['KSChannelID' + tag_suffix] = tag_suffix + _RemoveKeys(plist, 'KSChannelID') + for tag_suffix in _TagSuffixes(): + if tag_suffix: + plist['KSChannelID' + tag_suffix] = tag_suffix def _RemoveKeystoneKeys(plist): - """Removes any set Keystone keys.""" - _RemoveKeys(plist, - 'KSVersion', - 'KSProductID', - 'KSUpdateURL') + """Removes any set Keystone keys.""" + _RemoveKeys(plist, 'KSVersion', 'KSProductID', 'KSUpdateURL') - tag_keys = [] - for tag_suffix in _TagSuffixes(): - tag_keys.append('KSChannelID' + tag_suffix) - _RemoveKeys(plist, *tag_keys) + tag_keys = [] + for tag_suffix in _TagSuffixes(): + tag_keys.append('KSChannelID' + tag_suffix) + _RemoveKeys(plist, *tag_keys) def Main(argv): - parser = optparse.OptionParser('%prog [options]') - parser.add_option('--breakpad', dest='use_breakpad', action='store', - type='int', default=False, help='Enable Breakpad [1 or 0]') - parser.add_option('--breakpad_uploads', dest='breakpad_uploads', - action='store', type='int', default=False, - help='Enable Breakpad\'s uploading of crash dumps [1 or 0]') - parser.add_option('--keystone', dest='use_keystone', action='store', - type='int', default=False, help='Enable Keystone [1 or 0]') - parser.add_option('--scm', dest='add_scm_info', action='store', type='int', - default=True, help='Add SCM metadata [1 or 0]') - parser.add_option('--branding', dest='branding', action='store', - type='string', default=None, help='The branding of the binary') - parser.add_option('--bundle_id', dest='bundle_identifier', - action='store', type='string', default=None, - help='The bundle id of the binary') - parser.add_option('--version', dest='version', action='store', type='string', - default=None, help='The version string [major.minor.build.patch]') - (options, args) = parser.parse_args(argv) + parser = optparse.OptionParser('%prog [options]') + parser.add_option( + '--breakpad', + dest='use_breakpad', + action='store', + type='int', + default=False, + help='Enable Breakpad [1 or 0]') + parser.add_option( + '--breakpad_uploads', + dest='breakpad_uploads', + action='store', + type='int', + default=False, + help='Enable Breakpad\'s uploading of crash dumps [1 or 0]') + parser.add_option( + '--keystone', + dest='use_keystone', + action='store', + type='int', + default=False, + help='Enable Keystone [1 or 0]') + parser.add_option( + '--scm', + dest='add_scm_info', + action='store', + type='int', + default=True, + help='Add SCM metadata [1 or 0]') + parser.add_option( + '--branding', + dest='branding', + action='store', + type='string', + default=None, + help='The branding of the binary') + parser.add_option( + '--bundle_id', + dest='bundle_identifier', + action='store', + type='string', + default=None, + help='The bundle id of the binary') + parser.add_option( + '--version', + dest='version', + action='store', + type='string', + default=None, + help='The version string [major.minor.build.patch]') + (options, args) = parser.parse_args(argv) - if len(args) > 0: - print >>sys.stderr, parser.get_usage() - return 1 + if len(args) > 0: + print >> sys.stderr, parser.get_usage() + return 1 - # Read the plist into its parsed format. - DEST_INFO_PLIST = os.path.join(env['TARGET_BUILD_DIR'], env['INFOPLIST_PATH']) - plist = plistlib.readPlist(DEST_INFO_PLIST) + # Read the plist into its parsed format. + DEST_INFO_PLIST = os.path.join(env['TARGET_BUILD_DIR'], + env['INFOPLIST_PATH']) + plist = plistlib.readPlist(DEST_INFO_PLIST) - # Insert the product version. - if not _AddVersionKeys(plist, version=options.version): - return 2 + # Insert the product version. + if not _AddVersionKeys(plist, version=options.version): + return 2 - # Add Breakpad if configured to do so. - if options.use_breakpad: - if options.branding is None: - print >>sys.stderr, 'Use of Breakpad requires branding.' - return 1 - _AddBreakpadKeys(plist, options.branding) - if options.breakpad_uploads: - plist['BreakpadURL'] = 'https://clients2.google.com/cr/report' + # Add Breakpad if configured to do so. + if options.use_breakpad: + if options.branding is None: + print >> sys.stderr, 'Use of Breakpad requires branding.' + return 1 + _AddBreakpadKeys(plist, options.branding) + if options.breakpad_uploads: + plist['BreakpadURL'] = 'https://clients2.google.com/cr/report' + else: + # This allows crash dumping to a file without uploading the + # dump, for testing purposes. Breakpad does not recognise + # "none" as a special value, but this does stop crash dump + # uploading from happening. We need to specify something + # because if "BreakpadURL" is not present, Breakpad will not + # register its crash handler and no crash dumping will occur. + plist['BreakpadURL'] = 'none' else: - # This allows crash dumping to a file without uploading the - # dump, for testing purposes. Breakpad does not recognise - # "none" as a special value, but this does stop crash dump - # uploading from happening. We need to specify something - # because if "BreakpadURL" is not present, Breakpad will not - # register its crash handler and no crash dumping will occur. - plist['BreakpadURL'] = 'none' - else: - _RemoveBreakpadKeys(plist) + _RemoveBreakpadKeys(plist) - # Only add Keystone in Release builds. - if options.use_keystone and env['CONFIGURATION'] == 'Release': - if options.bundle_identifier is None: - print >>sys.stderr, 'Use of Keystone requires the bundle id.' - return 1 - _AddKeystoneKeys(plist, options.bundle_identifier) - else: - _RemoveKeystoneKeys(plist) + # Only add Keystone in Release builds. + if options.use_keystone and env['CONFIGURATION'] == 'Release': + if options.bundle_identifier is None: + print >> sys.stderr, 'Use of Keystone requires the bundle id.' + return 1 + _AddKeystoneKeys(plist, options.bundle_identifier) + else: + _RemoveKeystoneKeys(plist) - # Adds or removes any SCM keys. - if not _DoSCMKeys(plist, options.add_scm_info): - return 3 + # Adds or removes any SCM keys. + if not _DoSCMKeys(plist, options.add_scm_info): + return 3 - # Now that all keys have been mutated, rewrite the file. - temp_info_plist = tempfile.NamedTemporaryFile() - plistlib.writePlist(plist, temp_info_plist.name) + # Now that all keys have been mutated, rewrite the file. + temp_info_plist = tempfile.NamedTemporaryFile() + plistlib.writePlist(plist, temp_info_plist.name) - # Info.plist will work perfectly well in any plist format, but traditionally - # applications use xml1 for this, so convert it to ensure that it's valid. - proc = subprocess.Popen(['plutil', '-convert', 'xml1', '-o', DEST_INFO_PLIST, - temp_info_plist.name]) - proc.wait() - return proc.returncode + # Info.plist will work perfectly well in any plist format, but traditionally + # applications use xml1 for this, so convert it to ensure that it's valid. + proc = subprocess.Popen([ + 'plutil', '-convert', 'xml1', '-o', DEST_INFO_PLIST, + temp_info_plist.name + ]) + proc.wait() + return proc.returncode if __name__ == '__main__': - sys.exit(Main(sys.argv[1:])) + sys.exit(Main(sys.argv[1:])) diff --git a/build/toolchain/get_concurrent_links.py b/build/toolchain/get_concurrent_links.py index 6a401017eb7..45e3ff52f87 100644 --- a/build/toolchain/get_concurrent_links.py +++ b/build/toolchain/get_concurrent_links.py @@ -10,55 +10,58 @@ import re import subprocess import sys + def GetDefaultConcurrentLinks(): - # Inherit the legacy environment variable for people that have set it in GYP. - pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) - if pool_size: - return pool_size + # Inherit the legacy environment variable for people that have set it in GYP. + pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0)) + if pool_size: + return pool_size - if sys.platform in ('win32', 'cygwin'): - import ctypes + if sys.platform in ('win32', 'cygwin'): + import ctypes - class MEMORYSTATUSEX(ctypes.Structure): - _fields_ = [ - ("dwLength", ctypes.c_ulong), - ("dwMemoryLoad", ctypes.c_ulong), - ("ullTotalPhys", ctypes.c_ulonglong), - ("ullAvailPhys", ctypes.c_ulonglong), - ("ullTotalPageFile", ctypes.c_ulonglong), - ("ullAvailPageFile", ctypes.c_ulonglong), - ("ullTotalVirtual", ctypes.c_ulonglong), - ("ullAvailVirtual", ctypes.c_ulonglong), - ("sullAvailExtendedVirtual", ctypes.c_ulonglong), - ] + class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("sullAvailExtendedVirtual", ctypes.c_ulonglong), + ] - stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) - ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) + stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) + + mem_limit = max(1, stat.ullTotalPhys / (4 * (2**30))) # total / 4GB + hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) + return min(mem_limit, hard_cap) + elif sys.platform.startswith('linux'): + if os.path.exists("/proc/meminfo"): + with open("/proc/meminfo") as meminfo: + memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') + for line in meminfo: + match = memtotal_re.match(line) + if not match: + continue + # Allow 8Gb per link on Linux because Gold is quite memory hungry + return max(1, int(match.group(1)) / (8 * (2**20))) + return 1 + elif sys.platform == 'darwin': + try: + avail_bytes = int( + subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) + # A static library debug build of Chromium's unit_tests takes ~2.7GB, so + # 4GB per ld process allows for some more bloat. + return max(1, avail_bytes / (4 * (2**30))) # total / 4GB + except Exception: + return 1 + else: + # TODO(scottmg): Implement this for other platforms. + return 1 - mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30))) # total / 4GB - hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32))) - return min(mem_limit, hard_cap) - elif sys.platform.startswith('linux'): - if os.path.exists("/proc/meminfo"): - with open("/proc/meminfo") as meminfo: - memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') - for line in meminfo: - match = memtotal_re.match(line) - if not match: - continue - # Allow 8Gb per link on Linux because Gold is quite memory hungry - return max(1, int(match.group(1)) / (8 * (2 ** 20))) - return 1 - elif sys.platform == 'darwin': - try: - avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) - # A static library debug build of Chromium's unit_tests takes ~2.7GB, so - # 4GB per ld process allows for some more bloat. - return max(1, avail_bytes / (4 * (2 ** 30))) # total / 4GB - except Exception: - return 1 - else: - # TODO(scottmg): Implement this for other platforms. - return 1 print GetDefaultConcurrentLinks() diff --git a/build/toolchain/win/setup_toolchain.py b/build/toolchain/win/setup_toolchain.py index 8150d3a4e4a..fde043b210a 100644 --- a/build/toolchain/win/setup_toolchain.py +++ b/build/toolchain/win/setup_toolchain.py @@ -22,195 +22,205 @@ import gn_helpers SCRIPT_DIR = os.path.dirname(__file__) + def _ExtractImportantEnvironment(output_of_set): - """Extracts environment variables required for the toolchain to run from + """Extracts environment variables required for the toolchain to run from a textual dump output by the cmd.exe 'set' command.""" - envvars_to_save = ( - 'goma_.*', # TODO(scottmg): This is ugly, but needed for goma. - 'include', - 'lib', - 'libpath', - 'path', - 'pathext', - 'systemroot', - 'temp', - 'tmp', - ) - env = {} - # This occasionally happens and leads to misleading SYSTEMROOT error messages - # if not caught here. - if output_of_set.count('=') == 0: - raise Exception('Invalid output_of_set. Value is:\n%s' % output_of_set) - for line in output_of_set.splitlines(): - for envvar in envvars_to_save: - if re.match(envvar + '=', line.lower()): - var, setting = line.split('=', 1) - if envvar == 'path': - # Our own rules and actions in Chromium rely on python being in the - # path. Add the path to this python here so that if it's not in the - # path when ninja is run later, python will still be found. - setting = os.path.dirname(sys.executable) + os.pathsep + setting - env[var.upper()] = setting.lower() - break - if sys.platform in ('win32', 'cygwin'): - for required in ('SYSTEMROOT', 'TEMP', 'TMP'): - if required not in env: - raise Exception('Environment variable "%s" ' - 'required to be set to valid path' % required) - return env + envvars_to_save = ( + 'goma_.*', # TODO(scottmg): This is ugly, but needed for goma. + 'include', + 'lib', + 'libpath', + 'path', + 'pathext', + 'systemroot', + 'temp', + 'tmp', + ) + env = {} + # This occasionally happens and leads to misleading SYSTEMROOT error messages + # if not caught here. + if output_of_set.count('=') == 0: + raise Exception('Invalid output_of_set. Value is:\n%s' % output_of_set) + for line in output_of_set.splitlines(): + for envvar in envvars_to_save: + if re.match(envvar + '=', line.lower()): + var, setting = line.split('=', 1) + if envvar == 'path': + # Our own rules and actions in Chromium rely on python being in the + # path. Add the path to this python here so that if it's not in the + # path when ninja is run later, python will still be found. + setting = os.path.dirname( + sys.executable) + os.pathsep + setting + env[var.upper()] = setting.lower() + break + if sys.platform in ('win32', 'cygwin'): + for required in ('SYSTEMROOT', 'TEMP', 'TMP'): + if required not in env: + raise Exception('Environment variable "%s" ' + 'required to be set to valid path' % required) + return env def _DetectVisualStudioPath(): - """Return path to the GYP_MSVS_VERSION of Visual Studio. + """Return path to the GYP_MSVS_VERSION of Visual Studio. """ - # Use the code in build/vs_toolchain.py to avoid duplicating code. - chromium_dir = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) - sys.path.append(os.path.join(chromium_dir, 'build')) - import vs_toolchain - return vs_toolchain.DetectVisualStudioPath() + # Use the code in build/vs_toolchain.py to avoid duplicating code. + chromium_dir = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) + sys.path.append(os.path.join(chromium_dir, 'build')) + import vs_toolchain + return vs_toolchain.DetectVisualStudioPath() def _LoadEnvFromBat(args): - """Given a bat command, runs it and returns env vars set by it.""" - args = args[:] - args.extend(('&&', 'set')) - popen = subprocess.Popen( - args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - variables, _ = popen.communicate() - if popen.returncode != 0: - raise Exception('"%s" failed with error %d' % (args, popen.returncode)) - return variables + """Given a bat command, runs it and returns env vars set by it.""" + args = args[:] + args.extend(('&&', 'set')) + popen = subprocess.Popen( + args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + variables, _ = popen.communicate() + if popen.returncode != 0: + raise Exception('"%s" failed with error %d' % (args, popen.returncode)) + return variables def _LoadToolchainEnv(cpu, sdk_dir): - """Returns a dictionary with environment variables that must be set while + """Returns a dictionary with environment variables that must be set while running binaries from the toolchain (e.g. INCLUDE and PATH for cl.exe).""" - # Check if we are running in the SDK command line environment and use - # the setup script from the SDK if so. |cpu| should be either - # 'x86' or 'x64'. - assert cpu in ('x86', 'x64') - if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and sdk_dir: - # Load environment from json file. - env = os.path.normpath(os.path.join(sdk_dir, 'bin/SetEnv.%s.json' % cpu)) - env = json.load(open(env))['env'] - for k in env: - entries = [os.path.join(*([os.path.join(sdk_dir, 'bin')] + e)) - for e in env[k]] - # clang-cl wants INCLUDE to be ;-separated even on non-Windows, - # lld-link wants LIB to be ;-separated even on non-Windows. Path gets :. - # The separator for INCLUDE here must match the one used in main() below. - sep = os.pathsep if k == 'PATH' else ';' - env[k] = sep.join(entries) - # PATH is a bit of a special case, it's in addition to the current PATH. - env['PATH'] = env['PATH'] + os.pathsep + os.environ['PATH'] - # Augment with the current env to pick up TEMP and friends. - for k in os.environ: - if k not in env: - env[k] = os.environ[k] + # Check if we are running in the SDK command line environment and use + # the setup script from the SDK if so. |cpu| should be either + # 'x86' or 'x64'. + assert cpu in ('x86', 'x64') + if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and sdk_dir: + # Load environment from json file. + env = os.path.normpath( + os.path.join(sdk_dir, 'bin/SetEnv.%s.json' % cpu)) + env = json.load(open(env))['env'] + for k in env: + entries = [ + os.path.join(*([os.path.join(sdk_dir, 'bin')] + e)) + for e in env[k] + ] + # clang-cl wants INCLUDE to be ;-separated even on non-Windows, + # lld-link wants LIB to be ;-separated even on non-Windows. Path gets :. + # The separator for INCLUDE here must match the one used in main() below. + sep = os.pathsep if k == 'PATH' else ';' + env[k] = sep.join(entries) + # PATH is a bit of a special case, it's in addition to the current PATH. + env['PATH'] = env['PATH'] + os.pathsep + os.environ['PATH'] + # Augment with the current env to pick up TEMP and friends. + for k in os.environ: + if k not in env: + env[k] = os.environ[k] - varlines = [] - for k in sorted(env.keys()): - varlines.append('%s=%s' % (str(k), str(env[k]))) - variables = '\n'.join(varlines) + varlines = [] + for k in sorted(env.keys()): + varlines.append('%s=%s' % (str(k), str(env[k]))) + variables = '\n'.join(varlines) - # Check that the json file contained the same environment as the .cmd file. - if sys.platform in ('win32', 'cygwin'): - script = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.cmd')) - assert _ExtractImportantEnvironment(variables) == \ - _ExtractImportantEnvironment(_LoadEnvFromBat([script, '/' + cpu])) - else: - if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ: - os.environ['GYP_MSVS_OVERRIDE_PATH'] = _DetectVisualStudioPath() - # We only support x64-hosted tools. - script_path = os.path.normpath(os.path.join( - os.environ['GYP_MSVS_OVERRIDE_PATH'], - 'VC/vcvarsall.bat')) - if not os.path.exists(script_path): - # vcvarsall.bat for VS 2017 fails if run after running vcvarsall.bat from - # VS 2013 or VS 2015. Fix this by clearing the vsinstalldir environment - # variable. - if 'VSINSTALLDIR' in os.environ: - del os.environ['VSINSTALLDIR'] - other_path = os.path.normpath(os.path.join( - os.environ['GYP_MSVS_OVERRIDE_PATH'], - 'VC/Auxiliary/Build/vcvarsall.bat')) - if not os.path.exists(other_path): - raise Exception('%s is missing - make sure VC++ tools are installed.' % - script_path) - script_path = other_path - # Chromium requires the 10.0.14393.0 SDK or higher - previous versions don't - # have all of the required declarations. - args = [script_path, 'amd64_x86' if cpu == 'x86' else 'amd64'] - variables = _LoadEnvFromBat(args) - return _ExtractImportantEnvironment(variables) + # Check that the json file contained the same environment as the .cmd file. + if sys.platform in ('win32', 'cygwin'): + script = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.cmd')) + assert _ExtractImportantEnvironment(variables) == \ + _ExtractImportantEnvironment(_LoadEnvFromBat([script, '/' + cpu])) + else: + if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ: + os.environ['GYP_MSVS_OVERRIDE_PATH'] = _DetectVisualStudioPath() + # We only support x64-hosted tools. + script_path = os.path.normpath( + os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], + 'VC/vcvarsall.bat')) + if not os.path.exists(script_path): + # vcvarsall.bat for VS 2017 fails if run after running vcvarsall.bat from + # VS 2013 or VS 2015. Fix this by clearing the vsinstalldir environment + # variable. + if 'VSINSTALLDIR' in os.environ: + del os.environ['VSINSTALLDIR'] + other_path = os.path.normpath( + os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], + 'VC/Auxiliary/Build/vcvarsall.bat')) + if not os.path.exists(other_path): + raise Exception( + '%s is missing - make sure VC++ tools are installed.' % + script_path) + script_path = other_path + # Chromium requires the 10.0.14393.0 SDK or higher - previous versions don't + # have all of the required declarations. + args = [script_path, 'amd64_x86' if cpu == 'x86' else 'amd64'] + variables = _LoadEnvFromBat(args) + return _ExtractImportantEnvironment(variables) def _FormatAsEnvironmentBlock(envvar_dict): - """Format as an 'environment block' directly suitable for CreateProcess. + """Format as an 'environment block' directly suitable for CreateProcess. Briefly this is a list of key=value\0, terminated by an additional \0. See CreateProcess documentation for more details.""" - block = '' - nul = '\0' - for key, value in envvar_dict.iteritems(): - block += key + '=' + value + nul - block += nul - return block + block = '' + nul = '\0' + for key, value in envvar_dict.iteritems(): + block += key + '=' + value + nul + block += nul + return block def main(): - if len(sys.argv) != 5: - print('Usage setup_toolchain.py ' - ' ' - ' ') - sys.exit(2) - win_sdk_path = sys.argv[2] - runtime_dirs = sys.argv[3] - target_cpu = sys.argv[4] + if len(sys.argv) != 5: + print('Usage setup_toolchain.py ' + ' ' + ' ') + sys.exit(2) + win_sdk_path = sys.argv[2] + runtime_dirs = sys.argv[3] + target_cpu = sys.argv[4] - cpus = ('x86', 'x64') - assert target_cpu in cpus - vc_bin_dir = '' - include = '' + cpus = ('x86', 'x64') + assert target_cpu in cpus + vc_bin_dir = '' + include = '' - # TODO(scottmg|goma): Do we need an equivalent of - # ninja_use_custom_environment_files? + # TODO(scottmg|goma): Do we need an equivalent of + # ninja_use_custom_environment_files? - for cpu in cpus: - # Extract environment variables for subprocesses. - env = _LoadToolchainEnv(cpu, win_sdk_path) - env['PATH'] = runtime_dirs + os.pathsep + env['PATH'] + for cpu in cpus: + # Extract environment variables for subprocesses. + env = _LoadToolchainEnv(cpu, win_sdk_path) + env['PATH'] = runtime_dirs + os.pathsep + env['PATH'] - if cpu == target_cpu: - for path in env['PATH'].split(os.pathsep): - if os.path.exists(os.path.join(path, 'cl.exe')): - vc_bin_dir = os.path.realpath(path) - break - # The separator for INCLUDE here must match the one used in - # _LoadToolchainEnv() above. - include = [p.replace('"', r'\"') for p in env['INCLUDE'].split(';') if p] - include_I = ' '.join(['"/I' + i + '"' for i in include]) - include_imsvc = ' '.join(['"-imsvc' + i + '"' for i in include]) + if cpu == target_cpu: + for path in env['PATH'].split(os.pathsep): + if os.path.exists(os.path.join(path, 'cl.exe')): + vc_bin_dir = os.path.realpath(path) + break + # The separator for INCLUDE here must match the one used in + # _LoadToolchainEnv() above. + include = [ + p.replace('"', r'\"') for p in env['INCLUDE'].split(';') if p + ] + include_I = ' '.join(['"/I' + i + '"' for i in include]) + include_imsvc = ' '.join(['"-imsvc' + i + '"' for i in include]) - env_block = _FormatAsEnvironmentBlock(env) - with open('environment.' + cpu, 'wb') as f: - f.write(env_block) + env_block = _FormatAsEnvironmentBlock(env) + with open('environment.' + cpu, 'wb') as f: + f.write(env_block) - # Create a store app version of the environment. - if 'LIB' in env: - env['LIB'] = env['LIB'] .replace(r'\VC\LIB', r'\VC\LIB\STORE') - if 'LIBPATH' in env: - env['LIBPATH'] = env['LIBPATH'].replace(r'\VC\LIB', r'\VC\LIB\STORE') - env_block = _FormatAsEnvironmentBlock(env) - with open('environment.winrt_' + cpu, 'wb') as f: - f.write(env_block) + # Create a store app version of the environment. + if 'LIB' in env: + env['LIB'] = env['LIB'].replace(r'\VC\LIB', r'\VC\LIB\STORE') + if 'LIBPATH' in env: + env['LIBPATH'] = env['LIBPATH'].replace(r'\VC\LIB', + r'\VC\LIB\STORE') + env_block = _FormatAsEnvironmentBlock(env) + with open('environment.winrt_' + cpu, 'wb') as f: + f.write(env_block) + + assert vc_bin_dir + print 'vc_bin_dir = ' + gn_helpers.ToGNString(vc_bin_dir) + assert include_I + print 'include_flags_I = ' + gn_helpers.ToGNString(include_I) + assert include_imsvc + print 'include_flags_imsvc = ' + gn_helpers.ToGNString(include_imsvc) - assert vc_bin_dir - print 'vc_bin_dir = ' + gn_helpers.ToGNString(vc_bin_dir) - assert include_I - print 'include_flags_I = ' + gn_helpers.ToGNString(include_I) - assert include_imsvc - print 'include_flags_imsvc = ' + gn_helpers.ToGNString(include_imsvc) if __name__ == '__main__': - main() + main() diff --git a/build/toolchain/win/tool_wrapper.py b/build/toolchain/win/tool_wrapper.py index 4d749ef70a7..fbe924801bb 100644 --- a/build/toolchain/win/tool_wrapper.py +++ b/build/toolchain/win/tool_wrapper.py @@ -1,7 +1,6 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Utility functions for Windows builds. This file is copied to the build directory as part of toolchain setup and @@ -22,203 +21,218 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) # link.exe. _LINK_EXE_OUT_ARG = re.compile('/OUT:(?P.+)$', re.IGNORECASE) + def main(args): - executor = WinTool() - exit_code = executor.Dispatch(args) - if exit_code is not None: - sys.exit(exit_code) + executor = WinTool() + exit_code = executor.Dispatch(args) + if exit_code is not None: + sys.exit(exit_code) class WinTool(object): - """This class performs all the Windows tooling steps. The methods can either + """This class performs all the Windows tooling steps. The methods can either be executed directly, or dispatched from an argument list.""" - def _UseSeparateMspdbsrv(self, env, args): - """Allows to use a unique instance of mspdbsrv.exe per linker instead of a + def _UseSeparateMspdbsrv(self, env, args): + """Allows to use a unique instance of mspdbsrv.exe per linker instead of a shared one.""" - if len(args) < 1: - raise Exception("Not enough arguments") + if len(args) < 1: + raise Exception("Not enough arguments") - if args[0] != 'link.exe': - return + if args[0] != 'link.exe': + return - # Use the output filename passed to the linker to generate an endpoint name - # for mspdbsrv.exe. - endpoint_name = None - for arg in args: - m = _LINK_EXE_OUT_ARG.match(arg) - if m: - endpoint_name = re.sub(r'\W+', '', - '%s_%d' % (m.group('out'), os.getpid())) - break + # Use the output filename passed to the linker to generate an endpoint name + # for mspdbsrv.exe. + endpoint_name = None + for arg in args: + m = _LINK_EXE_OUT_ARG.match(arg) + if m: + endpoint_name = re.sub(r'\W+', '', + '%s_%d' % (m.group('out'), os.getpid())) + break - if endpoint_name is None: - return + if endpoint_name is None: + return - # Adds the appropriate environment variable. This will be read by link.exe - # to know which instance of mspdbsrv.exe it should connect to (if it's - # not set then the default endpoint is used). - env['_MSPDBSRV_ENDPOINT_'] = endpoint_name + # Adds the appropriate environment variable. This will be read by link.exe + # to know which instance of mspdbsrv.exe it should connect to (if it's + # not set then the default endpoint is used). + env['_MSPDBSRV_ENDPOINT_'] = endpoint_name - def Dispatch(self, args): - """Dispatches a string command to a method.""" - if len(args) < 1: - raise Exception("Not enough arguments") + def Dispatch(self, args): + """Dispatches a string command to a method.""" + if len(args) < 1: + raise Exception("Not enough arguments") - method = "Exec%s" % self._CommandifyName(args[0]) - return getattr(self, method)(*args[1:]) + method = "Exec%s" % self._CommandifyName(args[0]) + return getattr(self, method)(*args[1:]) - def _CommandifyName(self, name_string): - """Transforms a tool name like recursive-mirror to RecursiveMirror.""" - return name_string.title().replace('-', '') + def _CommandifyName(self, name_string): + """Transforms a tool name like recursive-mirror to RecursiveMirror.""" + return name_string.title().replace('-', '') - def _GetEnv(self, arch): - """Gets the saved environment from a file for a given architecture.""" - # The environment is saved as an "environment block" (see CreateProcess - # and msvs_emulation for details). We convert to a dict here. - # Drop last 2 NULs, one for list terminator, one for trailing vs. separator. - pairs = open(arch).read()[:-2].split('\0') - kvs = [item.split('=', 1) for item in pairs] - return dict(kvs) + def _GetEnv(self, arch): + """Gets the saved environment from a file for a given architecture.""" + # The environment is saved as an "environment block" (see CreateProcess + # and msvs_emulation for details). We convert to a dict here. + # Drop last 2 NULs, one for list terminator, one for trailing vs. separator. + pairs = open(arch).read()[:-2].split('\0') + kvs = [item.split('=', 1) for item in pairs] + return dict(kvs) - def ExecStamp(self, path): - """Simple stamp command.""" - open(path, 'w').close() + def ExecStamp(self, path): + """Simple stamp command.""" + open(path, 'w').close() - def ExecDeleteFile(self, path): - """Simple file delete command.""" - if os.path.exists(path): - os.unlink(path) + def ExecDeleteFile(self, path): + """Simple file delete command.""" + if os.path.exists(path): + os.unlink(path) - def ExecRecursiveMirror(self, source, dest): - """Emulation of rm -rf out && cp -af in out.""" - if os.path.exists(dest): - if os.path.isdir(dest): - def _on_error(fn, path, dummy_excinfo): - # The operation failed, possibly because the file is set to - # read-only. If that's why, make it writable and try the op again. - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWRITE) - fn(path) - shutil.rmtree(dest, onerror=_on_error) - else: - if not os.access(dest, os.W_OK): - # Attempt to make the file writable before deleting it. - os.chmod(dest, stat.S_IWRITE) - os.unlink(dest) + def ExecRecursiveMirror(self, source, dest): + """Emulation of rm -rf out && cp -af in out.""" + if os.path.exists(dest): + if os.path.isdir(dest): - if os.path.isdir(source): - shutil.copytree(source, dest) - else: - shutil.copy2(source, dest) - # Try to diagnose crbug.com/741603 - if not os.path.exists(dest): - raise Exception("Copying of %s to %s failed" % (source, dest)) + def _on_error(fn, path, dummy_excinfo): + # The operation failed, possibly because the file is set to + # read-only. If that's why, make it writable and try the op again. + if not os.access(path, os.W_OK): + os.chmod(path, stat.S_IWRITE) + fn(path) - def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): - """Filter diagnostic output from link that looks like: + shutil.rmtree(dest, onerror=_on_error) + else: + if not os.access(dest, os.W_OK): + # Attempt to make the file writable before deleting it. + os.chmod(dest, stat.S_IWRITE) + os.unlink(dest) + + if os.path.isdir(source): + shutil.copytree(source, dest) + else: + shutil.copy2(source, dest) + # Try to diagnose crbug.com/741603 + if not os.path.exists(dest): + raise Exception("Copying of %s to %s failed" % (source, dest)) + + def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args): + """Filter diagnostic output from link that looks like: ' Creating library ui.dll.lib and object ui.dll.exp' This happens when there are exports from the dll or exe. """ - env = self._GetEnv(arch) - if use_separate_mspdbsrv == 'True': - self._UseSeparateMspdbsrv(env, args) - if sys.platform == 'win32': - args = list(args) # *args is a tuple by default, which is read-only. - args[0] = args[0].replace('/', '\\') - # https://docs.python.org/2/library/subprocess.html: - # "On Unix with shell=True [...] if args is a sequence, the first item - # specifies the command string, and any additional items will be treated as - # additional arguments to the shell itself. That is to say, Popen does the - # equivalent of: - # Popen(['/bin/sh', '-c', args[0], args[1], ...])" - # For that reason, since going through the shell doesn't seem necessary on - # non-Windows don't do that there. - link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - # Read output one line at a time as it shows up to avoid OOM failures when - # GBs of output is produced. - for line in link.stdout: - if (not line.startswith(' Creating library ') and - not line.startswith('Generating code') and - not line.startswith('Finished generating code')): - print line, - return link.wait() + env = self._GetEnv(arch) + if use_separate_mspdbsrv == 'True': + self._UseSeparateMspdbsrv(env, args) + if sys.platform == 'win32': + args = list( + args) # *args is a tuple by default, which is read-only. + args[0] = args[0].replace('/', '\\') + # https://docs.python.org/2/library/subprocess.html: + # "On Unix with shell=True [...] if args is a sequence, the first item + # specifies the command string, and any additional items will be treated as + # additional arguments to the shell itself. That is to say, Popen does the + # equivalent of: + # Popen(['/bin/sh', '-c', args[0], args[1], ...])" + # For that reason, since going through the shell doesn't seem necessary on + # non-Windows don't do that there. + link = subprocess.Popen( + args, + shell=sys.platform == 'win32', + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + # Read output one line at a time as it shows up to avoid OOM failures when + # GBs of output is produced. + for line in link.stdout: + if (not line.startswith(' Creating library ') and + not line.startswith('Generating code') and + not line.startswith('Finished generating code')): + print line, + return link.wait() - def ExecMidlWrapper(self, arch, outdir, tlb, h, dlldata, iid, proxy, idl, - *flags): - """Filter noisy filenames output from MIDL compile step that isn't + def ExecMidlWrapper(self, arch, outdir, tlb, h, dlldata, iid, proxy, idl, + *flags): + """Filter noisy filenames output from MIDL compile step that isn't quietable via command line flags. """ - args = ['midl', '/nologo'] + list(flags) + [ - '/out', outdir, - '/tlb', tlb, - '/h', h, - '/dlldata', dlldata, - '/iid', iid, - '/proxy', proxy, - idl] - env = self._GetEnv(arch) - popen = subprocess.Popen(args, shell=True, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out, _ = popen.communicate() - # Filter junk out of stdout, and write filtered versions. Output we want - # to filter is pairs of lines that look like this: - # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl - # objidl.idl - lines = out.splitlines() - prefixes = ('Processing ', '64 bit Processing ') - processing = set(os.path.basename(x) - for x in lines if x.startswith(prefixes)) - for line in lines: - if not line.startswith(prefixes) and line not in processing: - print line - return popen.returncode + args = ['midl', '/nologo'] + list(flags) + [ + '/out', outdir, '/tlb', tlb, '/h', h, '/dlldata', dlldata, '/iid', + iid, '/proxy', proxy, idl + ] + env = self._GetEnv(arch) + popen = subprocess.Popen( + args, + shell=True, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = popen.communicate() + # Filter junk out of stdout, and write filtered versions. Output we want + # to filter is pairs of lines that look like this: + # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl + # objidl.idl + lines = out.splitlines() + prefixes = ('Processing ', '64 bit Processing ') + processing = set( + os.path.basename(x) for x in lines if x.startswith(prefixes)) + for line in lines: + if not line.startswith(prefixes) and line not in processing: + print line + return popen.returncode - def ExecAsmWrapper(self, arch, *args): - """Filter logo banner from invocations of asm.exe.""" - env = self._GetEnv(arch) - popen = subprocess.Popen(args, shell=True, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out, _ = popen.communicate() - for line in out.splitlines(): - # Split to avoid triggering license checks: - if (not line.startswith('Copy' + 'right (C' + - ') Microsoft Corporation') and - not line.startswith('Microsoft (R) Macro Assembler') and - not line.startswith(' Assembling: ') and - line): - print line - return popen.returncode + def ExecAsmWrapper(self, arch, *args): + """Filter logo banner from invocations of asm.exe.""" + env = self._GetEnv(arch) + popen = subprocess.Popen( + args, + shell=True, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = popen.communicate() + for line in out.splitlines(): + # Split to avoid triggering license checks: + if (not line.startswith('Copy' + 'right (C' + + ') Microsoft Corporation') and + not line.startswith('Microsoft (R) Macro Assembler') and + not line.startswith(' Assembling: ') and line): + print line + return popen.returncode - def ExecRcWrapper(self, arch, *args): - """Filter logo banner from invocations of rc.exe. Older versions of RC + def ExecRcWrapper(self, arch, *args): + """Filter logo banner from invocations of rc.exe. Older versions of RC don't support the /nologo flag.""" - env = self._GetEnv(arch) - popen = subprocess.Popen(args, shell=True, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out, _ = popen.communicate() - for line in out.splitlines(): - if (not line.startswith('Microsoft (R) Windows (R) Resource Compiler') and - not line.startswith('Copy' + 'right (C' + - ') Microsoft Corporation') and - line): - print line - return popen.returncode + env = self._GetEnv(arch) + popen = subprocess.Popen( + args, + shell=True, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + out, _ = popen.communicate() + for line in out.splitlines(): + if (not line.startswith( + 'Microsoft (R) Windows (R) Resource Compiler') and + not line.startswith('Copy' + 'right (C' + + ') Microsoft Corporation') and line): + print line + return popen.returncode - def ExecActionWrapper(self, arch, rspfile, *dirname): - """Runs an action command line from a response file using the environment + def ExecActionWrapper(self, arch, rspfile, *dirname): + """Runs an action command line from a response file using the environment for |arch|. If |dirname| is supplied, use that as the working directory.""" - env = self._GetEnv(arch) - # TODO(scottmg): This is a temporary hack to get some specific variables - # through to actions that are set after GN-time. http://crbug.com/333738. - for k, v in os.environ.iteritems(): - if k not in env: - env[k] = v - args = open(rspfile).read() - dirname = dirname[0] if dirname else None - return subprocess.call(args, shell=True, env=env, cwd=dirname) + env = self._GetEnv(arch) + # TODO(scottmg): This is a temporary hack to get some specific variables + # through to actions that are set after GN-time. http://crbug.com/333738. + for k, v in os.environ.iteritems(): + if k not in env: + env[k] = v + args = open(rspfile).read() + dirname = dirname[0] if dirname else None + return subprocess.call(args, shell=True, env=env, cwd=dirname) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/build/vs_toolchain.py b/build/vs_toolchain.py index c0f631affad..11376198979 100644 --- a/build/vs_toolchain.py +++ b/build/vs_toolchain.py @@ -27,7 +27,6 @@ import sys from gn_helpers import ToGNString - script_dir = os.path.dirname(os.path.realpath(__file__)) chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir)) SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -36,13 +35,13 @@ json_data_file = os.path.join(script_dir, 'win_toolchain.json') # VS versions are listed in descending order of priority (highest first). MSVS_VERSIONS = collections.OrderedDict([ - ('2017', '15.0'), - ('2019', '16.0'), + ('2017', '15.0'), + ('2019', '16.0'), ]) def SetEnvironmentAndGetRuntimeDllDirs(): - """Sets up os.environ to use the depot_tools VS toolchain with gyp, and + """Sets up os.environ to use the depot_tools VS toolchain with gyp, and returns the location of the VC runtime DLLs so they can be copied into the output directory after gyp generation. @@ -50,73 +49,75 @@ def SetEnvironmentAndGetRuntimeDllDirs(): generated separately because there are multiple folders for the arm64 VC runtime. """ - vs_runtime_dll_dirs = None - depot_tools_win_toolchain = \ - bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) - # When running on a non-Windows host, only do this if the SDK has explicitly - # been downloaded before (in which case json_data_file will exist). - if ((sys.platform in ('win32', 'cygwin') or os.path.exists(json_data_file)) - and depot_tools_win_toolchain): - if ShouldUpdateToolchain(): - if len(sys.argv) > 1 and sys.argv[1] == 'update': - update_result = Update() - else: - update_result = Update(no_download=True) - if update_result != 0: - raise Exception('Failed to update, error code %d.' % update_result) - with open(json_data_file, 'r') as tempf: - toolchain_data = json.load(tempf) + vs_runtime_dll_dirs = None + depot_tools_win_toolchain = \ + bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) + # When running on a non-Windows host, only do this if the SDK has explicitly + # been downloaded before (in which case json_data_file will exist). + if ((sys.platform in ('win32', 'cygwin') or os.path.exists(json_data_file)) + and depot_tools_win_toolchain): + if ShouldUpdateToolchain(): + if len(sys.argv) > 1 and sys.argv[1] == 'update': + update_result = Update() + else: + update_result = Update(no_download=True) + if update_result != 0: + raise Exception( + 'Failed to update, error code %d.' % update_result) + with open(json_data_file, 'r') as tempf: + toolchain_data = json.load(tempf) - toolchain = toolchain_data['path'] - version = toolchain_data['version'] - win_sdk = toolchain_data.get('win_sdk') - if not win_sdk: - win_sdk = toolchain_data['win8sdk'] - wdk = toolchain_data['wdk'] - # TODO(scottmg): The order unfortunately matters in these. They should be - # split into separate keys for x64/x86/arm64. (See CopyDlls call below). - # http://crbug.com/345992 - vs_runtime_dll_dirs = toolchain_data['runtime_dirs'] - # The number of runtime_dirs in the toolchain_data was two (x64/x86) but - # changed to three (x64/x86/arm64) and this code needs to handle both - # possibilities, which can change independently from this code. - if len(vs_runtime_dll_dirs) == 2: - vs_runtime_dll_dirs.append('Arm64Unused') + toolchain = toolchain_data['path'] + version = toolchain_data['version'] + win_sdk = toolchain_data.get('win_sdk') + if not win_sdk: + win_sdk = toolchain_data['win8sdk'] + wdk = toolchain_data['wdk'] + # TODO(scottmg): The order unfortunately matters in these. They should be + # split into separate keys for x64/x86/arm64. (See CopyDlls call below). + # http://crbug.com/345992 + vs_runtime_dll_dirs = toolchain_data['runtime_dirs'] + # The number of runtime_dirs in the toolchain_data was two (x64/x86) but + # changed to three (x64/x86/arm64) and this code needs to handle both + # possibilities, which can change independently from this code. + if len(vs_runtime_dll_dirs) == 2: + vs_runtime_dll_dirs.append('Arm64Unused') - os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain - os.environ['GYP_MSVS_VERSION'] = version + os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain + os.environ['GYP_MSVS_VERSION'] = version - os.environ['WINDOWSSDKDIR'] = win_sdk - os.environ['WDK_DIR'] = wdk - # Include the VS runtime in the PATH in case it's not machine-installed. - runtime_path = os.path.pathsep.join(vs_runtime_dll_dirs) - os.environ['PATH'] = runtime_path + os.path.pathsep + os.environ['PATH'] - elif sys.platform == 'win32' and not depot_tools_win_toolchain: - if not 'GYP_MSVS_OVERRIDE_PATH' in os.environ: - os.environ['GYP_MSVS_OVERRIDE_PATH'] = DetectVisualStudioPath() - if not 'GYP_MSVS_VERSION' in os.environ: - os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion() + os.environ['WINDOWSSDKDIR'] = win_sdk + os.environ['WDK_DIR'] = wdk + # Include the VS runtime in the PATH in case it's not machine-installed. + runtime_path = os.path.pathsep.join(vs_runtime_dll_dirs) + os.environ['PATH'] = runtime_path + os.path.pathsep + os.environ['PATH'] + elif sys.platform == 'win32' and not depot_tools_win_toolchain: + if not 'GYP_MSVS_OVERRIDE_PATH' in os.environ: + os.environ['GYP_MSVS_OVERRIDE_PATH'] = DetectVisualStudioPath() + if not 'GYP_MSVS_VERSION' in os.environ: + os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion() - # When using an installed toolchain these files aren't needed in the output - # directory in order to run binaries locally, but they are needed in order - # to create isolates or the mini_installer. Copying them to the output - # directory ensures that they are available when needed. - bitness = platform.architecture()[0] - # When running 64-bit python the x64 DLLs will be in System32 - # ARM64 binaries will not be available in the system directories because we - # don't build on ARM64 machines. - x64_path = 'System32' if bitness == '64bit' else 'Sysnative' - x64_path = os.path.join(os.path.expandvars('%windir%'), x64_path) - vs_runtime_dll_dirs = [x64_path, - os.path.join(os.path.expandvars('%windir%'), - 'SysWOW64'), - 'Arm64Unused'] + # When using an installed toolchain these files aren't needed in the output + # directory in order to run binaries locally, but they are needed in order + # to create isolates or the mini_installer. Copying them to the output + # directory ensures that they are available when needed. + bitness = platform.architecture()[0] + # When running 64-bit python the x64 DLLs will be in System32 + # ARM64 binaries will not be available in the system directories because we + # don't build on ARM64 machines. + x64_path = 'System32' if bitness == '64bit' else 'Sysnative' + x64_path = os.path.join(os.path.expandvars('%windir%'), x64_path) + vs_runtime_dll_dirs = [ + x64_path, + os.path.join(os.path.expandvars('%windir%'), 'SysWOW64'), + 'Arm64Unused' + ] - return vs_runtime_dll_dirs + return vs_runtime_dll_dirs def _RegistryGetValueUsingWinReg(key, value): - """Use the _winreg module to obtain the value of a registry key. + """Use the _winreg module to obtain the value of a registry key. Args: key: The registry key. @@ -125,234 +126,241 @@ def _RegistryGetValueUsingWinReg(key, value): contents of the registry key's value, or None on failure. Throws ImportError if _winreg is unavailable. """ - import _winreg - try: - root, subkey = key.split('\\', 1) - assert root == 'HKLM' # Only need HKLM for now. - with _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, subkey) as hkey: - return _winreg.QueryValueEx(hkey, value)[0] - except WindowsError: - return None + import _winreg + try: + root, subkey = key.split('\\', 1) + assert root == 'HKLM' # Only need HKLM for now. + with _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, subkey) as hkey: + return _winreg.QueryValueEx(hkey, value)[0] + except WindowsError: + return None def _RegistryGetValue(key, value): - try: - return _RegistryGetValueUsingWinReg(key, value) - except ImportError: - raise Exception('The python library _winreg not found.') + try: + return _RegistryGetValueUsingWinReg(key, value) + except ImportError: + raise Exception('The python library _winreg not found.') def GetVisualStudioVersion(): - """Return best available version of Visual Studio. + """Return best available version of Visual Studio. """ - env_version = os.environ.get('GYP_MSVS_VERSION') - if env_version: - return env_version + env_version = os.environ.get('GYP_MSVS_VERSION') + if env_version: + return env_version - supported_versions = MSVS_VERSIONS.keys() + supported_versions = MSVS_VERSIONS.keys() - # VS installed in depot_tools for Googlers - if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))): - return supported_versions[0] + # VS installed in depot_tools for Googlers + if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))): + return supported_versions[0] - # VS installed in system for external developers - supported_versions_str = ', '.join('{} ({})'.format(v,k) - for k,v in MSVS_VERSIONS.items()) - available_versions = [] - for version in supported_versions: - for path in ( - os.environ.get('vs%s_install' % version), - os.path.expandvars('%ProgramFiles(x86)%' + - '/Microsoft Visual Studio/%s' % version)): - if path and os.path.exists(path): - available_versions.append(version) - break + # VS installed in system for external developers + supported_versions_str = ', '.join( + '{} ({})'.format(v, k) for k, v in MSVS_VERSIONS.items()) + available_versions = [] + for version in supported_versions: + for path in ( + os.environ.get('vs%s_install' % version), + os.path.expandvars('%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s' % version)): + if path and os.path.exists(path): + available_versions.append(version) + break - if not available_versions: - raise Exception('No supported Visual Studio can be found.' - ' Supported versions are: %s.' % supported_versions_str) - return available_versions[0] + if not available_versions: + raise Exception('No supported Visual Studio can be found.' + ' Supported versions are: %s.' % supported_versions_str) + return available_versions[0] def DetectVisualStudioPath(): - """Return path to the GYP_MSVS_VERSION of Visual Studio. + """Return path to the GYP_MSVS_VERSION of Visual Studio. """ - # Note that this code is used from - # build/toolchain/win/setup_toolchain.py as well. - version_as_year = GetVisualStudioVersion() + # Note that this code is used from + # build/toolchain/win/setup_toolchain.py as well. + version_as_year = GetVisualStudioVersion() - # The VC++ >=2017 install location needs to be located using COM instead of - # the registry. For details see: - # https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/ - # For now we use a hardcoded default with an environment variable override. - for path in ( - os.environ.get('vs%s_install' % version_as_year), - os.path.expandvars('%ProgramFiles(x86)%' + - '/Microsoft Visual Studio/%s/Enterprise' % - version_as_year), - os.path.expandvars('%ProgramFiles(x86)%' + - '/Microsoft Visual Studio/%s/Professional' % - version_as_year), - os.path.expandvars('%ProgramFiles(x86)%' + - '/Microsoft Visual Studio/%s/Community' % - version_as_year), - os.path.expandvars('%ProgramFiles(x86)%' + - '/Microsoft Visual Studio/%s/Preview' % - version_as_year)): - if path and os.path.exists(path): - return path + # The VC++ >=2017 install location needs to be located using COM instead of + # the registry. For details see: + # https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/ + # For now we use a hardcoded default with an environment variable override. + for path in ( + os.environ.get('vs%s_install' % version_as_year), + os.path.expandvars( + '%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Enterprise' % version_as_year), + os.path.expandvars( + '%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Professional' % version_as_year), + os.path.expandvars( + '%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Community' % version_as_year), + os.path.expandvars( + '%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Preview' % version_as_year)): + if path and os.path.exists(path): + return path - raise Exception('Visual Studio Version %s (from GYP_MSVS_VERSION)' - ' not found.' % version_as_year) + raise Exception('Visual Studio Version %s (from GYP_MSVS_VERSION)' + ' not found.' % version_as_year) def _CopyRuntimeImpl(target, source, verbose=True): - """Copy |source| to |target| if it doesn't already exist or if it needs to be + """Copy |source| to |target| if it doesn't already exist or if it needs to be updated (comparing last modified time as an approximate float match as for some reason the values tend to differ by ~1e-07 despite being copies of the same file... https://crbug.com/603603). """ - if (os.path.isdir(os.path.dirname(target)) and - (not os.path.isfile(target) or - abs(os.stat(target).st_mtime - os.stat(source).st_mtime) >= 0.01)): - if verbose: - print('Copying %s to %s...' % (source, target)) - if os.path.exists(target): - # Make the file writable so that we can delete it now, and keep it - # readable. - os.chmod(target, stat.S_IWRITE | stat.S_IREAD) - os.unlink(target) - shutil.copy2(source, target) - # Make the file writable so that we can overwrite or delete it later, - # keep it readable. - os.chmod(target, stat.S_IWRITE | stat.S_IREAD) + if (os.path.isdir(os.path.dirname(target)) and + (not os.path.isfile(target) or + abs(os.stat(target).st_mtime - os.stat(source).st_mtime) >= 0.01)): + if verbose: + print('Copying %s to %s...' % (source, target)) + if os.path.exists(target): + # Make the file writable so that we can delete it now, and keep it + # readable. + os.chmod(target, stat.S_IWRITE | stat.S_IREAD) + os.unlink(target) + shutil.copy2(source, target) + # Make the file writable so that we can overwrite or delete it later, + # keep it readable. + os.chmod(target, stat.S_IWRITE | stat.S_IREAD) + def _SortByHighestVersionNumberFirst(list_of_str_versions): - """This sorts |list_of_str_versions| according to version number rules + """This sorts |list_of_str_versions| according to version number rules so that version "1.12" is higher than version "1.9". Does not work with non-numeric versions like 1.4.a8 which will be higher than 1.4.a12. It does handle the versions being embedded in file paths. """ - def to_int_if_int(x): - try: - return int(x) - except ValueError: - return x - def to_number_sequence(x): - part_sequence = re.split(r'[\\/\.]', x) - return [to_int_if_int(x) for x in part_sequence] + def to_int_if_int(x): + try: + return int(x) + except ValueError: + return x + + def to_number_sequence(x): + part_sequence = re.split(r'[\\/\.]', x) + return [to_int_if_int(x) for x in part_sequence] + + list_of_str_versions.sort(key=to_number_sequence, reverse=True) - list_of_str_versions.sort(key=to_number_sequence, reverse=True) def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix): - """Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't + """Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't exist, but the target directory does exist.""" - if target_cpu == 'arm64': - # Windows ARM64 VCRuntime is located at {toolchain_root}/VC/Redist/MSVC/ - # {x.y.z}/[debug_nonredist/]arm64/Microsoft.VC141.CRT/. - vc_redist_root = FindVCRedistRoot() - if suffix.startswith('.'): - source_dir = os.path.join(vc_redist_root, - 'arm64', 'Microsoft.VC141.CRT') - else: - source_dir = os.path.join(vc_redist_root, 'debug_nonredist', - 'arm64', 'Microsoft.VC141.DebugCRT') - for file_part in ('msvcp', 'vccorlib', 'vcruntime'): - dll = dll_pattern % file_part - target = os.path.join(target_dir, dll) - source = os.path.join(source_dir, dll) - _CopyRuntimeImpl(target, source) - # Copy the UCRT files from the Windows SDK. This location includes the - # api-ms-win-crt-*.dll files that are not found in the Windows directory. - # These files are needed for component builds. If WINDOWSSDKDIR is not set - # use the default SDK path. This will be the case when - # DEPOT_TOOLS_WIN_TOOLCHAIN=0 and vcvarsall.bat has not been run. - win_sdk_dir = os.path.normpath( - os.environ.get('WINDOWSSDKDIR', - os.path.expandvars('%ProgramFiles(x86)%' - '\\Windows Kits\\10'))) - # ARM64 doesn't have a redist for the ucrt DLLs because they are always - # present in the OS. - if target_cpu != 'arm64': - # Starting with the 10.0.17763 SDK the ucrt files are in a version-named - # directory - this handles both cases. - redist_dir = os.path.join(win_sdk_dir, 'Redist') - version_dirs = glob.glob(os.path.join(redist_dir, '10.*')) - if len(version_dirs) > 0: - _SortByHighestVersionNumberFirst(version_dirs) - redist_dir = version_dirs[0] - ucrt_dll_dirs = os.path.join(redist_dir, 'ucrt', 'DLLs', target_cpu) - ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll')) - assert len(ucrt_files) > 0 - for ucrt_src_file in ucrt_files: - file_part = os.path.basename(ucrt_src_file) - ucrt_dst_file = os.path.join(target_dir, file_part) - _CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False) - # We must copy ucrtbase.dll for x64/x86, and ucrtbased.dll for all CPU types. - if target_cpu != 'arm64' or not suffix.startswith('.'): - if not suffix.startswith('.'): - # ucrtbased.dll is located at {win_sdk_dir}/bin/{a.b.c.d}/{target_cpu}/ - # ucrt/. - sdk_redist_root = os.path.join(win_sdk_dir, 'bin') - sdk_bin_sub_dirs = os.listdir(sdk_redist_root) - # Select the most recent SDK if there are multiple versions installed. - _SortByHighestVersionNumberFirst(sdk_bin_sub_dirs) - for directory in sdk_bin_sub_dirs: - sdk_redist_root_version = os.path.join(sdk_redist_root, directory) - if not os.path.isdir(sdk_redist_root_version): - continue - if re.match(r'10\.\d+\.\d+\.\d+', directory): - source_dir = os.path.join(sdk_redist_root_version, target_cpu, 'ucrt') - break - _CopyRuntimeImpl(os.path.join(target_dir, 'ucrtbase' + suffix), - os.path.join(source_dir, 'ucrtbase' + suffix)) + if target_cpu == 'arm64': + # Windows ARM64 VCRuntime is located at {toolchain_root}/VC/Redist/MSVC/ + # {x.y.z}/[debug_nonredist/]arm64/Microsoft.VC141.CRT/. + vc_redist_root = FindVCRedistRoot() + if suffix.startswith('.'): + source_dir = os.path.join(vc_redist_root, 'arm64', + 'Microsoft.VC141.CRT') + else: + source_dir = os.path.join(vc_redist_root, 'debug_nonredist', + 'arm64', 'Microsoft.VC141.DebugCRT') + for file_part in ('msvcp', 'vccorlib', 'vcruntime'): + dll = dll_pattern % file_part + target = os.path.join(target_dir, dll) + source = os.path.join(source_dir, dll) + _CopyRuntimeImpl(target, source) + # Copy the UCRT files from the Windows SDK. This location includes the + # api-ms-win-crt-*.dll files that are not found in the Windows directory. + # These files are needed for component builds. If WINDOWSSDKDIR is not set + # use the default SDK path. This will be the case when + # DEPOT_TOOLS_WIN_TOOLCHAIN=0 and vcvarsall.bat has not been run. + win_sdk_dir = os.path.normpath( + os.environ.get( + 'WINDOWSSDKDIR', + os.path.expandvars('%ProgramFiles(x86)%' + '\\Windows Kits\\10'))) + # ARM64 doesn't have a redist for the ucrt DLLs because they are always + # present in the OS. + if target_cpu != 'arm64': + # Starting with the 10.0.17763 SDK the ucrt files are in a version-named + # directory - this handles both cases. + redist_dir = os.path.join(win_sdk_dir, 'Redist') + version_dirs = glob.glob(os.path.join(redist_dir, '10.*')) + if len(version_dirs) > 0: + _SortByHighestVersionNumberFirst(version_dirs) + redist_dir = version_dirs[0] + ucrt_dll_dirs = os.path.join(redist_dir, 'ucrt', 'DLLs', target_cpu) + ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll')) + assert len(ucrt_files) > 0 + for ucrt_src_file in ucrt_files: + file_part = os.path.basename(ucrt_src_file) + ucrt_dst_file = os.path.join(target_dir, file_part) + _CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False) + # We must copy ucrtbase.dll for x64/x86, and ucrtbased.dll for all CPU types. + if target_cpu != 'arm64' or not suffix.startswith('.'): + if not suffix.startswith('.'): + # ucrtbased.dll is located at {win_sdk_dir}/bin/{a.b.c.d}/{target_cpu}/ + # ucrt/. + sdk_redist_root = os.path.join(win_sdk_dir, 'bin') + sdk_bin_sub_dirs = os.listdir(sdk_redist_root) + # Select the most recent SDK if there are multiple versions installed. + _SortByHighestVersionNumberFirst(sdk_bin_sub_dirs) + for directory in sdk_bin_sub_dirs: + sdk_redist_root_version = os.path.join(sdk_redist_root, + directory) + if not os.path.isdir(sdk_redist_root_version): + continue + if re.match(r'10\.\d+\.\d+\.\d+', directory): + source_dir = os.path.join(sdk_redist_root_version, + target_cpu, 'ucrt') + break + _CopyRuntimeImpl( + os.path.join(target_dir, 'ucrtbase' + suffix), + os.path.join(source_dir, 'ucrtbase' + suffix)) def FindVCComponentRoot(component): - """Find the most recent Tools or Redist or other directory in an MSVC install. + """Find the most recent Tools or Redist or other directory in an MSVC install. Typical results are {toolchain_root}/VC/{component}/MSVC/{x.y.z}. The {x.y.z} version number part changes frequently so the highest version number found is used. """ - SetEnvironmentAndGetRuntimeDllDirs() - assert ('GYP_MSVS_OVERRIDE_PATH' in os.environ) - vc_component_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], - 'VC', component, 'MSVC') - vc_component_msvc_contents = os.listdir(vc_component_msvc_root) - # Select the most recent toolchain if there are several. - _SortByHighestVersionNumberFirst(vc_component_msvc_contents) - for directory in vc_component_msvc_contents: - if not os.path.isdir(os.path.join(vc_component_msvc_root, directory)): - continue - if re.match(r'14\.\d+\.\d+', directory): - return os.path.join(vc_component_msvc_root, directory) - raise Exception('Unable to find the VC %s directory.' % component) + SetEnvironmentAndGetRuntimeDllDirs() + assert ('GYP_MSVS_OVERRIDE_PATH' in os.environ) + vc_component_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], + 'VC', component, 'MSVC') + vc_component_msvc_contents = os.listdir(vc_component_msvc_root) + # Select the most recent toolchain if there are several. + _SortByHighestVersionNumberFirst(vc_component_msvc_contents) + for directory in vc_component_msvc_contents: + if not os.path.isdir(os.path.join(vc_component_msvc_root, directory)): + continue + if re.match(r'14\.\d+\.\d+', directory): + return os.path.join(vc_component_msvc_root, directory) + raise Exception('Unable to find the VC %s directory.' % component) def FindVCRedistRoot(): - """In >=VS2017, Redist binaries are located in + """In >=VS2017, Redist binaries are located in {toolchain_root}/VC/Redist/MSVC/{x.y.z}/{target_cpu}/. This returns the '{toolchain_root}/VC/Redist/MSVC/{x.y.z}/' path. """ - return FindVCComponentRoot('Redist') + return FindVCComponentRoot('Redist') def _CopyRuntime(target_dir, source_dir, target_cpu, debug): - """Copy the VS runtime DLLs, only if the target doesn't exist, but the target + """Copy the VS runtime DLLs, only if the target doesn't exist, but the target directory does exist. Handles VS 2015, 2017 and 2019.""" - suffix = 'd.dll' if debug else '.dll' - # VS 2015, 2017 and 2019 use the same CRT DLLs. - _CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix, - suffix) + suffix = 'd.dll' if debug else '.dll' + # VS 2015, 2017 and 2019 use the same CRT DLLs. + _CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix, + suffix) def CopyDlls(target_dir, configuration, target_cpu): - """Copy the VS runtime DLLs into the requested directory as needed. + """Copy the VS runtime DLLs into the requested directory as needed. configuration is one of 'Debug' or 'Release'. target_cpu is one of 'x86', 'x64' or 'arm64'. @@ -360,27 +368,27 @@ def CopyDlls(target_dir, configuration, target_cpu): The debug configuration gets both the debug and release DLLs; the release config only the latter. """ - vs_runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() - if not vs_runtime_dll_dirs: - return + vs_runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() + if not vs_runtime_dll_dirs: + return - x64_runtime, x86_runtime, arm64_runtime = vs_runtime_dll_dirs - if target_cpu == 'x64': - runtime_dir = x64_runtime - elif target_cpu == 'x86': - runtime_dir = x86_runtime - elif target_cpu == 'arm64': - runtime_dir = arm64_runtime - else: - raise Exception('Unknown target_cpu: ' + target_cpu) - _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False) - if configuration == 'Debug': - _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True) - _CopyDebugger(target_dir, target_cpu) + x64_runtime, x86_runtime, arm64_runtime = vs_runtime_dll_dirs + if target_cpu == 'x64': + runtime_dir = x64_runtime + elif target_cpu == 'x86': + runtime_dir = x86_runtime + elif target_cpu == 'arm64': + runtime_dir = arm64_runtime + else: + raise Exception('Unknown target_cpu: ' + target_cpu) + _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False) + if configuration == 'Debug': + _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True) + _CopyDebugger(target_dir, target_cpu) def _CopyDebugger(target_dir, target_cpu): - """Copy dbghelp.dll and dbgcore.dll into the requested directory as needed. + """Copy dbghelp.dll and dbgcore.dll into the requested directory as needed. target_cpu is one of 'x86', 'x64' or 'arm64'. @@ -392,148 +400,153 @@ def _CopyDebugger(target_dir, target_cpu): dbgcore.dll is needed when using some functions from dbghelp.dll (like MinidumpWriteDump). """ - win_sdk_dir = SetEnvironmentAndGetSDKDir() - if not win_sdk_dir: - return + win_sdk_dir = SetEnvironmentAndGetSDKDir() + if not win_sdk_dir: + return - # List of debug files that should be copied, the first element of the tuple is - # the name of the file and the second indicates if it's optional. - debug_files = [('dbghelp.dll', False), ('dbgcore.dll', True)] - for debug_file, is_optional in debug_files: - full_path = os.path.join(win_sdk_dir, 'Debuggers', target_cpu, debug_file) - if not os.path.exists(full_path): - if is_optional: - continue - else: - # TODO(crbug.com/773476): remove version requirement. - raise Exception('%s not found in "%s"\r\nYou must install the ' - '"Debugging Tools for Windows" feature from the Windows' - ' 10 SDK.' - % (debug_file, full_path)) - target_path = os.path.join(target_dir, debug_file) - _CopyRuntimeImpl(target_path, full_path) + # List of debug files that should be copied, the first element of the tuple is + # the name of the file and the second indicates if it's optional. + debug_files = [('dbghelp.dll', False), ('dbgcore.dll', True)] + for debug_file, is_optional in debug_files: + full_path = os.path.join(win_sdk_dir, 'Debuggers', target_cpu, + debug_file) + if not os.path.exists(full_path): + if is_optional: + continue + else: + # TODO(crbug.com/773476): remove version requirement. + raise Exception( + '%s not found in "%s"\r\nYou must install the ' + '"Debugging Tools for Windows" feature from the Windows' + ' 10 SDK.' % (debug_file, full_path)) + target_path = os.path.join(target_dir, debug_file) + _CopyRuntimeImpl(target_path, full_path) def _GetDesiredVsToolchainHashes(): - """Load a list of SHA1s corresponding to the toolchains that we want installed + """Load a list of SHA1s corresponding to the toolchains that we want installed to build with.""" - env_version = GetVisualStudioVersion() - if env_version == '2017': - # VS 2017 Update 9 (15.9.12) with 10.0.18362 SDK, 10.0.17763 version of - # Debuggers, and 10.0.17134 version of d3dcompiler_47.dll, with ARM64 - # libraries. - toolchain_hash = '418b3076791776573a815eb298c8aa590307af63' - # Third parties that do not have access to the canonical toolchain can map - # canonical toolchain version to their own toolchain versions. - toolchain_hash_mapping_key = 'GYP_MSVS_HASH_%s' % toolchain_hash - return [os.environ.get(toolchain_hash_mapping_key, toolchain_hash)] - raise Exception('Unsupported VS version %s' % env_version) + env_version = GetVisualStudioVersion() + if env_version == '2017': + # VS 2017 Update 9 (15.9.12) with 10.0.18362 SDK, 10.0.17763 version of + # Debuggers, and 10.0.17134 version of d3dcompiler_47.dll, with ARM64 + # libraries. + toolchain_hash = '418b3076791776573a815eb298c8aa590307af63' + # Third parties that do not have access to the canonical toolchain can map + # canonical toolchain version to their own toolchain versions. + toolchain_hash_mapping_key = 'GYP_MSVS_HASH_%s' % toolchain_hash + return [os.environ.get(toolchain_hash_mapping_key, toolchain_hash)] + raise Exception('Unsupported VS version %s' % env_version) def ShouldUpdateToolchain(): - """Check if the toolchain should be upgraded.""" - if not os.path.exists(json_data_file): - return True - with open(json_data_file, 'r') as tempf: - toolchain_data = json.load(tempf) - version = toolchain_data['version'] - env_version = GetVisualStudioVersion() - # If there's a mismatch between the version set in the environment and the one - # in the json file then the toolchain should be updated. - return version != env_version + """Check if the toolchain should be upgraded.""" + if not os.path.exists(json_data_file): + return True + with open(json_data_file, 'r') as tempf: + toolchain_data = json.load(tempf) + version = toolchain_data['version'] + env_version = GetVisualStudioVersion() + # If there's a mismatch between the version set in the environment and the one + # in the json file then the toolchain should be updated. + return version != env_version def Update(force=False, no_download=False): - """Requests an update of the toolchain to the specific hashes we have at + """Requests an update of the toolchain to the specific hashes we have at this revision. The update outputs a .json of the various configuration information required to pass to gyp which we use in |GetToolchainDir()|. If no_download is true then the toolchain will be configured if present but will not be downloaded. """ - if force != False and force != '--force': - print('Unknown parameter "%s"' % force, file=sys.stderr) - return 1 - if force == '--force' or os.path.exists(json_data_file): - force = True + if force != False and force != '--force': + print('Unknown parameter "%s"' % force, file=sys.stderr) + return 1 + if force == '--force' or os.path.exists(json_data_file): + force = True - depot_tools_win_toolchain = \ - bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) - if ((sys.platform in ('win32', 'cygwin') or force) and - depot_tools_win_toolchain): - import find_depot_tools - depot_tools_path = find_depot_tools.add_depot_tools_to_path() + depot_tools_win_toolchain = \ + bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) + if ((sys.platform in ('win32', 'cygwin') or force) and + depot_tools_win_toolchain): + import find_depot_tools + depot_tools_path = find_depot_tools.add_depot_tools_to_path() - # On Linux, the file system is usually case-sensitive while the Windows - # SDK only works on case-insensitive file systems. If it doesn't already - # exist, set up a ciopfs fuse mount to put the SDK in a case-insensitive - # part of the file system. - toolchain_dir = os.path.join(depot_tools_path, 'win_toolchain', 'vs_files') - # For testing this block, unmount existing mounts with - # fusermount -u third_party/depot_tools/win_toolchain/vs_files - if sys.platform.startswith('linux') and not os.path.ismount(toolchain_dir): - import distutils.spawn - ciopfs = distutils.spawn.find_executable('ciopfs') - if not ciopfs: - # ciopfs not found in PATH; try the one downloaded from the DEPS hook. - ciopfs = os.path.join(script_dir, 'ciopfs') - if not os.path.isdir(toolchain_dir): - os.mkdir(toolchain_dir) - if not os.path.isdir(toolchain_dir + '.ciopfs'): - os.mkdir(toolchain_dir + '.ciopfs') - # Without use_ino, clang's #pragma once and Wnonportable-include-path - # both don't work right, see https://llvm.org/PR34931 - # use_ino doesn't slow down builds, so it seems there's no drawback to - # just using it always. - subprocess.check_call([ - ciopfs, '-o', 'use_ino', toolchain_dir + '.ciopfs', toolchain_dir]) + # On Linux, the file system is usually case-sensitive while the Windows + # SDK only works on case-insensitive file systems. If it doesn't already + # exist, set up a ciopfs fuse mount to put the SDK in a case-insensitive + # part of the file system. + toolchain_dir = os.path.join(depot_tools_path, 'win_toolchain', + 'vs_files') + # For testing this block, unmount existing mounts with + # fusermount -u third_party/depot_tools/win_toolchain/vs_files + if sys.platform.startswith( + 'linux') and not os.path.ismount(toolchain_dir): + import distutils.spawn + ciopfs = distutils.spawn.find_executable('ciopfs') + if not ciopfs: + # ciopfs not found in PATH; try the one downloaded from the DEPS hook. + ciopfs = os.path.join(script_dir, 'ciopfs') + if not os.path.isdir(toolchain_dir): + os.mkdir(toolchain_dir) + if not os.path.isdir(toolchain_dir + '.ciopfs'): + os.mkdir(toolchain_dir + '.ciopfs') + # Without use_ino, clang's #pragma once and Wnonportable-include-path + # both don't work right, see https://llvm.org/PR34931 + # use_ino doesn't slow down builds, so it seems there's no drawback to + # just using it always. + subprocess.check_call([ + ciopfs, '-o', 'use_ino', toolchain_dir + '.ciopfs', + toolchain_dir + ]) - # Necessary so that get_toolchain_if_necessary.py will put the VS toolkit - # in the correct directory. - os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion() - get_toolchain_args = [ - sys.executable, - os.path.join(depot_tools_path, - 'win_toolchain', - 'get_toolchain_if_necessary.py'), - '--output-json', json_data_file, - ] + _GetDesiredVsToolchainHashes() - if force: - get_toolchain_args.append('--force') - if no_download: - get_toolchain_args.append('--no-download') - subprocess.check_call(get_toolchain_args) + # Necessary so that get_toolchain_if_necessary.py will put the VS toolkit + # in the correct directory. + os.environ['GYP_MSVS_VERSION'] = GetVisualStudioVersion() + get_toolchain_args = [ + sys.executable, + os.path.join(depot_tools_path, 'win_toolchain', + 'get_toolchain_if_necessary.py'), + '--output-json', + json_data_file, + ] + _GetDesiredVsToolchainHashes() + if force: + get_toolchain_args.append('--force') + if no_download: + get_toolchain_args.append('--no-download') + subprocess.check_call(get_toolchain_args) - return 0 + return 0 def NormalizePath(path): - while path.endswith('\\'): - path = path[:-1] - return path + while path.endswith('\\'): + path = path[:-1] + return path def SetEnvironmentAndGetSDKDir(): - """Gets location information about the current sdk (must have been + """Gets location information about the current sdk (must have been previously updated by 'update'). This is used for the GN build.""" - SetEnvironmentAndGetRuntimeDllDirs() + SetEnvironmentAndGetRuntimeDllDirs() - # If WINDOWSSDKDIR is not set, search the default SDK path and set it. - if not 'WINDOWSSDKDIR' in os.environ: - default_sdk_path = os.path.expandvars('%ProgramFiles(x86)%' - '\\Windows Kits\\10') - if os.path.isdir(default_sdk_path): - os.environ['WINDOWSSDKDIR'] = default_sdk_path + # If WINDOWSSDKDIR is not set, search the default SDK path and set it. + if not 'WINDOWSSDKDIR' in os.environ: + default_sdk_path = os.path.expandvars('%ProgramFiles(x86)%' + '\\Windows Kits\\10') + if os.path.isdir(default_sdk_path): + os.environ['WINDOWSSDKDIR'] = default_sdk_path - return NormalizePath(os.environ['WINDOWSSDKDIR']) + return NormalizePath(os.environ['WINDOWSSDKDIR']) def GetToolchainDir(): - """Gets location information about the current toolchain (must have been + """Gets location information about the current toolchain (must have been previously updated by 'update'). This is used for the GN build.""" - runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() - win_sdk_dir = SetEnvironmentAndGetSDKDir() + runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() + win_sdk_dir = SetEnvironmentAndGetSDKDir() - print('''vs_path = %s + print('''vs_path = %s sdk_path = %s vs_version = %s wdk_dir = %s @@ -545,16 +558,16 @@ runtime_dirs = %s def main(): - commands = { - 'update': Update, - 'get_toolchain_dir': GetToolchainDir, - 'copy_dlls': CopyDlls, - } - if len(sys.argv) < 2 or sys.argv[1] not in commands: - print('Expected one of: %s' % ', '.join(commands), file=sys.stderr) - return 1 - return commands[sys.argv[1]](*sys.argv[2:]) + commands = { + 'update': Update, + 'get_toolchain_dir': GetToolchainDir, + 'copy_dlls': CopyDlls, + } + if len(sys.argv) < 2 or sys.argv[1] not in commands: + print('Expected one of: %s' % ', '.join(commands), file=sys.stderr) + return 1 + return commands[sys.argv[1]](*sys.argv[2:]) if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/build/win/importlibs/create_importlib_win.py b/build/win/importlibs/create_importlib_win.py index bb6a2f02636..401b6e8aae4 100755 --- a/build/win/importlibs/create_importlib_win.py +++ b/build/win/importlibs/create_importlib_win.py @@ -14,7 +14,6 @@ import subprocess import sys import tempfile - _USAGE = """\ Usage: %prog [options] [imports-file] @@ -24,7 +23,6 @@ Note: this script uses the microsoft assembler (ml.exe) and the library tool (lib.exe), both of which must be in path. """ - _ASM_STUB_HEADER = """\ ; This file is autogenerated by create_importlib_win.py, do not edit. .386 @@ -35,55 +33,53 @@ _ASM_STUB_HEADER = """\ ; correct generation of import libs. """ - _DEF_STUB_HEADER = """\ ; This file is autogenerated by create_importlib_win.py, do not edit. ; Export declarations for generating import libs. """ - _LOGGER = logging.getLogger() - class _Error(Exception): - pass + pass class _ImportLibraryGenerator(object): - def __init__(self, temp_dir): - self._temp_dir = temp_dir - def _Shell(self, cmd, **kw): - ret = subprocess.call(cmd, **kw) - _LOGGER.info('Running "%s" returned %d.', cmd, ret) - if ret != 0: - raise _Error('Command "%s" returned %d.' % (cmd, ret)) + def __init__(self, temp_dir): + self._temp_dir = temp_dir - def _ReadImportsFile(self, imports_file): - # Slurp the imports file. - return ast.literal_eval(open(imports_file).read()) + def _Shell(self, cmd, **kw): + ret = subprocess.call(cmd, **kw) + _LOGGER.info('Running "%s" returned %d.', cmd, ret) + if ret != 0: + raise _Error('Command "%s" returned %d.' % (cmd, ret)) - def _WriteStubsFile(self, import_names, output_file): - output_file.write(_ASM_STUB_HEADER) + def _ReadImportsFile(self, imports_file): + # Slurp the imports file. + return ast.literal_eval(open(imports_file).read()) - for name in import_names: - output_file.write('%s PROC\n' % name) - output_file.write('%s ENDP\n' % name) + def _WriteStubsFile(self, import_names, output_file): + output_file.write(_ASM_STUB_HEADER) - output_file.write('END\n') + for name in import_names: + output_file.write('%s PROC\n' % name) + output_file.write('%s ENDP\n' % name) - def _WriteDefFile(self, dll_name, import_names, output_file): - output_file.write(_DEF_STUB_HEADER) - output_file.write('NAME %s\n' % dll_name) - output_file.write('EXPORTS\n') - for name in import_names: - name = name.split('@')[0] - output_file.write(' %s\n' % name) + output_file.write('END\n') - def _CreateObj(self, dll_name, imports): - """Writes an assembly file containing empty declarations. + def _WriteDefFile(self, dll_name, import_names, output_file): + output_file.write(_DEF_STUB_HEADER) + output_file.write('NAME %s\n' % dll_name) + output_file.write('EXPORTS\n') + for name in import_names: + name = name.split('@')[0] + output_file.write(' %s\n' % name) + + def _CreateObj(self, dll_name, imports): + """Writes an assembly file containing empty declarations. For each imported function of the form: @@ -106,112 +102,112 @@ class _ImportLibraryGenerator(object): artifacts are quick and easy to produce, and of no interest to anyone after the fact.""" - # Create an .asm file to provide stdcall-like stub names to lib.exe. - asm_name = dll_name + '.asm' - _LOGGER.info('Writing asm file "%s".', asm_name) - with open(os.path.join(self._temp_dir, asm_name), 'wb') as stubs_file: - self._WriteStubsFile(imports, stubs_file) + # Create an .asm file to provide stdcall-like stub names to lib.exe. + asm_name = dll_name + '.asm' + _LOGGER.info('Writing asm file "%s".', asm_name) + with open(os.path.join(self._temp_dir, asm_name), 'wb') as stubs_file: + self._WriteStubsFile(imports, stubs_file) - # Invoke on the assembler to compile it to .obj. - obj_name = dll_name + '.obj' - cmdline = ['ml.exe', '/nologo', '/c', asm_name, '/Fo', obj_name] - self._Shell(cmdline, cwd=self._temp_dir, stdout=open(os.devnull)) + # Invoke on the assembler to compile it to .obj. + obj_name = dll_name + '.obj' + cmdline = ['ml.exe', '/nologo', '/c', asm_name, '/Fo', obj_name] + self._Shell(cmdline, cwd=self._temp_dir, stdout=open(os.devnull)) - return obj_name + return obj_name - def _CreateImportLib(self, dll_name, imports, architecture, output_file): - """Creates an import lib binding imports to dll_name for architecture. + def _CreateImportLib(self, dll_name, imports, architecture, output_file): + """Creates an import lib binding imports to dll_name for architecture. On success, writes the import library to output file. """ - obj_file = None + obj_file = None - # For x86 architecture we have to provide an object file for correct - # name mangling between the import stubs and the exported functions. - if architecture == 'x86': - obj_file = self._CreateObj(dll_name, imports) + # For x86 architecture we have to provide an object file for correct + # name mangling between the import stubs and the exported functions. + if architecture == 'x86': + obj_file = self._CreateObj(dll_name, imports) - # Create the corresponding .def file. This file has the non stdcall-adorned - # names, as exported by the destination DLL. - def_name = dll_name + '.def' - _LOGGER.info('Writing def file "%s".', def_name) - with open(os.path.join(self._temp_dir, def_name), 'wb') as def_file: - self._WriteDefFile(dll_name, imports, def_file) + # Create the corresponding .def file. This file has the non stdcall-adorned + # names, as exported by the destination DLL. + def_name = dll_name + '.def' + _LOGGER.info('Writing def file "%s".', def_name) + with open(os.path.join(self._temp_dir, def_name), 'wb') as def_file: + self._WriteDefFile(dll_name, imports, def_file) - # Invoke on lib.exe to create the import library. - # We generate everything into the temporary directory, as the .exp export - # files will be generated at the same path as the import library, and we - # don't want those files potentially gunking the works. - dll_base_name, ext = os.path.splitext(dll_name) - lib_name = dll_base_name + '.lib' - cmdline = ['lib.exe', - '/machine:%s' % architecture, - '/def:%s' % def_name, - '/out:%s' % lib_name] - if obj_file: - cmdline.append(obj_file) + # Invoke on lib.exe to create the import library. + # We generate everything into the temporary directory, as the .exp export + # files will be generated at the same path as the import library, and we + # don't want those files potentially gunking the works. + dll_base_name, ext = os.path.splitext(dll_name) + lib_name = dll_base_name + '.lib' + cmdline = [ + 'lib.exe', + '/machine:%s' % architecture, + '/def:%s' % def_name, + '/out:%s' % lib_name + ] + if obj_file: + cmdline.append(obj_file) - self._Shell(cmdline, cwd=self._temp_dir, stdout=open(os.devnull)) + self._Shell(cmdline, cwd=self._temp_dir, stdout=open(os.devnull)) - # Copy the .lib file to the output directory. - shutil.copyfile(os.path.join(self._temp_dir, lib_name), output_file) - _LOGGER.info('Created "%s".', output_file) + # Copy the .lib file to the output directory. + shutil.copyfile(os.path.join(self._temp_dir, lib_name), output_file) + _LOGGER.info('Created "%s".', output_file) - def CreateImportLib(self, imports_file, output_file): - # Read the imports file. - imports = self._ReadImportsFile(imports_file) + def CreateImportLib(self, imports_file, output_file): + # Read the imports file. + imports = self._ReadImportsFile(imports_file) - # Creates the requested import library in the output directory. - self._CreateImportLib(imports['dll_name'], - imports['imports'], - imports.get('architecture', 'x86'), - output_file) + # Creates the requested import library in the output directory. + self._CreateImportLib(imports['dll_name'], imports['imports'], + imports.get('architecture', 'x86'), output_file) def main(): - parser = optparse.OptionParser(usage=_USAGE) - parser.add_option('-o', '--output-file', - help='Specifies the output file path.') - parser.add_option('-k', '--keep-temp-dir', - action='store_true', - help='Keep the temporary directory.') - parser.add_option('-v', '--verbose', - action='store_true', - help='Verbose logging.') + parser = optparse.OptionParser(usage=_USAGE) + parser.add_option( + '-o', '--output-file', help='Specifies the output file path.') + parser.add_option( + '-k', + '--keep-temp-dir', + action='store_true', + help='Keep the temporary directory.') + parser.add_option( + '-v', '--verbose', action='store_true', help='Verbose logging.') - options, args = parser.parse_args() + options, args = parser.parse_args() - if len(args) != 1: - parser.error('You must provide an imports file.') + if len(args) != 1: + parser.error('You must provide an imports file.') - if not options.output_file: - parser.error('You must provide an output file.') + if not options.output_file: + parser.error('You must provide an output file.') - options.output_file = os.path.abspath(options.output_file) + options.output_file = os.path.abspath(options.output_file) - if options.verbose: - logging.basicConfig(level=logging.INFO) - else: - logging.basicConfig(level=logging.WARN) + if options.verbose: + logging.basicConfig(level=logging.INFO) + else: + logging.basicConfig(level=logging.WARN) + temp_dir = tempfile.mkdtemp() + _LOGGER.info('Created temporary directory "%s."', temp_dir) + try: + # Create a generator and create the import lib. + generator = _ImportLibraryGenerator(temp_dir) - temp_dir = tempfile.mkdtemp() - _LOGGER.info('Created temporary directory "%s."', temp_dir) - try: - # Create a generator and create the import lib. - generator = _ImportLibraryGenerator(temp_dir) + ret = generator.CreateImportLib(args[0], options.output_file) + except Exception, e: + _LOGGER.exception('Failed to create import lib.') + ret = 1 + finally: + if not options.keep_temp_dir: + shutil.rmtree(temp_dir) + _LOGGER.info('Deleted temporary directory "%s."', temp_dir) - ret = generator.CreateImportLib(args[0], options.output_file) - except Exception, e: - _LOGGER.exception('Failed to create import lib.') - ret = 1 - finally: - if not options.keep_temp_dir: - shutil.rmtree(temp_dir) - _LOGGER.info('Deleted temporary directory "%s."', temp_dir) - - return ret + return ret if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/build/win/importlibs/filter_export_list.py b/build/win/importlibs/filter_export_list.py index c2489a9da7c..23fe3627d29 100755 --- a/build/win/importlibs/filter_export_list.py +++ b/build/win/importlibs/filter_export_list.py @@ -9,15 +9,14 @@ import optparse import re import sys - -_EXPORT_RE = re.compile(r""" +_EXPORT_RE = re.compile( + r""" ^\s*(?P[0-9]+) # The ordinal field. \s+(?P[0-9A-F]+) # The hint field. \s(?P........) # The RVA field. \s+(?P[^ ]+) # And finally the name we're really after. """, re.VERBOSE) - _USAGE = r"""\ Usage: %prog [options] [master-file] @@ -38,48 +37,51 @@ e.g. they are suffixed with "@" and the number of argument bytes to the function. """ + def _ReadMasterFile(master_file): - # Slurp the master file. - with open(master_file) as f: - master_exports = ast.literal_eval(f.read()) + # Slurp the master file. + with open(master_file) as f: + master_exports = ast.literal_eval(f.read()) - master_mapping = {} - for export in master_exports: - name = export.split('@')[0] - master_mapping[name] = export + master_mapping = {} + for export in master_exports: + name = export.split('@')[0] + master_mapping[name] = export - return master_mapping + return master_mapping def main(): - parser = optparse.OptionParser(usage=_USAGE) - parser.add_option('-r', '--reverse', - action='store_true', - help='Reverse the matching, e.g. return the functions ' - 'in the master list that aren\'t in the input.') + parser = optparse.OptionParser(usage=_USAGE) + parser.add_option( + '-r', + '--reverse', + action='store_true', + help='Reverse the matching, e.g. return the functions ' + 'in the master list that aren\'t in the input.') - options, args = parser.parse_args() - if len(args) != 1: - parser.error('Must provide a master file.') + options, args = parser.parse_args() + if len(args) != 1: + parser.error('Must provide a master file.') - master_mapping = _ReadMasterFile(args[0]) + master_mapping = _ReadMasterFile(args[0]) - found_exports = [] - for line in sys.stdin: - match = _EXPORT_RE.match(line) - if match: - export_name = master_mapping.get(match.group('name'), None) - if export_name: - found_exports.append(export_name) + found_exports = [] + for line in sys.stdin: + match = _EXPORT_RE.match(line) + if match: + export_name = master_mapping.get(match.group('name'), None) + if export_name: + found_exports.append(export_name) - if options.reverse: - # Invert the found_exports list. - found_exports = set(master_mapping.values()) - set(found_exports) + if options.reverse: + # Invert the found_exports list. + found_exports = set(master_mapping.values()) - set(found_exports) - # Sort the found exports for tidy output. - print '\n'.join(sorted(found_exports)) - return 0 + # Sort the found exports for tidy output. + print '\n'.join(sorted(found_exports)) + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/build/win/reorder-imports.py b/build/win/reorder-imports.py index 281668fff5b..f96a9ec85e5 100755 --- a/build/win/reorder-imports.py +++ b/build/win/reorder-imports.py @@ -10,48 +10,53 @@ import shutil import subprocess import sys + def reorder_imports(input_dir, output_dir, architecture): - """Run swapimports.exe on the initial chrome.exe, and write to the output + """Run swapimports.exe on the initial chrome.exe, and write to the output directory. Also copy over any related files that might be needed (pdbs, manifests etc.). """ - input_image = os.path.join(input_dir, 'chrome.exe') - output_image = os.path.join(output_dir, 'chrome.exe') + input_image = os.path.join(input_dir, 'chrome.exe') + output_image = os.path.join(output_dir, 'chrome.exe') - swap_exe = os.path.join( - __file__, - '..\\..\\..\\third_party\\syzygy\\binaries\\exe\\swapimport.exe') + swap_exe = os.path.join( + __file__, + '..\\..\\..\\third_party\\syzygy\\binaries\\exe\\swapimport.exe') - args = [swap_exe, '--input-image=%s' % input_image, - '--output-image=%s' % output_image, '--overwrite', '--no-logo'] + args = [ + swap_exe, + '--input-image=%s' % input_image, + '--output-image=%s' % output_image, '--overwrite', '--no-logo' + ] - if architecture == 'x64': - args.append('--x64'); + if architecture == 'x64': + args.append('--x64') - args.append('chrome_elf.dll'); + args.append('chrome_elf.dll') - subprocess.call(args) + subprocess.call(args) - for fname in glob.iglob(os.path.join(input_dir, 'chrome.exe.*')): - shutil.copy(fname, os.path.join(output_dir, os.path.basename(fname))) - return 0 + for fname in glob.iglob(os.path.join(input_dir, 'chrome.exe.*')): + shutil.copy(fname, os.path.join(output_dir, os.path.basename(fname))) + return 0 def main(argv): - usage = 'reorder_imports.py -i -o -a ' - parser = optparse.OptionParser(usage=usage) - parser.add_option('-i', '--input', help='reorder chrome.exe in DIR', - metavar='DIR') - parser.add_option('-o', '--output', help='write new chrome.exe to DIR', - metavar='DIR') - parser.add_option('-a', '--arch', help='architecture of build (optional)', - default='ia32') - opts, args = parser.parse_args() + usage = 'reorder_imports.py -i -o -a ' + parser = optparse.OptionParser(usage=usage) + parser.add_option( + '-i', '--input', help='reorder chrome.exe in DIR', metavar='DIR') + parser.add_option( + '-o', '--output', help='write new chrome.exe to DIR', metavar='DIR') + parser.add_option( + '-a', '--arch', help='architecture of build (optional)', default='ia32') + opts, args = parser.parse_args() + + if not opts.input or not opts.output: + parser.error('Please provide and input and output directory') + return reorder_imports(opts.input, opts.output, opts.arch) - if not opts.input or not opts.output: - parser.error('Please provide and input and output directory') - return reorder_imports(opts.input, opts.output, opts.arch) if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) + sys.exit(main(sys.argv[1:])) diff --git a/runtime/PRESUBMIT.py b/runtime/PRESUBMIT.py index 02396a76b70..217e40d93dc 100644 --- a/runtime/PRESUBMIT.py +++ b/runtime/PRESUBMIT.py @@ -7,73 +7,73 @@ import cpplint import re import StringIO + # memcpy does not handle overlapping memory regions. Even though this # is well documented it seems to be used in error quite often. To avoid # problems we disallow the direct use of memcpy. The exceptions are in # third-party code and in platform/globals.h which uses it to implement # bit_cast and bit_copy. def CheckMemcpy(filename): - if filename.endswith(os.path.join('platform', 'globals.h')) or \ - filename.find('third_party') != -1: + if filename.endswith(os.path.join('platform', 'globals.h')) or \ + filename.find('third_party') != -1: + return 0 + fh = open(filename, 'r') + content = fh.read() + match = re.search('\\bmemcpy\\b', content) + if match: + offset = match.start() + end_of_line = content.index('\n', offset) + # We allow explicit use of memcpy with an opt-in via NOLINT + if 'NOLINT' not in content[offset:end_of_line]: + line_number = content[0:match.start()].count('\n') + 1 + print("%s:%d: use of memcpy is forbidden" % (filename, line_number)) + return 1 return 0 - fh = open(filename, 'r') - content = fh.read() - match = re.search('\\bmemcpy\\b', content) - if match: - offset = match.start() - end_of_line = content.index('\n', offset) - # We allow explicit use of memcpy with an opt-in via NOLINT - if 'NOLINT' not in content[offset:end_of_line]: - line_number = content[0:match.start()].count('\n') + 1 - print("%s:%d: use of memcpy is forbidden" % (filename, line_number)) - return 1 - return 0 def RunLint(input_api, output_api): - result = [] - cpplint._cpplint_state.ResetErrorCounts() - memcpy_match_count = 0 - # Find all .cc and .h files in the change list. - for git_file in input_api.AffectedTextFiles(): - filename = git_file.AbsoluteLocalPath() - if filename.endswith('.cc') or filename.endswith('.h'): - # Run cpplint on the file. - cpplint.ProcessFile(filename, 1) - # Check for memcpy use. - memcpy_match_count += CheckMemcpy(filename) + result = [] + cpplint._cpplint_state.ResetErrorCounts() + memcpy_match_count = 0 + # Find all .cc and .h files in the change list. + for git_file in input_api.AffectedTextFiles(): + filename = git_file.AbsoluteLocalPath() + if filename.endswith('.cc') or filename.endswith('.h'): + # Run cpplint on the file. + cpplint.ProcessFile(filename, 1) + # Check for memcpy use. + memcpy_match_count += CheckMemcpy(filename) - # Report a presubmit error if any of the files had an error. - if cpplint._cpplint_state.error_count > 0 or memcpy_match_count > 0: - result = [output_api.PresubmitError('Failed cpplint check.')] - return result + # Report a presubmit error if any of the files had an error. + if cpplint._cpplint_state.error_count > 0 or memcpy_match_count > 0: + result = [output_api.PresubmitError('Failed cpplint check.')] + return result def CheckGn(input_api, output_api): - return input_api.canned_checks.CheckGNFormatted(input_api, output_api) + return input_api.canned_checks.CheckGNFormatted(input_api, output_api) def CheckFormatted(input_api, output_api): - def convert_warning_to_error(presubmit_result): - if not presubmit_result.fatal: - # Convert this warning to an error. - stream = StringIO.StringIO() - presubmit_result.handle(stream) - message = stream.getvalue() - return output_api.PresubmitError(message) - return presubmit_result - results = input_api.canned_checks.CheckPatchFormatted(input_api, output_api) - return [convert_warning_to_error(r) for r in results] + def convert_warning_to_error(presubmit_result): + if not presubmit_result.fatal: + # Convert this warning to an error. + stream = StringIO.StringIO() + presubmit_result.handle(stream) + message = stream.getvalue() + return output_api.PresubmitError(message) + return presubmit_result + + results = input_api.canned_checks.CheckPatchFormatted(input_api, output_api) + return [convert_warning_to_error(r) for r in results] def CheckChangeOnUpload(input_api, output_api): - return (RunLint(input_api, output_api) + - CheckGn(input_api, output_api) + - CheckFormatted(input_api, output_api)) + return (RunLint(input_api, output_api) + CheckGn(input_api, output_api) + + CheckFormatted(input_api, output_api)) def CheckChangeOnCommit(input_api, output_api): - return (RunLint(input_api, output_api) + - CheckGn(input_api, output_api) + - CheckFormatted(input_api, output_api)) + return (RunLint(input_api, output_api) + CheckGn(input_api, output_api) + + CheckFormatted(input_api, output_api)) diff --git a/runtime/observatory/update_sources.py b/runtime/observatory/update_sources.py index cc6eb169cfd..39f6ad3db54 100755 --- a/runtime/observatory/update_sources.py +++ b/runtime/observatory/update_sources.py @@ -10,14 +10,16 @@ import os import sys from datetime import date + def getDir(rootdir, target): - sources = [] - for root, subdirs, files in os.walk(rootdir): - subdirs.sort() - files.sort() - for f in files: - sources.append(root + '/' + f) - return sources + sources = [] + for root, subdirs, files in os.walk(rootdir): + subdirs.sort() + files.sort() + for f in files: + sources.append(root + '/' + f) + return sources + HEADER = """# Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a @@ -28,18 +30,20 @@ HEADER = """# Copyright (c) 2017, the Dart project authors. Please see the AUTHO # This file contains all dart, css, and html sources for Observatory. """ + def main(): - with open('observatory_sources.gni', 'w') as target: - target.write(HEADER) - target.write('observatory_sources = [\n') - sources = [] - for rootdir in ['lib', 'web']: - sources.extend(getDir(rootdir, target)) - sources.sort() - for s in sources: - if (s[-9:] != 'README.md'): - target.write(' "' + s + '",\n') - target.write(']\n') + with open('observatory_sources.gni', 'w') as target: + target.write(HEADER) + target.write('observatory_sources = [\n') + sources = [] + for rootdir in ['lib', 'web']: + sources.extend(getDir(rootdir, target)) + sources.sort() + for s in sources: + if (s[-9:] != 'README.md'): + target.write(' "' + s + '",\n') + target.write(']\n') + if __name__ == "__main__": - main() + main() diff --git a/runtime/third_party/binary_size/src/binary_size_utils.py b/runtime/third_party/binary_size/src/binary_size_utils.py index 67335c2b6d3..8ef283e0f72 100644 --- a/runtime/third_party/binary_size/src/binary_size_utils.py +++ b/runtime/third_party/binary_size/src/binary_size_utils.py @@ -1,7 +1,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Common utilities for tools that deal with binary size information. """ @@ -10,7 +9,7 @@ import re def ParseNm(nm_lines): - """Parse nm output, returning data for all relevant (to binary size) + """Parse nm output, returning data for all relevant (to binary size) symbols and ignoring the rest. Args: @@ -22,50 +21,49 @@ def ParseNm(nm_lines): Path may be None if nm couldn't figure out the source file. """ - # Match lines with size, symbol, optional location, optional discriminator - sym_re = re.compile(r'^([0-9a-f]{8,}) ' # address (8+ hex digits) - '([0-9a-f]{8,}) ' # size (8+ hex digits) - '(.) ' # symbol type, one character - '([^\t]+)' # symbol name, separated from next by tab - '(?:\t(.*):[\d\?]+)?.*$') # location - # Match lines with addr but no size. - addr_re = re.compile(r'^[0-9a-f]{8,} (.) ([^\t]+)(?:\t.*)?$') - # Match lines that don't have an address at all -- typically external symbols. - noaddr_re = re.compile(r'^ {8,} (.) (.*)$') - # Match lines with no symbol name, only addr and type - addr_only_re = re.compile(r'^[0-9a-f]{8,} (.)$') + # Match lines with size, symbol, optional location, optional discriminator + sym_re = re.compile(r'^([0-9a-f]{8,}) ' # address (8+ hex digits) + '([0-9a-f]{8,}) ' # size (8+ hex digits) + '(.) ' # symbol type, one character + '([^\t]+)' # symbol name, separated from next by tab + '(?:\t(.*):[\d\?]+)?.*$') # location + # Match lines with addr but no size. + addr_re = re.compile(r'^[0-9a-f]{8,} (.) ([^\t]+)(?:\t.*)?$') + # Match lines that don't have an address at all -- typically external symbols. + noaddr_re = re.compile(r'^ {8,} (.) (.*)$') + # Match lines with no symbol name, only addr and type + addr_only_re = re.compile(r'^[0-9a-f]{8,} (.)$') - seen_lines = set() - for line in nm_lines: - line = line.rstrip() - if line in seen_lines: - # nm outputs identical lines at times. We don't want to treat - # those as distinct symbols because that would make no sense. - continue - seen_lines.add(line) - match = sym_re.match(line) - if match: - address, size, sym_type, sym = match.groups()[0:4] - size = int(size, 16) - if sym_type in ('B', 'b'): - continue # skip all BSS for now. - path = match.group(5) - yield sym, sym_type, size, path, address - continue - match = addr_re.match(line) - if match: - # sym_type, sym = match.groups()[0:2] - continue # No size == we don't care. - match = noaddr_re.match(line) - if match: - sym_type, sym = match.groups() - if sym_type in ('U', 'w'): - continue # external or weak symbol - match = addr_only_re.match(line) - if match: - continue # Nothing to do. + seen_lines = set() + for line in nm_lines: + line = line.rstrip() + if line in seen_lines: + # nm outputs identical lines at times. We don't want to treat + # those as distinct symbols because that would make no sense. + continue + seen_lines.add(line) + match = sym_re.match(line) + if match: + address, size, sym_type, sym = match.groups()[0:4] + size = int(size, 16) + if sym_type in ('B', 'b'): + continue # skip all BSS for now. + path = match.group(5) + yield sym, sym_type, size, path, address + continue + match = addr_re.match(line) + if match: + # sym_type, sym = match.groups()[0:2] + continue # No size == we don't care. + match = noaddr_re.match(line) + if match: + sym_type, sym = match.groups() + if sym_type in ('U', 'w'): + continue # external or weak symbol + match = addr_only_re.match(line) + if match: + continue # Nothing to do. - - # If we reach this part of the loop, there was something in the - # line that we didn't expect or recognize. - logging.warning('nm output parser failed to parse: %s', repr(line)) + # If we reach this part of the loop, there was something in the + # line that we didn't expect or recognize. + logging.warning('nm output parser failed to parse: %s', repr(line)) diff --git a/runtime/third_party/binary_size/src/elf_symbolizer.py b/runtime/third_party/binary_size/src/elf_symbolizer.py index 374063a57f8..86a0a07d63f 100644 --- a/runtime/third_party/binary_size/src/elf_symbolizer.py +++ b/runtime/third_party/binary_size/src/elf_symbolizer.py @@ -15,7 +15,6 @@ import sys import threading import time - # addr2line builds a possibly infinite memory cache that can exhaust # the computer's memory if allowed to grow for too long. This constant # controls how many lookups we do before restarting the process. 4000 @@ -24,7 +23,7 @@ ADDR2LINE_RECYCLE_LIMIT = 4000 class ELFSymbolizer(object): - """An uber-fast (multiprocessing, pipelined and asynchronous) ELF symbolizer. + """An uber-fast (multiprocessing, pipelined and asynchronous) ELF symbolizer. This class is a frontend for addr2line (part of GNU binutils), designed to symbolize batches of large numbers of symbols for a given ELF file. It @@ -75,10 +74,17 @@ class ELFSymbolizer(object): other modules in this project), to allow easy reuse in external projects. """ - def __init__(self, elf_file_path, addr2line_path, callback, inlines=False, - max_concurrent_jobs=None, addr2line_timeout=30, max_queue_size=50, - source_root_path=None, strip_base_path=None): - """Args: + def __init__(self, + elf_file_path, + addr2line_path, + callback, + inlines=False, + max_concurrent_jobs=None, + addr2line_timeout=30, + max_queue_size=50, + source_root_path=None, + strip_base_path=None): + """Args: elf_file_path: path of the elf file to be symbolized. addr2line_path: path of the toolchain's addr2line binary. callback: a callback which will be invoked for each resolved symbol with @@ -106,32 +112,32 @@ class ELFSymbolizer(object): strip_base_path: Rebases the symbols source paths onto |source_root_path| (i.e replace |strip_base_path| with |source_root_path). """ - assert(os.path.isfile(addr2line_path)), 'Cannot find ' + addr2line_path - self.elf_file_path = elf_file_path - self.addr2line_path = addr2line_path - self.callback = callback - self.inlines = inlines - self.max_concurrent_jobs = (max_concurrent_jobs or - min(multiprocessing.cpu_count(), 4)) - self.max_queue_size = max_queue_size - self.addr2line_timeout = addr2line_timeout - self.requests_counter = 0 # For generating monotonic request IDs. - self._a2l_instances = [] # Up to |max_concurrent_jobs| _Addr2Line inst. + assert (os.path.isfile(addr2line_path)), 'Cannot find ' + addr2line_path + self.elf_file_path = elf_file_path + self.addr2line_path = addr2line_path + self.callback = callback + self.inlines = inlines + self.max_concurrent_jobs = (max_concurrent_jobs or + min(multiprocessing.cpu_count(), 4)) + self.max_queue_size = max_queue_size + self.addr2line_timeout = addr2line_timeout + self.requests_counter = 0 # For generating monotonic request IDs. + self._a2l_instances = [] # Up to |max_concurrent_jobs| _Addr2Line inst. - # If necessary, create disambiguation lookup table - self.disambiguate = source_root_path is not None - self.disambiguation_table = {} - self.strip_base_path = strip_base_path - if(self.disambiguate): - self.source_root_path = os.path.abspath(source_root_path) - self._CreateDisambiguationTable() + # If necessary, create disambiguation lookup table + self.disambiguate = source_root_path is not None + self.disambiguation_table = {} + self.strip_base_path = strip_base_path + if (self.disambiguate): + self.source_root_path = os.path.abspath(source_root_path) + self._CreateDisambiguationTable() - # Create one addr2line instance. More instances will be created on demand - # (up to |max_concurrent_jobs|) depending on the rate of the requests. - self._CreateNewA2LInstance() + # Create one addr2line instance. More instances will be created on demand + # (up to |max_concurrent_jobs|) depending on the rate of the requests. + self._CreateNewA2LInstance() - def SymbolizeAsync(self, addr, callback_arg=None): - """Requests symbolization of a given address. + def SymbolizeAsync(self, addr, callback_arg=None): + """Requests symbolization of a given address. This method is not guaranteed to return immediately. It generally does, but in some scenarios (e.g. all addr2line instances have full queues) it can @@ -140,66 +146,66 @@ class ELFSymbolizer(object): Args: addr: address to symbolize. callback_arg: optional argument which will be passed to the |callback|.""" - assert(isinstance(addr, int)) + assert (isinstance(addr, int)) - # Process all the symbols that have been resolved in the meanwhile. - # Essentially, this drains all the addr2line(s) out queues. - for a2l_to_purge in self._a2l_instances: - a2l_to_purge.ProcessAllResolvedSymbolsInQueue() - a2l_to_purge.RecycleIfNecessary() + # Process all the symbols that have been resolved in the meanwhile. + # Essentially, this drains all the addr2line(s) out queues. + for a2l_to_purge in self._a2l_instances: + a2l_to_purge.ProcessAllResolvedSymbolsInQueue() + a2l_to_purge.RecycleIfNecessary() - # Find the best instance according to this logic: - # 1. Find an existing instance with the shortest queue. - # 2. If all of instances' queues are full, but there is room in the pool, - # (i.e. < |max_concurrent_jobs|) create a new instance. - # 3. If there were already |max_concurrent_jobs| instances and all of them - # had full queues, make back-pressure. + # Find the best instance according to this logic: + # 1. Find an existing instance with the shortest queue. + # 2. If all of instances' queues are full, but there is room in the pool, + # (i.e. < |max_concurrent_jobs|) create a new instance. + # 3. If there were already |max_concurrent_jobs| instances and all of them + # had full queues, make back-pressure. - # 1. - def _SortByQueueSizeAndReqID(a2l): - return (a2l.queue_size, a2l.first_request_id) - a2l = min(self._a2l_instances, key=_SortByQueueSizeAndReqID) + # 1. + def _SortByQueueSizeAndReqID(a2l): + return (a2l.queue_size, a2l.first_request_id) - # 2. - if (a2l.queue_size >= self.max_queue_size and - len(self._a2l_instances) < self.max_concurrent_jobs): - a2l = self._CreateNewA2LInstance() + a2l = min(self._a2l_instances, key=_SortByQueueSizeAndReqID) - # 3. - if a2l.queue_size >= self.max_queue_size: - a2l.WaitForNextSymbolInQueue() + # 2. + if (a2l.queue_size >= self.max_queue_size and + len(self._a2l_instances) < self.max_concurrent_jobs): + a2l = self._CreateNewA2LInstance() - a2l.EnqueueRequest(addr, callback_arg) + # 3. + if a2l.queue_size >= self.max_queue_size: + a2l.WaitForNextSymbolInQueue() - def Join(self): - """Waits for all the outstanding requests to complete and terminates.""" - for a2l in self._a2l_instances: - a2l.WaitForIdle() - a2l.Terminate() + a2l.EnqueueRequest(addr, callback_arg) - def _CreateNewA2LInstance(self): - assert(len(self._a2l_instances) < self.max_concurrent_jobs) - a2l = ELFSymbolizer.Addr2Line(self) - self._a2l_instances.append(a2l) - return a2l + def Join(self): + """Waits for all the outstanding requests to complete and terminates.""" + for a2l in self._a2l_instances: + a2l.WaitForIdle() + a2l.Terminate() - def _CreateDisambiguationTable(self): - """ Non-unique file names will result in None entries""" - start_time = time.time() - logging.info('Collecting information about available source files...') - self.disambiguation_table = {} + def _CreateNewA2LInstance(self): + assert (len(self._a2l_instances) < self.max_concurrent_jobs) + a2l = ELFSymbolizer.Addr2Line(self) + self._a2l_instances.append(a2l) + return a2l - for root, _, filenames in os.walk(self.source_root_path): - for f in filenames: - self.disambiguation_table[f] = os.path.join(root, f) if (f not in - self.disambiguation_table) else None - logging.info('Finished collecting information about ' - 'possible files (took %.1f s).', - (time.time() - start_time)) + def _CreateDisambiguationTable(self): + """ Non-unique file names will result in None entries""" + start_time = time.time() + logging.info('Collecting information about available source files...') + self.disambiguation_table = {} + for root, _, filenames in os.walk(self.source_root_path): + for f in filenames: + self.disambiguation_table[f] = os.path.join( + root, f) if (f not in self.disambiguation_table) else None + logging.info( + 'Finished collecting information about ' + 'possible files (took %.1f s).', (time.time() - start_time)) - class Addr2Line(object): - """A python wrapper around an addr2line instance. + class Addr2Line(object): + """A python wrapper around an addr2line instance. The communication with the addr2line process looks as follows: [STDIN] [STDOUT] (from addr2line's viewpoint) @@ -214,254 +220,272 @@ class ELFSymbolizer(object): < /path/to/source/file.c:line_number """ - SYM_ADDR_RE = re.compile(r'([^:]+):(\?|\d+).*') + SYM_ADDR_RE = re.compile(r'([^:]+):(\?|\d+).*') - def __init__(self, symbolizer): - self._symbolizer = symbolizer - self._lib_file_name = posixpath.basename(symbolizer.elf_file_path) + def __init__(self, symbolizer): + self._symbolizer = symbolizer + self._lib_file_name = posixpath.basename(symbolizer.elf_file_path) - # The request queue (i.e. addresses pushed to addr2line's stdin and not - # yet retrieved on stdout) - self._request_queue = collections.deque() + # The request queue (i.e. addresses pushed to addr2line's stdin and not + # yet retrieved on stdout) + self._request_queue = collections.deque() - # This is essentially len(self._request_queue). It has been optimized to a - # separate field because turned out to be a perf hot-spot. - self.queue_size = 0 + # This is essentially len(self._request_queue). It has been optimized to a + # separate field because turned out to be a perf hot-spot. + self.queue_size = 0 - # Keep track of the number of symbols a process has processed to - # avoid a single process growing too big and using all the memory. - self._processed_symbols_count = 0 + # Keep track of the number of symbols a process has processed to + # avoid a single process growing too big and using all the memory. + self._processed_symbols_count = 0 - # Objects required to handle the addr2line subprocess. - self._proc = None # Subprocess.Popen(...) instance. - self._thread = None # Threading.thread instance. - self._out_queue = None # Queue.Queue instance (for buffering a2l stdout). - self._RestartAddr2LineProcess() - - def EnqueueRequest(self, addr, callback_arg): - """Pushes an address to addr2line's stdin (and keeps track of it).""" - self._symbolizer.requests_counter += 1 # For global "age" of requests. - req_idx = self._symbolizer.requests_counter - self._request_queue.append((addr, callback_arg, req_idx)) - self.queue_size += 1 - self._WriteToA2lStdin(addr) - - def WaitForIdle(self): - """Waits until all the pending requests have been symbolized.""" - while self.queue_size > 0: - self.WaitForNextSymbolInQueue() - - def WaitForNextSymbolInQueue(self): - """Waits for the next pending request to be symbolized.""" - if not self.queue_size: - return - - # This outer loop guards against a2l hanging (detecting stdout timeout). - while True: - start_time = datetime.datetime.now() - timeout = datetime.timedelta(seconds=self._symbolizer.addr2line_timeout) - - # The inner loop guards against a2l crashing (checking if it exited). - while (datetime.datetime.now() - start_time < timeout): - # poll() returns !None if the process exited. a2l should never exit. - if self._proc.poll(): - logging.warning('addr2line crashed, respawning (lib: %s).' % - self._lib_file_name) + # Objects required to handle the addr2line subprocess. + self._proc = None # Subprocess.Popen(...) instance. + self._thread = None # Threading.thread instance. + self._out_queue = None # Queue.Queue instance (for buffering a2l stdout). self._RestartAddr2LineProcess() - # TODO(primiano): the best thing to do in this case would be - # shrinking the pool size as, very likely, addr2line is crashed - # due to low memory (and the respawned one will die again soon). - try: - lines = self._out_queue.get(block=True, timeout=0.25) - except Queue.Empty: - # On timeout (1/4 s.) repeat the inner loop and check if either the - # addr2line process did crash or we waited its output for too long. - continue + def EnqueueRequest(self, addr, callback_arg): + """Pushes an address to addr2line's stdin (and keeps track of it).""" + self._symbolizer.requests_counter += 1 # For global "age" of requests. + req_idx = self._symbolizer.requests_counter + self._request_queue.append((addr, callback_arg, req_idx)) + self.queue_size += 1 + self._WriteToA2lStdin(addr) - # In nominal conditions, we get straight to this point. - self._ProcessSymbolOutput(lines) - return + def WaitForIdle(self): + """Waits until all the pending requests have been symbolized.""" + while self.queue_size > 0: + self.WaitForNextSymbolInQueue() - # If this point is reached, we waited more than |addr2line_timeout|. - logging.warning('Hung addr2line process, respawning (lib: %s).' % - self._lib_file_name) - self._RestartAddr2LineProcess() + def WaitForNextSymbolInQueue(self): + """Waits for the next pending request to be symbolized.""" + if not self.queue_size: + return - def ProcessAllResolvedSymbolsInQueue(self): - """Consumes all the addr2line output lines produced (without blocking).""" - if not self.queue_size: - return - while True: - try: - lines = self._out_queue.get_nowait() - except Queue.Empty: - break - self._ProcessSymbolOutput(lines) + # This outer loop guards against a2l hanging (detecting stdout timeout). + while True: + start_time = datetime.datetime.now() + timeout = datetime.timedelta( + seconds=self._symbolizer.addr2line_timeout) - def RecycleIfNecessary(self): - """Restarts the process if it has been used for too long. + # The inner loop guards against a2l crashing (checking if it exited). + while (datetime.datetime.now() - start_time < timeout): + # poll() returns !None if the process exited. a2l should never exit. + if self._proc.poll(): + logging.warning( + 'addr2line crashed, respawning (lib: %s).' % + self._lib_file_name) + self._RestartAddr2LineProcess() + # TODO(primiano): the best thing to do in this case would be + # shrinking the pool size as, very likely, addr2line is crashed + # due to low memory (and the respawned one will die again soon). + + try: + lines = self._out_queue.get(block=True, timeout=0.25) + except Queue.Empty: + # On timeout (1/4 s.) repeat the inner loop and check if either the + # addr2line process did crash or we waited its output for too long. + continue + + # In nominal conditions, we get straight to this point. + self._ProcessSymbolOutput(lines) + return + + # If this point is reached, we waited more than |addr2line_timeout|. + logging.warning('Hung addr2line process, respawning (lib: %s).' + % self._lib_file_name) + self._RestartAddr2LineProcess() + + def ProcessAllResolvedSymbolsInQueue(self): + """Consumes all the addr2line output lines produced (without blocking).""" + if not self.queue_size: + return + while True: + try: + lines = self._out_queue.get_nowait() + except Queue.Empty: + break + self._ProcessSymbolOutput(lines) + + def RecycleIfNecessary(self): + """Restarts the process if it has been used for too long. A long running addr2line process will consume excessive amounts of memory without any gain in performance.""" - if self._processed_symbols_count >= ADDR2LINE_RECYCLE_LIMIT: - self._RestartAddr2LineProcess() + if self._processed_symbols_count >= ADDR2LINE_RECYCLE_LIMIT: + self._RestartAddr2LineProcess() - - def Terminate(self): - """Kills the underlying addr2line process. + def Terminate(self): + """Kills the underlying addr2line process. The poller |_thread| will terminate as well due to the broken pipe.""" - try: - self._proc.kill() - self._proc.communicate() # Essentially wait() without risking deadlock. - except Exception: # An exception while terminating? How interesting. - pass - self._proc = None + try: + self._proc.kill() + self._proc.communicate( + ) # Essentially wait() without risking deadlock. + except Exception: # An exception while terminating? How interesting. + pass + self._proc = None - def _WriteToA2lStdin(self, addr): - self._proc.stdin.write('%s\n' % hex(addr)) - if self._symbolizer.inlines: - # In the case of inlines we output an extra blank line, which causes - # addr2line to emit a (??,??:0) tuple that we use as a boundary marker. - self._proc.stdin.write('\n') - self._proc.stdin.flush() + def _WriteToA2lStdin(self, addr): + self._proc.stdin.write('%s\n' % hex(addr)) + if self._symbolizer.inlines: + # In the case of inlines we output an extra blank line, which causes + # addr2line to emit a (??,??:0) tuple that we use as a boundary marker. + self._proc.stdin.write('\n') + self._proc.stdin.flush() - def _ProcessSymbolOutput(self, lines): - """Parses an addr2line symbol output and triggers the client callback.""" - (_, callback_arg, _) = self._request_queue.popleft() - self.queue_size -= 1 + def _ProcessSymbolOutput(self, lines): + """Parses an addr2line symbol output and triggers the client callback.""" + (_, callback_arg, _) = self._request_queue.popleft() + self.queue_size -= 1 - innermost_sym_info = None - sym_info = None - for (line1, line2) in lines: - prev_sym_info = sym_info - name = line1 if not line1.startswith('?') else None - source_path = None - source_line = None - m = ELFSymbolizer.Addr2Line.SYM_ADDR_RE.match(line2) - if m: - if not m.group(1).startswith('?'): - source_path = m.group(1) - if not m.group(2).startswith('?'): - source_line = int(m.group(2)) - else: - logging.warning('Got invalid symbol path from addr2line: %s' % line2) + innermost_sym_info = None + sym_info = None + for (line1, line2) in lines: + prev_sym_info = sym_info + name = line1 if not line1.startswith('?') else None + source_path = None + source_line = None + m = ELFSymbolizer.Addr2Line.SYM_ADDR_RE.match(line2) + if m: + if not m.group(1).startswith('?'): + source_path = m.group(1) + if not m.group(2).startswith('?'): + source_line = int(m.group(2)) + else: + logging.warning( + 'Got invalid symbol path from addr2line: %s' % line2) - # In case disambiguation is on, and needed - was_ambiguous = False - disambiguated = False - if self._symbolizer.disambiguate: - if source_path and not posixpath.isabs(source_path): - path = self._symbolizer.disambiguation_table.get(source_path) - was_ambiguous = True - disambiguated = path is not None - source_path = path if disambiguated else source_path + # In case disambiguation is on, and needed + was_ambiguous = False + disambiguated = False + if self._symbolizer.disambiguate: + if source_path and not posixpath.isabs(source_path): + path = self._symbolizer.disambiguation_table.get( + source_path) + was_ambiguous = True + disambiguated = path is not None + source_path = path if disambiguated else source_path - # Use absolute paths (so that paths are consistent, as disambiguation - # uses absolute paths) - if source_path and not was_ambiguous: - source_path = os.path.abspath(source_path) + # Use absolute paths (so that paths are consistent, as disambiguation + # uses absolute paths) + if source_path and not was_ambiguous: + source_path = os.path.abspath(source_path) - if source_path and self._symbolizer.strip_base_path: - # Strip the base path - source_path = re.sub('^' + self._symbolizer.strip_base_path, - self._symbolizer.source_root_path or '', source_path) + if source_path and self._symbolizer.strip_base_path: + # Strip the base path + source_path = re.sub( + '^' + self._symbolizer.strip_base_path, + self._symbolizer.source_root_path or '', source_path) - sym_info = ELFSymbolInfo(name, source_path, source_line, was_ambiguous, - disambiguated) - if prev_sym_info: - prev_sym_info.inlined_by = sym_info - if not innermost_sym_info: - innermost_sym_info = sym_info + sym_info = ELFSymbolInfo(name, source_path, source_line, + was_ambiguous, disambiguated) + if prev_sym_info: + prev_sym_info.inlined_by = sym_info + if not innermost_sym_info: + innermost_sym_info = sym_info - self._processed_symbols_count += 1 - self._symbolizer.callback(innermost_sym_info, callback_arg) + self._processed_symbols_count += 1 + self._symbolizer.callback(innermost_sym_info, callback_arg) - def _RestartAddr2LineProcess(self): - if self._proc: - self.Terminate() + def _RestartAddr2LineProcess(self): + if self._proc: + self.Terminate() - # The only reason of existence of this Queue (and the corresponding - # Thread below) is the lack of a subprocess.stdout.poll_avail_lines(). - # Essentially this is a pipe able to extract a couple of lines atomically. - self._out_queue = Queue.Queue() + # The only reason of existence of this Queue (and the corresponding + # Thread below) is the lack of a subprocess.stdout.poll_avail_lines(). + # Essentially this is a pipe able to extract a couple of lines atomically. + self._out_queue = Queue.Queue() - # Start the underlying addr2line process in line buffered mode. + # Start the underlying addr2line process in line buffered mode. - cmd = [self._symbolizer.addr2line_path, '--functions', '--demangle', - '--exe=' + self._symbolizer.elf_file_path] - if self._symbolizer.inlines: - cmd += ['--inlines'] - self._proc = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE, - stdin=subprocess.PIPE, stderr=sys.stderr, close_fds=True) + cmd = [ + self._symbolizer.addr2line_path, '--functions', '--demangle', + '--exe=' + self._symbolizer.elf_file_path + ] + if self._symbolizer.inlines: + cmd += ['--inlines'] + self._proc = subprocess.Popen( + cmd, + bufsize=1, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=sys.stderr, + close_fds=True) - # Start the poller thread, which simply moves atomically the lines read - # from the addr2line's stdout to the |_out_queue|. - self._thread = threading.Thread( - target=ELFSymbolizer.Addr2Line.StdoutReaderThread, - args=(self._proc.stdout, self._out_queue, self._symbolizer.inlines)) - self._thread.daemon = True # Don't prevent early process exit. - self._thread.start() + # Start the poller thread, which simply moves atomically the lines read + # from the addr2line's stdout to the |_out_queue|. + self._thread = threading.Thread( + target=ELFSymbolizer.Addr2Line.StdoutReaderThread, + args=(self._proc.stdout, self._out_queue, + self._symbolizer.inlines)) + self._thread.daemon = True # Don't prevent early process exit. + self._thread.start() - self._processed_symbols_count = 0 + self._processed_symbols_count = 0 - # Replay the pending requests on the new process (only for the case - # of a hung addr2line timing out during the game). - for (addr, _, _) in self._request_queue: - self._WriteToA2lStdin(addr) + # Replay the pending requests on the new process (only for the case + # of a hung addr2line timing out during the game). + for (addr, _, _) in self._request_queue: + self._WriteToA2lStdin(addr) - @staticmethod - def StdoutReaderThread(process_pipe, queue, inlines): - """The poller thread fn, which moves the addr2line stdout to the |queue|. + @staticmethod + def StdoutReaderThread(process_pipe, queue, inlines): + """The poller thread fn, which moves the addr2line stdout to the |queue|. This is the only piece of code not running on the main thread. It merely writes to a Queue, which is thread-safe. In the case of inlines, it detects the ??,??:0 marker and sends the lines atomically, such that the main thread always receives all the lines corresponding to one symbol in one shot.""" - try: - lines_for_one_symbol = [] - while True: - line1 = process_pipe.readline().rstrip('\r\n') - line2 = process_pipe.readline().rstrip('\r\n') - if not line1 or not line2: - break - inline_has_more_lines = inlines and (len(lines_for_one_symbol) == 0 or - (line1 != '??' and line2 != '??:0')) - if not inlines or inline_has_more_lines: - lines_for_one_symbol += [(line1, line2)] - if inline_has_more_lines: - continue - queue.put(lines_for_one_symbol) - lines_for_one_symbol = [] - process_pipe.close() + try: + lines_for_one_symbol = [] + while True: + line1 = process_pipe.readline().rstrip('\r\n') + line2 = process_pipe.readline().rstrip('\r\n') + if not line1 or not line2: + break + inline_has_more_lines = inlines and ( + len(lines_for_one_symbol) == 0 or + (line1 != '??' and line2 != '??:0')) + if not inlines or inline_has_more_lines: + lines_for_one_symbol += [(line1, line2)] + if inline_has_more_lines: + continue + queue.put(lines_for_one_symbol) + lines_for_one_symbol = [] + process_pipe.close() - # Every addr2line processes will die at some point, please die silently. - except (IOError, OSError): - pass + # Every addr2line processes will die at some point, please die silently. + except (IOError, OSError): + pass - @property - def first_request_id(self): - """Returns the request_id of the oldest pending request in the queue.""" - return self._request_queue[0][2] if self._request_queue else 0 + @property + def first_request_id(self): + """Returns the request_id of the oldest pending request in the queue.""" + return self._request_queue[0][2] if self._request_queue else 0 class ELFSymbolInfo(object): - """The result of the symbolization passed as first arg. of each callback.""" + """The result of the symbolization passed as first arg. of each callback.""" - def __init__(self, name, source_path, source_line, was_ambiguous=False, - disambiguated=False): - """All the fields here can be None (if addr2line replies with '??').""" - self.name = name - self.source_path = source_path - self.source_line = source_line - # In the case of |inlines|=True, the |inlined_by| points to the outer - # function inlining the current one (and so on, to form a chain). - self.inlined_by = None - self.disambiguated = disambiguated - self.was_ambiguous = was_ambiguous + def __init__(self, + name, + source_path, + source_line, + was_ambiguous=False, + disambiguated=False): + """All the fields here can be None (if addr2line replies with '??').""" + self.name = name + self.source_path = source_path + self.source_line = source_line + # In the case of |inlines|=True, the |inlined_by| points to the outer + # function inlining the current one (and so on, to form a chain). + self.inlined_by = None + self.disambiguated = disambiguated + self.was_ambiguous = was_ambiguous - def __str__(self): - return '%s [%s:%d]' % ( - self.name or '??', self.source_path or '??', self.source_line or 0) + def __str__(self): + return '%s [%s:%d]' % (self.name or '??', self.source_path or '??', + self.source_line or 0) diff --git a/runtime/third_party/binary_size/src/explain_binary_size_delta.py b/runtime/third_party/binary_size/src/explain_binary_size_delta.py index 45c1236271f..b6a02704558 100755 --- a/runtime/third_party/binary_size/src/explain_binary_size_delta.py +++ b/runtime/third_party/binary_size/src/explain_binary_size_delta.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Describe the size difference of two binaries. Generates a description of the size difference of two binaries based @@ -49,399 +48,424 @@ import binary_size_utils def CalculateSharedAddresses(symbols): - """Checks how many symbols share the same memory space. This returns a + """Checks how many symbols share the same memory space. This returns a Counter result where result[address] will tell you how many times address was used by symbols.""" - count = Counter() - for _, _, _, _, address in symbols: - count[address] += 1 + count = Counter() + for _, _, _, _, address in symbols: + count[address] += 1 - return count + return count def CalculateEffectiveSize(share_count, address, symbol_size): - """Given a raw symbol_size and an address, this method returns the + """Given a raw symbol_size and an address, this method returns the size we should blame on this symbol considering it might share the machine code/data with other symbols. Using the raw symbol_size for each symbol would in those cases over estimate the true cost of that block. """ - shared_count = share_count[address] - if shared_count == 1: - return symbol_size + shared_count = share_count[address] + if shared_count == 1: + return symbol_size + + assert shared_count > 1 + return int(ceil(symbol_size / float(shared_count))) - assert shared_count > 1 - return int(ceil(symbol_size / float(shared_count))) class SymbolDelta(object): - """Stores old size, new size and some metadata.""" - def __init__(self, shared): - self.old_size = None - self.new_size = None - self.shares_space_with_other_symbols = shared + """Stores old size, new size and some metadata.""" - def __eq__(self, other): - return (self.old_size == other.old_size and - self.new_size == other.new_size and - self.shares_space_with_other_symbols == - other.shares_space_with_other_symbols) + def __init__(self, shared): + self.old_size = None + self.new_size = None + self.shares_space_with_other_symbols = shared - def __ne__(self, other): - return not self.__eq__(other) + def __eq__(self, other): + return (self.old_size == other.old_size and + self.new_size == other.new_size and + self.shares_space_with_other_symbols == other. + shares_space_with_other_symbols) + + def __ne__(self, other): + return not self.__eq__(other) + + def copy_symbol_delta(self): + symbol_delta = SymbolDelta(self.shares_space_with_other_symbols) + symbol_delta.old_size = self.old_size + symbol_delta.new_size = self.new_size + return symbol_delta - def copy_symbol_delta(self): - symbol_delta = SymbolDelta(self.shares_space_with_other_symbols) - symbol_delta.old_size = self.old_size - symbol_delta.new_size = self.new_size - return symbol_delta class DeltaInfo(SymbolDelta): - """Summary of a the change for one symbol between two instances.""" - def __init__(self, file_path, symbol_type, symbol_name, shared): - SymbolDelta.__init__(self, shared) - self.file_path = file_path - self.symbol_type = symbol_type - self.symbol_name = symbol_name + """Summary of a the change for one symbol between two instances.""" - def __eq__(self, other): - return (self.file_path == other.file_path and - self.symbol_type == other.symbol_type and - self.symbol_name == other.symbol_name and - SymbolDelta.__eq__(self, other)) + def __init__(self, file_path, symbol_type, symbol_name, shared): + SymbolDelta.__init__(self, shared) + self.file_path = file_path + self.symbol_type = symbol_type + self.symbol_name = symbol_name - def __ne__(self, other): - return not self.__eq__(other) + def __eq__(self, other): + return (self.file_path == other.file_path and + self.symbol_type == other.symbol_type and + self.symbol_name == other.symbol_name and + SymbolDelta.__eq__(self, other)) + + def __ne__(self, other): + return not self.__eq__(other) + + def ExtractSymbolDelta(self): + """Returns a copy of the SymbolDelta for this DeltaInfo.""" + return SymbolDelta.copy_symbol_delta(self) - def ExtractSymbolDelta(self): - """Returns a copy of the SymbolDelta for this DeltaInfo.""" - return SymbolDelta.copy_symbol_delta(self) def Compare(symbols1, symbols2): - """Executes a comparison of the symbols in symbols1 and symbols2. + """Executes a comparison of the symbols in symbols1 and symbols2. Returns: tuple of lists: (added_symbols, removed_symbols, changed_symbols, others) where each list contains DeltaInfo objects. """ - added = [] # tuples - removed = [] # tuples - changed = [] # tuples - unchanged = [] # tuples + added = [] # tuples + removed = [] # tuples + changed = [] # tuples + unchanged = [] # tuples - cache1 = {} - cache2 = {} - # Make a map of (file, symbol_type) : (symbol_name, effective_symbol_size) - share_count1 = CalculateSharedAddresses(symbols1) - share_count2 = CalculateSharedAddresses(symbols2) - for cache, symbols, share_count in ((cache1, symbols1, share_count1), - (cache2, symbols2, share_count2)): - for symbol_name, symbol_type, symbol_size, file_path, address in symbols: - if 'vtable for ' in symbol_name: - symbol_type = '@' # hack to categorize these separately - if file_path: - file_path = os.path.normpath(file_path) - if sys.platform.startswith('win'): - file_path = file_path.replace('\\', '/') - else: - file_path = '(No Path)' - # Take into consideration that multiple symbols might share the same - # block of code. - effective_symbol_size = CalculateEffectiveSize(share_count, address, - symbol_size) - key = (file_path, symbol_type) - bucket = cache.setdefault(key, {}) - size_list = bucket.setdefault(symbol_name, []) - size_list.append((effective_symbol_size, - effective_symbol_size != symbol_size)) - - # Now diff them. We iterate over the elements in cache1. For each symbol - # that we find in cache2, we record whether it was deleted, changed, or - # unchanged. We then remove it from cache2; all the symbols that remain - # in cache2 at the end of the iteration over cache1 are the 'new' symbols. - for key, bucket1 in cache1.items(): - bucket2 = cache2.get(key) - file_path, symbol_type = key; - if not bucket2: - # A file was removed. Everything in bucket1 is dead. - for symbol_name, symbol_size_list in bucket1.items(): - for (symbol_size, shared) in symbol_size_list: - delta_info = DeltaInfo(file_path, symbol_type, symbol_name, shared) - delta_info.old_size = symbol_size - removed.append(delta_info) - else: - # File still exists, look for changes within. - for symbol_name, symbol_size_list in bucket1.items(): - size_list2 = bucket2.get(symbol_name) - if size_list2 is None: - # Symbol no longer exists in bucket2. - for (symbol_size, shared) in symbol_size_list: - delta_info = DeltaInfo(file_path, symbol_type, symbol_name, shared) - delta_info.old_size = symbol_size - removed.append(delta_info) - else: - del bucket2[symbol_name] # Symbol is not new, delete from cache2. - if len(symbol_size_list) == 1 and len(size_list2) == 1: - symbol_size, shared1 = symbol_size_list[0] - size2, shared2 = size_list2[0] - delta_info = DeltaInfo(file_path, symbol_type, symbol_name, - shared1 or shared2) - delta_info.old_size = symbol_size - delta_info.new_size = size2 - if symbol_size != size2: - # Symbol has change size in bucket. - changed.append(delta_info) + cache1 = {} + cache2 = {} + # Make a map of (file, symbol_type) : (symbol_name, effective_symbol_size) + share_count1 = CalculateSharedAddresses(symbols1) + share_count2 = CalculateSharedAddresses(symbols2) + for cache, symbols, share_count in ((cache1, symbols1, share_count1), + (cache2, symbols2, share_count2)): + for symbol_name, symbol_type, symbol_size, file_path, address in symbols: + if 'vtable for ' in symbol_name: + symbol_type = '@' # hack to categorize these separately + if file_path: + file_path = os.path.normpath(file_path) + if sys.platform.startswith('win'): + file_path = file_path.replace('\\', '/') else: - # Symbol is unchanged. - unchanged.append(delta_info) - else: - # Complex comparison for when a symbol exists multiple times - # in the same file (where file can be "unknown file"). - symbol_size_counter = collections.Counter(symbol_size_list) - delta_counter = collections.Counter(symbol_size_list) - delta_counter.subtract(size_list2) - for delta_counter_key in sorted(delta_counter.keys()): - delta = delta_counter[delta_counter_key] - unchanged_count = symbol_size_counter[delta_counter_key] - (symbol_size, shared) = delta_counter_key - if delta > 0: - unchanged_count -= delta - for _ in range(unchanged_count): - delta_info = DeltaInfo(file_path, symbol_type, - symbol_name, shared) - delta_info.old_size = symbol_size + file_path = '(No Path)' + # Take into consideration that multiple symbols might share the same + # block of code. + effective_symbol_size = CalculateEffectiveSize( + share_count, address, symbol_size) + key = (file_path, symbol_type) + bucket = cache.setdefault(key, {}) + size_list = bucket.setdefault(symbol_name, []) + size_list.append((effective_symbol_size, + effective_symbol_size != symbol_size)) + + # Now diff them. We iterate over the elements in cache1. For each symbol + # that we find in cache2, we record whether it was deleted, changed, or + # unchanged. We then remove it from cache2; all the symbols that remain + # in cache2 at the end of the iteration over cache1 are the 'new' symbols. + for key, bucket1 in cache1.items(): + bucket2 = cache2.get(key) + file_path, symbol_type = key + if not bucket2: + # A file was removed. Everything in bucket1 is dead. + for symbol_name, symbol_size_list in bucket1.items(): + for (symbol_size, shared) in symbol_size_list: + delta_info = DeltaInfo(file_path, symbol_type, symbol_name, + shared) + delta_info.old_size = symbol_size + removed.append(delta_info) + else: + # File still exists, look for changes within. + for symbol_name, symbol_size_list in bucket1.items(): + size_list2 = bucket2.get(symbol_name) + if size_list2 is None: + # Symbol no longer exists in bucket2. + for (symbol_size, shared) in symbol_size_list: + delta_info = DeltaInfo(file_path, symbol_type, + symbol_name, shared) + delta_info.old_size = symbol_size + removed.append(delta_info) + else: + del bucket2[ + symbol_name] # Symbol is not new, delete from cache2. + if len(symbol_size_list) == 1 and len(size_list2) == 1: + symbol_size, shared1 = symbol_size_list[0] + size2, shared2 = size_list2[0] + delta_info = DeltaInfo(file_path, symbol_type, + symbol_name, shared1 or shared2) + delta_info.old_size = symbol_size + delta_info.new_size = size2 + if symbol_size != size2: + # Symbol has change size in bucket. + changed.append(delta_info) + else: + # Symbol is unchanged. + unchanged.append(delta_info) + else: + # Complex comparison for when a symbol exists multiple times + # in the same file (where file can be "unknown file"). + symbol_size_counter = collections.Counter( + symbol_size_list) + delta_counter = collections.Counter(symbol_size_list) + delta_counter.subtract(size_list2) + for delta_counter_key in sorted(delta_counter.keys()): + delta = delta_counter[delta_counter_key] + unchanged_count = symbol_size_counter[ + delta_counter_key] + (symbol_size, shared) = delta_counter_key + if delta > 0: + unchanged_count -= delta + for _ in range(unchanged_count): + delta_info = DeltaInfo(file_path, symbol_type, + symbol_name, shared) + delta_info.old_size = symbol_size + delta_info.new_size = symbol_size + unchanged.append(delta_info) + if delta > 0: # Used to be more of these than there is now. + for _ in range(delta): + delta_info = DeltaInfo( + file_path, symbol_type, symbol_name, + shared) + delta_info.old_size = symbol_size + removed.append(delta_info) + elif delta < 0: # More of this (symbol,size) now. + for _ in range(-delta): + delta_info = DeltaInfo( + file_path, symbol_type, symbol_name, + shared) + delta_info.new_size = symbol_size + added.append(delta_info) + + if len(bucket2) == 0: + del cache1[ + key] # Entire bucket is empty, delete from cache2 + + # We have now analyzed all symbols that are in cache1 and removed all of + # the encountered symbols from cache2. What's left in cache2 is the new + # symbols. + for key, bucket2 in cache2.iteritems(): + file_path, symbol_type = key + for symbol_name, symbol_size_list in bucket2.items(): + for (symbol_size, shared) in symbol_size_list: + delta_info = DeltaInfo(file_path, symbol_type, symbol_name, + shared) delta_info.new_size = symbol_size - unchanged.append(delta_info) - if delta > 0: # Used to be more of these than there is now. - for _ in range(delta): - delta_info = DeltaInfo(file_path, symbol_type, - symbol_name, shared) - delta_info.old_size = symbol_size - removed.append(delta_info) - elif delta < 0: # More of this (symbol,size) now. - for _ in range(-delta): - delta_info = DeltaInfo(file_path, symbol_type, - symbol_name, shared) - delta_info.new_size = symbol_size - added.append(delta_info) - - if len(bucket2) == 0: - del cache1[key] # Entire bucket is empty, delete from cache2 - - # We have now analyzed all symbols that are in cache1 and removed all of - # the encountered symbols from cache2. What's left in cache2 is the new - # symbols. - for key, bucket2 in cache2.iteritems(): - file_path, symbol_type = key; - for symbol_name, symbol_size_list in bucket2.items(): - for (symbol_size, shared) in symbol_size_list: - delta_info = DeltaInfo(file_path, symbol_type, symbol_name, shared) - delta_info.new_size = symbol_size - added.append(delta_info) - return (added, removed, changed, unchanged) + added.append(delta_info) + return (added, removed, changed, unchanged) def DeltaStr(number): - """Returns the number as a string with a '+' prefix if it's > 0 and + """Returns the number as a string with a '+' prefix if it's > 0 and a '-' prefix if it's < 0.""" - result = str(number) - if number > 0: - result = '+' + result - return result + result = str(number) + if number > 0: + result = '+' + result + return result def SharedInfoStr(symbol_info): - """Returns a string (prefixed by space) explaining that numbers are + """Returns a string (prefixed by space) explaining that numbers are adjusted because of shared space between symbols, or an empty string if space had not been shared.""" - if symbol_info.shares_space_with_other_symbols: - return " (adjusted sizes because of memory sharing)" + if symbol_info.shares_space_with_other_symbols: + return " (adjusted sizes because of memory sharing)" + + return "" - return "" class CrunchStatsData(object): - """Stores a summary of data of a certain kind.""" - def __init__(self, symbols): - self.symbols = symbols - self.sources = set() - self.before_size = 0 - self.after_size = 0 - self.symbols_by_path = {} + """Stores a summary of data of a certain kind.""" + + def __init__(self, symbols): + self.symbols = symbols + self.sources = set() + self.before_size = 0 + self.after_size = 0 + self.symbols_by_path = {} def CrunchStats(added, removed, changed, unchanged, showsources, showsymbols): - """Outputs to stdout a summary of changes based on the symbol lists.""" - # Split changed into grown and shrunk because that is easier to - # discuss. - grown = [] - shrunk = [] - for item in changed: - if item.old_size < item.new_size: - grown.append(item) - else: - shrunk.append(item) - - new_symbols = CrunchStatsData(added) - removed_symbols = CrunchStatsData(removed) - grown_symbols = CrunchStatsData(grown) - shrunk_symbols = CrunchStatsData(shrunk) - sections = [new_symbols, removed_symbols, grown_symbols, shrunk_symbols] - for section in sections: - for item in section.symbols: - section.sources.add(item.file_path) - if item.old_size is not None: - section.before_size += item.old_size - if item.new_size is not None: - section.after_size += item.new_size - bucket = section.symbols_by_path.setdefault(item.file_path, []) - bucket.append((item.symbol_name, item.symbol_type, - item.ExtractSymbolDelta())) - - total_change = sum(s.after_size - s.before_size for s in sections) - summary = 'Total change: %s bytes' % DeltaStr(total_change) - print(summary) - print('=' * len(summary)) - for section in sections: - if not section.symbols: - continue - if section.before_size == 0: - description = ('added, totalling %s bytes' % DeltaStr(section.after_size)) - elif section.after_size == 0: - description = ('removed, totalling %s bytes' % - DeltaStr(-section.before_size)) - else: - if section.after_size > section.before_size: - type_str = 'grown' - else: - type_str = 'shrunk' - description = ('%s, for a net change of %s bytes ' - '(%d bytes before, %d bytes after)' % - (type_str, DeltaStr(section.after_size - section.before_size), - section.before_size, section.after_size)) - print(' %d %s across %d sources' % - (len(section.symbols), description, len(section.sources))) - - maybe_unchanged_sources = set() - unchanged_symbols_size = 0 - for item in unchanged: - maybe_unchanged_sources.add(item.file_path) - unchanged_symbols_size += item.old_size # == item.new_size - print(' %d unchanged, totalling %d bytes' % - (len(unchanged), unchanged_symbols_size)) - - # High level analysis, always output. - unchanged_sources = maybe_unchanged_sources - for section in sections: - unchanged_sources = unchanged_sources - section.sources - new_sources = (new_symbols.sources - - maybe_unchanged_sources - - removed_symbols.sources) - removed_sources = (removed_symbols.sources - - maybe_unchanged_sources - - new_symbols.sources) - partially_changed_sources = (grown_symbols.sources | - shrunk_symbols.sources | new_symbols.sources | - removed_symbols.sources) - removed_sources - new_sources - allFiles = set() - for section in sections: - allFiles = allFiles | section.sources - allFiles = allFiles | maybe_unchanged_sources - print 'Source stats:' - print(' %d sources encountered.' % len(allFiles)) - print(' %d completely new.' % len(new_sources)) - print(' %d removed completely.' % len(removed_sources)) - print(' %d partially changed.' % len(partially_changed_sources)) - print(' %d completely unchanged.' % len(unchanged_sources)) - remainder = (allFiles - new_sources - removed_sources - - partially_changed_sources - unchanged_sources) - assert len(remainder) == 0 - - if not showsources: - return # Per-source analysis, only if requested - print 'Per-source Analysis:' - delta_by_path = {} - for section in sections: - for path in section.symbols_by_path: - entry = delta_by_path.get(path) - if not entry: - entry = {'plus': 0, 'minus': 0} - delta_by_path[path] = entry - for symbol_name, symbol_type, symbol_delta in \ - section.symbols_by_path[path]: - if symbol_delta.old_size is None: - delta = symbol_delta.new_size - elif symbol_delta.new_size is None: - delta = -symbol_delta.old_size + """Outputs to stdout a summary of changes based on the symbol lists.""" + # Split changed into grown and shrunk because that is easier to + # discuss. + grown = [] + shrunk = [] + for item in changed: + if item.old_size < item.new_size: + grown.append(item) else: - delta = symbol_delta.new_size - symbol_delta.old_size + shrunk.append(item) - if delta > 0: - entry['plus'] += delta + new_symbols = CrunchStatsData(added) + removed_symbols = CrunchStatsData(removed) + grown_symbols = CrunchStatsData(grown) + shrunk_symbols = CrunchStatsData(shrunk) + sections = [new_symbols, removed_symbols, grown_symbols, shrunk_symbols] + for section in sections: + for item in section.symbols: + section.sources.add(item.file_path) + if item.old_size is not None: + section.before_size += item.old_size + if item.new_size is not None: + section.after_size += item.new_size + bucket = section.symbols_by_path.setdefault(item.file_path, []) + bucket.append((item.symbol_name, item.symbol_type, + item.ExtractSymbolDelta())) + + total_change = sum(s.after_size - s.before_size for s in sections) + summary = 'Total change: %s bytes' % DeltaStr(total_change) + print(summary) + print('=' * len(summary)) + for section in sections: + if not section.symbols: + continue + if section.before_size == 0: + description = ( + 'added, totalling %s bytes' % DeltaStr(section.after_size)) + elif section.after_size == 0: + description = ( + 'removed, totalling %s bytes' % DeltaStr(-section.before_size)) else: - entry['minus'] += (-1 * delta) + if section.after_size > section.before_size: + type_str = 'grown' + else: + type_str = 'shrunk' + description = ( + '%s, for a net change of %s bytes ' + '(%d bytes before, %d bytes after)' % + (type_str, DeltaStr(section.after_size - section.before_size), + section.before_size, section.after_size)) + print(' %d %s across %d sources' % (len(section.symbols), description, + len(section.sources))) - def delta_sort_key(item): - _path, size_data = item - growth = size_data['plus'] - size_data['minus'] - return growth + maybe_unchanged_sources = set() + unchanged_symbols_size = 0 + for item in unchanged: + maybe_unchanged_sources.add(item.file_path) + unchanged_symbols_size += item.old_size # == item.new_size + print(' %d unchanged, totalling %d bytes' % (len(unchanged), + unchanged_symbols_size)) - for path, size_data in sorted(delta_by_path.iteritems(), key=delta_sort_key, - reverse=True): - gain = size_data['plus'] - loss = size_data['minus'] - delta = size_data['plus'] - size_data['minus'] - header = ' %s - Source: %s - (gained %d, lost %d)' % (DeltaStr(delta), - path, gain, loss) - divider = '-' * len(header) - print '' - print divider - print header - print divider - if showsymbols: - def ExtractNewSize(tup): - symbol_delta = tup[2] - return symbol_delta.new_size - def ExtractOldSize(tup): - symbol_delta = tup[2] - return symbol_delta.old_size - if path in new_symbols.symbols_by_path: - print ' New symbols:' - for symbol_name, symbol_type, symbol_delta in \ - sorted(new_symbols.symbols_by_path[path], - key=ExtractNewSize, - reverse=True): - print (' %8s: %s type=%s, size=%d bytes%s' % - (DeltaStr(symbol_delta.new_size), symbol_name, symbol_type, - symbol_delta.new_size, SharedInfoStr(symbol_delta))) - if path in removed_symbols.symbols_by_path: - print ' Removed symbols:' - for symbol_name, symbol_type, symbol_delta in \ - sorted(removed_symbols.symbols_by_path[path], - key=ExtractOldSize): - print (' %8s: %s type=%s, size=%d bytes%s' % - (DeltaStr(-symbol_delta.old_size), symbol_name, symbol_type, - symbol_delta.old_size, - SharedInfoStr(symbol_delta))) - for (changed_symbols_by_path, type_str) in [ - (grown_symbols.symbols_by_path, "Grown"), - (shrunk_symbols.symbols_by_path, "Shrunk")]: - if path in changed_symbols_by_path: - print ' %s symbols:' % type_str - def changed_symbol_sortkey(item): - symbol_name, _symbol_type, symbol_delta = item - return (symbol_delta.old_size - symbol_delta.new_size, symbol_name) - for symbol_name, symbol_type, symbol_delta in \ - sorted(changed_symbols_by_path[path], key=changed_symbol_sortkey): - print (' %8s: %s type=%s, (was %d bytes, now %d bytes)%s' - % (DeltaStr(symbol_delta.new_size - symbol_delta.old_size), - symbol_name, symbol_type, - symbol_delta.old_size, symbol_delta.new_size, - SharedInfoStr(symbol_delta))) + # High level analysis, always output. + unchanged_sources = maybe_unchanged_sources + for section in sections: + unchanged_sources = unchanged_sources - section.sources + new_sources = ( + new_symbols.sources - maybe_unchanged_sources - removed_symbols.sources) + removed_sources = ( + removed_symbols.sources - maybe_unchanged_sources - new_symbols.sources) + partially_changed_sources = ( + grown_symbols.sources | shrunk_symbols.sources | new_symbols.sources | + removed_symbols.sources) - removed_sources - new_sources + allFiles = set() + for section in sections: + allFiles = allFiles | section.sources + allFiles = allFiles | maybe_unchanged_sources + print 'Source stats:' + print(' %d sources encountered.' % len(allFiles)) + print(' %d completely new.' % len(new_sources)) + print(' %d removed completely.' % len(removed_sources)) + print(' %d partially changed.' % len(partially_changed_sources)) + print(' %d completely unchanged.' % len(unchanged_sources)) + remainder = (allFiles - new_sources - removed_sources - + partially_changed_sources - unchanged_sources) + assert len(remainder) == 0 + + if not showsources: + return # Per-source analysis, only if requested + print 'Per-source Analysis:' + delta_by_path = {} + for section in sections: + for path in section.symbols_by_path: + entry = delta_by_path.get(path) + if not entry: + entry = {'plus': 0, 'minus': 0} + delta_by_path[path] = entry + for symbol_name, symbol_type, symbol_delta in \ + section.symbols_by_path[path]: + if symbol_delta.old_size is None: + delta = symbol_delta.new_size + elif symbol_delta.new_size is None: + delta = -symbol_delta.old_size + else: + delta = symbol_delta.new_size - symbol_delta.old_size + + if delta > 0: + entry['plus'] += delta + else: + entry['minus'] += (-1 * delta) + + def delta_sort_key(item): + _path, size_data = item + growth = size_data['plus'] - size_data['minus'] + return growth + + for path, size_data in sorted( + delta_by_path.iteritems(), key=delta_sort_key, reverse=True): + gain = size_data['plus'] + loss = size_data['minus'] + delta = size_data['plus'] - size_data['minus'] + header = ' %s - Source: %s - (gained %d, lost %d)' % (DeltaStr(delta), + path, gain, loss) + divider = '-' * len(header) + print '' + print divider + print header + print divider + if showsymbols: + + def ExtractNewSize(tup): + symbol_delta = tup[2] + return symbol_delta.new_size + + def ExtractOldSize(tup): + symbol_delta = tup[2] + return symbol_delta.old_size + + if path in new_symbols.symbols_by_path: + print ' New symbols:' + for symbol_name, symbol_type, symbol_delta in \ + sorted(new_symbols.symbols_by_path[path], + key=ExtractNewSize, + reverse=True): + print(' %8s: %s type=%s, size=%d bytes%s' % + (DeltaStr(symbol_delta.new_size), symbol_name, + symbol_type, symbol_delta.new_size, + SharedInfoStr(symbol_delta))) + if path in removed_symbols.symbols_by_path: + print ' Removed symbols:' + for symbol_name, symbol_type, symbol_delta in \ + sorted(removed_symbols.symbols_by_path[path], + key=ExtractOldSize): + print(' %8s: %s type=%s, size=%d bytes%s' % + (DeltaStr(-symbol_delta.old_size), symbol_name, + symbol_type, symbol_delta.old_size, + SharedInfoStr(symbol_delta))) + for (changed_symbols_by_path, + type_str) in [(grown_symbols.symbols_by_path, "Grown"), + (shrunk_symbols.symbols_by_path, "Shrunk")]: + if path in changed_symbols_by_path: + print ' %s symbols:' % type_str + + def changed_symbol_sortkey(item): + symbol_name, _symbol_type, symbol_delta = item + return (symbol_delta.old_size - symbol_delta.new_size, + symbol_name) + + for symbol_name, symbol_type, symbol_delta in \ + sorted(changed_symbols_by_path[path], key=changed_symbol_sortkey): + print( + ' %8s: %s type=%s, (was %d bytes, now %d bytes)%s' + % (DeltaStr(symbol_delta.new_size - + symbol_delta.old_size), symbol_name, + symbol_type, + symbol_delta.old_size, symbol_delta.new_size, + SharedInfoStr(symbol_delta))) def main(): - usage = """%prog [options] + usage = """%prog [options] Analyzes the symbolic differences between two binary files (typically, not necessarily, two different builds of the same @@ -453,32 +477,42 @@ def main(): Options are available via '--help'. """ - parser = optparse.OptionParser(usage=usage) - parser.add_option('--nm1', metavar='PATH', - help='the nm dump of the first library') - parser.add_option('--nm2', metavar='PATH', - help='the nm dump of the second library') - parser.add_option('--showsources', action='store_true', default=False, - help='show per-source statistics') - parser.add_option('--showsymbols', action='store_true', default=False, - help='show all symbol information; implies --showsources') - parser.add_option('--verbose', action='store_true', default=False, - help='output internal debugging stuff') - opts, _args = parser.parse_args() + parser = optparse.OptionParser(usage=usage) + parser.add_option( + '--nm1', metavar='PATH', help='the nm dump of the first library') + parser.add_option( + '--nm2', metavar='PATH', help='the nm dump of the second library') + parser.add_option( + '--showsources', + action='store_true', + default=False, + help='show per-source statistics') + parser.add_option( + '--showsymbols', + action='store_true', + default=False, + help='show all symbol information; implies --showsources') + parser.add_option( + '--verbose', + action='store_true', + default=False, + help='output internal debugging stuff') + opts, _args = parser.parse_args() + + if not opts.nm1: + parser.error('--nm1 is required') + if not opts.nm2: + parser.error('--nm2 is required') + symbols = [] + for path in [opts.nm1, opts.nm2]: + with file(path, 'r') as nm_input: + if opts.verbose: + print 'parsing ' + path + '...' + symbols.append(list(binary_size_utils.ParseNm(nm_input))) + (added, removed, changed, unchanged) = Compare(symbols[0], symbols[1]) + CrunchStats(added, removed, changed, unchanged, + opts.showsources | opts.showsymbols, opts.showsymbols) - if not opts.nm1: - parser.error('--nm1 is required') - if not opts.nm2: - parser.error('--nm2 is required') - symbols = [] - for path in [opts.nm1, opts.nm2]: - with file(path, 'r') as nm_input: - if opts.verbose: - print 'parsing ' + path + '...' - symbols.append(list(binary_size_utils.ParseNm(nm_input))) - (added, removed, changed, unchanged) = Compare(symbols[0], symbols[1]) - CrunchStats(added, removed, changed, unchanged, - opts.showsources | opts.showsymbols, opts.showsymbols) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/runtime/third_party/binary_size/src/run_binary_size_analysis.py b/runtime/third_party/binary_size/src/run_binary_size_analysis.py index d97858d4539..d78e02d7dbf 100755 --- a/runtime/third_party/binary_size/src/run_binary_size_analysis.py +++ b/runtime/third_party/binary_size/src/run_binary_size_analysis.py @@ -2,7 +2,6 @@ # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Generate a spatial analysis against an arbitrary library. To use, build the 'binary_size_tool' target. Then run this tool, passing @@ -47,166 +46,170 @@ BIG_BUCKET_LIMIT = 3000 def _MkChild(node, name): - child = node[NODE_CHILDREN_KEY].get(name) - if child is None: - child = {NODE_NAME_KEY: name, - NODE_CHILDREN_KEY: {}} - node[NODE_CHILDREN_KEY][name] = child - return child - + child = node[NODE_CHILDREN_KEY].get(name) + if child is None: + child = {NODE_NAME_KEY: name, NODE_CHILDREN_KEY: {}} + node[NODE_CHILDREN_KEY][name] = child + return child def SplitNoPathBucket(node): - """NAME_NO_PATH_BUCKET can be too large for the graphing lib to + """NAME_NO_PATH_BUCKET can be too large for the graphing lib to handle. Split it into sub-buckets in that case.""" - root_children = node[NODE_CHILDREN_KEY] - if NAME_NO_PATH_BUCKET in root_children: - no_path_bucket = root_children[NAME_NO_PATH_BUCKET] - old_children = no_path_bucket[NODE_CHILDREN_KEY] - count = 0 - for symbol_type, symbol_bucket in old_children.iteritems(): - count += len(symbol_bucket[NODE_CHILDREN_KEY]) - if count > BIG_BUCKET_LIMIT: - new_children = {} - no_path_bucket[NODE_CHILDREN_KEY] = new_children - current_bucket = None - index = 0 - for symbol_type, symbol_bucket in old_children.iteritems(): - for symbol_name, value in symbol_bucket[NODE_CHILDREN_KEY].iteritems(): - if index % BIG_BUCKET_LIMIT == 0: - group_no = (index / BIG_BUCKET_LIMIT) + 1 - current_bucket = _MkChild(no_path_bucket, - '%s subgroup %d' % (NAME_NO_PATH_BUCKET, - group_no)) - assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'p' - node[NODE_TYPE_KEY] = 'p' # p for path - index += 1 - symbol_size = value[NODE_SYMBOL_SIZE_KEY] - AddSymbolIntoFileNode(current_bucket, symbol_type, - symbol_name, symbol_size) + root_children = node[NODE_CHILDREN_KEY] + if NAME_NO_PATH_BUCKET in root_children: + no_path_bucket = root_children[NAME_NO_PATH_BUCKET] + old_children = no_path_bucket[NODE_CHILDREN_KEY] + count = 0 + for symbol_type, symbol_bucket in old_children.iteritems(): + count += len(symbol_bucket[NODE_CHILDREN_KEY]) + if count > BIG_BUCKET_LIMIT: + new_children = {} + no_path_bucket[NODE_CHILDREN_KEY] = new_children + current_bucket = None + index = 0 + for symbol_type, symbol_bucket in old_children.iteritems(): + for symbol_name, value in symbol_bucket[ + NODE_CHILDREN_KEY].iteritems(): + if index % BIG_BUCKET_LIMIT == 0: + group_no = (index / BIG_BUCKET_LIMIT) + 1 + current_bucket = _MkChild( + no_path_bucket, + '%s subgroup %d' % (NAME_NO_PATH_BUCKET, group_no)) + assert not NODE_TYPE_KEY in node or node[ + NODE_TYPE_KEY] == 'p' + node[NODE_TYPE_KEY] = 'p' # p for path + index += 1 + symbol_size = value[NODE_SYMBOL_SIZE_KEY] + AddSymbolIntoFileNode(current_bucket, symbol_type, + symbol_name, symbol_size) def MakeChildrenDictsIntoLists(node): - largest_list_len = 0 - if NODE_CHILDREN_KEY in node: - largest_list_len = len(node[NODE_CHILDREN_KEY]) - child_list = [] - for child in node[NODE_CHILDREN_KEY].itervalues(): - child_largest_list_len = MakeChildrenDictsIntoLists(child) - if child_largest_list_len > largest_list_len: - largest_list_len = child_largest_list_len - child_list.append(child) - node[NODE_CHILDREN_KEY] = child_list + largest_list_len = 0 + if NODE_CHILDREN_KEY in node: + largest_list_len = len(node[NODE_CHILDREN_KEY]) + child_list = [] + for child in node[NODE_CHILDREN_KEY].itervalues(): + child_largest_list_len = MakeChildrenDictsIntoLists(child) + if child_largest_list_len > largest_list_len: + largest_list_len = child_largest_list_len + child_list.append(child) + node[NODE_CHILDREN_KEY] = child_list - return largest_list_len + return largest_list_len def AddSymbolIntoFileNode(node, symbol_type, symbol_name, symbol_size): - """Puts symbol into the file path node |node|. + """Puts symbol into the file path node |node|. Returns the number of added levels in tree. I.e. returns 2.""" - # 'node' is the file node and first step is to find its symbol-type bucket. - node[NODE_LAST_PATH_ELEMENT_KEY] = True - node = _MkChild(node, symbol_type) - assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'b' - node[NODE_SYMBOL_TYPE_KEY] = symbol_type - node[NODE_TYPE_KEY] = 'b' # b for bucket + # 'node' is the file node and first step is to find its symbol-type bucket. + node[NODE_LAST_PATH_ELEMENT_KEY] = True + node = _MkChild(node, symbol_type) + assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'b' + node[NODE_SYMBOL_TYPE_KEY] = symbol_type + node[NODE_TYPE_KEY] = 'b' # b for bucket - # 'node' is now the symbol-type bucket. Make the child entry. - node = _MkChild(node, symbol_name) - if NODE_CHILDREN_KEY in node: - if node[NODE_CHILDREN_KEY]: - logging.warning('A container node used as symbol for %s.' % symbol_name) - # This is going to be used as a leaf so no use for child list. - del node[NODE_CHILDREN_KEY] - node[NODE_SYMBOL_SIZE_KEY] = symbol_size - node[NODE_SYMBOL_TYPE_KEY] = symbol_type - node[NODE_TYPE_KEY] = 's' # s for symbol + # 'node' is now the symbol-type bucket. Make the child entry. + node = _MkChild(node, symbol_name) + if NODE_CHILDREN_KEY in node: + if node[NODE_CHILDREN_KEY]: + logging.warning( + 'A container node used as symbol for %s.' % symbol_name) + # This is going to be used as a leaf so no use for child list. + del node[NODE_CHILDREN_KEY] + node[NODE_SYMBOL_SIZE_KEY] = symbol_size + node[NODE_SYMBOL_TYPE_KEY] = symbol_type + node[NODE_TYPE_KEY] = 's' # s for symbol - return 2 # Depth of the added subtree. + return 2 # Depth of the added subtree. def MakeCompactTree(symbols, symbol_path_origin_dir): - result = {NODE_NAME_KEY: '/', - NODE_CHILDREN_KEY: {}, - NODE_TYPE_KEY: 'p', - NODE_MAX_DEPTH_KEY: 0} - seen_symbol_with_path = False - cwd = os.path.abspath(os.getcwd()) - for symbol_name, symbol_type, symbol_size, file_path, _address in symbols: + result = { + NODE_NAME_KEY: '/', + NODE_CHILDREN_KEY: {}, + NODE_TYPE_KEY: 'p', + NODE_MAX_DEPTH_KEY: 0 + } + seen_symbol_with_path = False + cwd = os.path.abspath(os.getcwd()) + for symbol_name, symbol_type, symbol_size, file_path, _address in symbols: - if 'vtable for ' in symbol_name: - symbol_type = '@' # hack to categorize these separately - # Take path like '/foo/bar/baz', convert to ['foo', 'bar', 'baz'] - if file_path and file_path != "??": - file_path = os.path.abspath(os.path.join(symbol_path_origin_dir, - file_path)) - # Let the output structure be relative to $CWD if inside $CWD, - # otherwise relative to the disk root. This is to avoid - # unnecessary click-through levels in the output. - if file_path.startswith(cwd + os.sep): - file_path = file_path[len(cwd):] - if file_path.startswith('/'): - file_path = file_path[1:] - seen_symbol_with_path = True - else: - file_path = NAME_NO_PATH_BUCKET + if 'vtable for ' in symbol_name: + symbol_type = '@' # hack to categorize these separately + # Take path like '/foo/bar/baz', convert to ['foo', 'bar', 'baz'] + if file_path and file_path != "??": + file_path = os.path.abspath( + os.path.join(symbol_path_origin_dir, file_path)) + # Let the output structure be relative to $CWD if inside $CWD, + # otherwise relative to the disk root. This is to avoid + # unnecessary click-through levels in the output. + if file_path.startswith(cwd + os.sep): + file_path = file_path[len(cwd):] + if file_path.startswith('/'): + file_path = file_path[1:] + seen_symbol_with_path = True + else: + file_path = NAME_NO_PATH_BUCKET - path_parts = file_path.split('/') + path_parts = file_path.split('/') - # Find pre-existing node in tree, or update if it already exists - node = result - depth = 0 - while len(path_parts) > 0: - path_part = path_parts.pop(0) - if len(path_part) == 0: - continue - depth += 1 - node = _MkChild(node, path_part) - assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'p' - node[NODE_TYPE_KEY] = 'p' # p for path + # Find pre-existing node in tree, or update if it already exists + node = result + depth = 0 + while len(path_parts) > 0: + path_part = path_parts.pop(0) + if len(path_part) == 0: + continue + depth += 1 + node = _MkChild(node, path_part) + assert not NODE_TYPE_KEY in node or node[NODE_TYPE_KEY] == 'p' + node[NODE_TYPE_KEY] = 'p' # p for path - depth += AddSymbolIntoFileNode(node, symbol_type, symbol_name, symbol_size) - result[NODE_MAX_DEPTH_KEY] = max(result[NODE_MAX_DEPTH_KEY], depth) + depth += AddSymbolIntoFileNode(node, symbol_type, symbol_name, + symbol_size) + result[NODE_MAX_DEPTH_KEY] = max(result[NODE_MAX_DEPTH_KEY], depth) - if not seen_symbol_with_path: - logging.warning('Symbols lack paths. Data will not be structured.') + if not seen_symbol_with_path: + logging.warning('Symbols lack paths. Data will not be structured.') - # The (no path) bucket can be extremely large if we failed to get - # path information. Split it into subgroups if needed. - SplitNoPathBucket(result) + # The (no path) bucket can be extremely large if we failed to get + # path information. Split it into subgroups if needed. + SplitNoPathBucket(result) - largest_list_len = MakeChildrenDictsIntoLists(result) + largest_list_len = MakeChildrenDictsIntoLists(result) - if largest_list_len > BIG_BUCKET_LIMIT: - logging.warning('There are sections with %d nodes. ' - 'Results might be unusable.' % largest_list_len) - return result + if largest_list_len > BIG_BUCKET_LIMIT: + logging.warning('There are sections with %d nodes. ' + 'Results might be unusable.' % largest_list_len) + return result def DumpCompactTree(symbols, symbol_path_origin_dir, outfile): - tree_root = MakeCompactTree(symbols, symbol_path_origin_dir) - with open(outfile, 'w') as out: - out.write('var tree_data=') - # Use separators without whitespace to get a smaller file. - json.dump(tree_root, out, separators=(',', ':')) - print('Writing %d bytes json' % os.path.getsize(outfile)) + tree_root = MakeCompactTree(symbols, symbol_path_origin_dir) + with open(outfile, 'w') as out: + out.write('var tree_data=') + # Use separators without whitespace to get a smaller file. + json.dump(tree_root, out, separators=(',', ':')) + print('Writing %d bytes json' % os.path.getsize(outfile)) def MakeSourceMap(symbols): - sources = {} - for _sym, _symbol_type, size, path, _address in symbols: - key = None - if path: - key = os.path.normpath(path) - else: - key = '[no path]' - if key not in sources: - sources[key] = {'path': path, 'symbol_count': 0, 'size': 0} - record = sources[key] - record['size'] += size - record['symbol_count'] += 1 - return sources + sources = {} + for _sym, _symbol_type, size, path, _address in symbols: + key = None + if path: + key = os.path.normpath(path) + else: + key = '[no path]' + if key not in sources: + sources[key] = {'path': path, 'symbol_count': 0, 'size': 0} + record = sources[key] + record['size'] += size + record['symbol_count'] += 1 + return sources # Regex for parsing "nm" output. A sample line looks like this: @@ -224,285 +227,302 @@ def MakeSourceMap(symbols): # [\t]? Tab separator # (.*) The location (filename[:linennum|?][ (discriminator n)] sNmPattern = re.compile( - r'([0-9a-f]{8,})[\s]+([0-9a-f]{8,})[\s]*(\S?)[\s*]([^\t]*)[\t]?(.*)') + r'([0-9a-f]{8,})[\s]+([0-9a-f]{8,})[\s]*(\S?)[\s*]([^\t]*)[\t]?(.*)') + class Progress(): - def __init__(self): - self.count = 0 - self.skip_count = 0 - self.collisions = 0 - self.time_last_output = time.time() - self.count_last_output = 0 - self.disambiguations = 0 - self.was_ambiguous = 0 + + def __init__(self): + self.count = 0 + self.skip_count = 0 + self.collisions = 0 + self.time_last_output = time.time() + self.count_last_output = 0 + self.disambiguations = 0 + self.was_ambiguous = 0 def RunElfSymbolizer(outfile, library, addr2line_binary, nm_binary, jobs, disambiguate, src_path): - nm_output = RunNm(library, nm_binary) - nm_output_lines = nm_output.splitlines() - nm_output_lines_len = len(nm_output_lines) - address_symbol = {} - progress = Progress() - def map_address_symbol(symbol, addr): - progress.count += 1 - if addr in address_symbol: - # 'Collision between %s and %s.' % (str(symbol.name), - # str(address_symbol[addr].name)) - progress.collisions += 1 - else: - if symbol.disambiguated: - progress.disambiguations += 1 - if symbol.was_ambiguous: - progress.was_ambiguous += 1 + nm_output = RunNm(library, nm_binary) + nm_output_lines = nm_output.splitlines() + nm_output_lines_len = len(nm_output_lines) + address_symbol = {} + progress = Progress() - address_symbol[addr] = symbol - - progress_output() - - def progress_output(): - progress_chunk = 100 - if progress.count % progress_chunk == 0: - time_now = time.time() - time_spent = time_now - progress.time_last_output - if time_spent > 1.0: - # Only output at most once per second. - progress.time_last_output = time_now - chunk_size = progress.count - progress.count_last_output - progress.count_last_output = progress.count - if time_spent > 0: - speed = chunk_size / time_spent + def map_address_symbol(symbol, addr): + progress.count += 1 + if addr in address_symbol: + # 'Collision between %s and %s.' % (str(symbol.name), + # str(address_symbol[addr].name)) + progress.collisions += 1 else: - speed = 0 - progress_percent = (100.0 * (progress.count + progress.skip_count) / - nm_output_lines_len) - disambiguation_percent = 0 - if progress.disambiguations != 0: - disambiguation_percent = (100.0 * progress.disambiguations / - progress.was_ambiguous) + if symbol.disambiguated: + progress.disambiguations += 1 + if symbol.was_ambiguous: + progress.was_ambiguous += 1 - sys.stdout.write('\r%.1f%%: Looked up %d symbols (%d collisions, ' - '%d disambiguations where %.1f%% succeeded)' - ' - %.1f lookups/s.' % - (progress_percent, progress.count, progress.collisions, - progress.disambiguations, disambiguation_percent, speed)) + address_symbol[addr] = symbol - # In case disambiguation was disabled, we remove the source path (which upon - # being set signals the symbolizer to enable disambiguation) - if not disambiguate: - src_path = None - symbolizer = elf_symbolizer.ELFSymbolizer(library, addr2line_binary, - map_address_symbol, - max_concurrent_jobs=jobs, - source_root_path=src_path) - user_interrupted = False - try: - for line in nm_output_lines: - match = sNmPattern.match(line) - if match: - location = match.group(5) - if not location: - addr = int(match.group(1), 16) - size = int(match.group(2), 16) - if addr in address_symbol: # Already looked up, shortcut - # ELFSymbolizer. - map_address_symbol(address_symbol[addr], addr) - continue - elif size == 0: - # Save time by not looking up empty symbols (do they even exist?) - print('Empty symbol: ' + line) - else: - symbolizer.SymbolizeAsync(addr, addr) - continue + progress_output() - progress.skip_count += 1 - except KeyboardInterrupt: - user_interrupted = True - print('Interrupting - killing subprocesses. Please wait.') + def progress_output(): + progress_chunk = 100 + if progress.count % progress_chunk == 0: + time_now = time.time() + time_spent = time_now - progress.time_last_output + if time_spent > 1.0: + # Only output at most once per second. + progress.time_last_output = time_now + chunk_size = progress.count - progress.count_last_output + progress.count_last_output = progress.count + if time_spent > 0: + speed = chunk_size / time_spent + else: + speed = 0 + progress_percent = (100.0 * ( + progress.count + progress.skip_count) / nm_output_lines_len) + disambiguation_percent = 0 + if progress.disambiguations != 0: + disambiguation_percent = (100.0 * progress.disambiguations / + progress.was_ambiguous) - try: - symbolizer.Join() - except KeyboardInterrupt: - # Don't want to abort here since we will be finished in a few seconds. - user_interrupted = True - print('Patience you must have my young padawan.') + sys.stdout.write( + '\r%.1f%%: Looked up %d symbols (%d collisions, ' + '%d disambiguations where %.1f%% succeeded)' + ' - %.1f lookups/s.' % + (progress_percent, progress.count, progress.collisions, + progress.disambiguations, disambiguation_percent, speed)) - print '' + # In case disambiguation was disabled, we remove the source path (which upon + # being set signals the symbolizer to enable disambiguation) + if not disambiguate: + src_path = None + symbolizer = elf_symbolizer.ELFSymbolizer( + library, + addr2line_binary, + map_address_symbol, + max_concurrent_jobs=jobs, + source_root_path=src_path) + user_interrupted = False + try: + for line in nm_output_lines: + match = sNmPattern.match(line) + if match: + location = match.group(5) + if not location: + addr = int(match.group(1), 16) + size = int(match.group(2), 16) + if addr in address_symbol: # Already looked up, shortcut + # ELFSymbolizer. + map_address_symbol(address_symbol[addr], addr) + continue + elif size == 0: + # Save time by not looking up empty symbols (do they even exist?) + print('Empty symbol: ' + line) + else: + symbolizer.SymbolizeAsync(addr, addr) + continue - if user_interrupted: - print('Skipping the rest of the file mapping. ' - 'Output will not be fully classified.') + progress.skip_count += 1 + except KeyboardInterrupt: + user_interrupted = True + print('Interrupting - killing subprocesses. Please wait.') - symbol_path_origin_dir = os.path.dirname(os.path.abspath(library)) + try: + symbolizer.Join() + except KeyboardInterrupt: + # Don't want to abort here since we will be finished in a few seconds. + user_interrupted = True + print('Patience you must have my young padawan.') - with open(outfile, 'w') as out: - for line in nm_output_lines: - match = sNmPattern.match(line) - if match: - location = match.group(5) - if not location: - addr = int(match.group(1), 16) - symbol = address_symbol.get(addr) - if symbol is not None: - path = '??' - if symbol.source_path is not None: - path = os.path.abspath(os.path.join(symbol_path_origin_dir, - symbol.source_path)) - line_number = 0 - if symbol.source_line is not None: - line_number = symbol.source_line - out.write('%s\t%s:%d\n' % (line, path, line_number)) - continue + print '' - out.write('%s\n' % line) + if user_interrupted: + print('Skipping the rest of the file mapping. ' + 'Output will not be fully classified.') - print('%d symbols in the results.' % len(address_symbol)) + symbol_path_origin_dir = os.path.dirname(os.path.abspath(library)) + + with open(outfile, 'w') as out: + for line in nm_output_lines: + match = sNmPattern.match(line) + if match: + location = match.group(5) + if not location: + addr = int(match.group(1), 16) + symbol = address_symbol.get(addr) + if symbol is not None: + path = '??' + if symbol.source_path is not None: + path = os.path.abspath( + os.path.join(symbol_path_origin_dir, + symbol.source_path)) + line_number = 0 + if symbol.source_line is not None: + line_number = symbol.source_line + out.write('%s\t%s:%d\n' % (line, path, line_number)) + continue + + out.write('%s\n' % line) + + print('%d symbols in the results.' % len(address_symbol)) def RunNm(binary, nm_binary): - cmd = [nm_binary, '-C', '--print-size', '--size-sort', '--reverse-sort', - binary] - nm_process = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - (process_output, err_output) = nm_process.communicate() + cmd = [ + nm_binary, '-C', '--print-size', '--size-sort', '--reverse-sort', binary + ] + nm_process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (process_output, err_output) = nm_process.communicate() - if nm_process.returncode != 0: - if err_output: - raise Exception, err_output - else: - raise Exception, process_output + if nm_process.returncode != 0: + if err_output: + raise Exception, err_output + else: + raise Exception, process_output - return process_output + return process_output -def GetNmSymbols(nm_infile, outfile, library, jobs, verbose, - addr2line_binary, nm_binary, disambiguate, src_path): - if nm_infile is None: - if outfile is None: - outfile = tempfile.NamedTemporaryFile(delete=False).name +def GetNmSymbols(nm_infile, outfile, library, jobs, verbose, addr2line_binary, + nm_binary, disambiguate, src_path): + if nm_infile is None: + if outfile is None: + outfile = tempfile.NamedTemporaryFile(delete=False).name - if verbose: - print 'Running parallel addr2line, dumping symbols to ' + outfile - RunElfSymbolizer(outfile, library, addr2line_binary, nm_binary, jobs, - disambiguate, src_path) + if verbose: + print 'Running parallel addr2line, dumping symbols to ' + outfile + RunElfSymbolizer(outfile, library, addr2line_binary, nm_binary, jobs, + disambiguate, src_path) - nm_infile = outfile + nm_infile = outfile - elif verbose: - print 'Using nm input from ' + nm_infile - with file(nm_infile, 'r') as infile: - return list(binary_size_utils.ParseNm(infile)) + elif verbose: + print 'Using nm input from ' + nm_infile + with file(nm_infile, 'r') as infile: + return list(binary_size_utils.ParseNm(infile)) -PAK_RESOURCE_ID_TO_STRING = { "inited": False } +PAK_RESOURCE_ID_TO_STRING = {"inited": False} + def LoadPakIdsFromResourceFile(filename): - """Given a file name, it loads everything that looks like a resource id + """Given a file name, it loads everything that looks like a resource id into PAK_RESOURCE_ID_TO_STRING.""" - with open(filename) as resource_header: - for line in resource_header: - if line.startswith("#define "): - line_data = line.split() - if len(line_data) == 3: - try: - resource_number = int(line_data[2]) - resource_name = line_data[1] - PAK_RESOURCE_ID_TO_STRING[resource_number] = resource_name - except ValueError: - pass + with open(filename) as resource_header: + for line in resource_header: + if line.startswith("#define "): + line_data = line.split() + if len(line_data) == 3: + try: + resource_number = int(line_data[2]) + resource_name = line_data[1] + PAK_RESOURCE_ID_TO_STRING[ + resource_number] = resource_name + except ValueError: + pass + def GetReadablePakResourceName(pak_file, resource_id): - """Pak resources have a numeric identifier. It is not helpful when + """Pak resources have a numeric identifier. It is not helpful when trying to locate where footprint is generated. This does its best to map the number to a usable string.""" - if not PAK_RESOURCE_ID_TO_STRING['inited']: - # Try to find resource header files generated by grit when - # building the pak file. We'll look for files named *resources.h" - # and lines of the type: - # #define MY_RESOURCE_JS 1234 - PAK_RESOURCE_ID_TO_STRING['inited'] = True - gen_dir = os.path.join(os.path.dirname(pak_file), 'gen') - if os.path.isdir(gen_dir): - for dirname, _dirs, files in os.walk(gen_dir): - for filename in files: - if filename.endswith('resources.h'): - LoadPakIdsFromResourceFile(os.path.join(dirname, filename)) - return PAK_RESOURCE_ID_TO_STRING.get(resource_id, - 'Pak Resource %d' % resource_id) + if not PAK_RESOURCE_ID_TO_STRING['inited']: + # Try to find resource header files generated by grit when + # building the pak file. We'll look for files named *resources.h" + # and lines of the type: + # #define MY_RESOURCE_JS 1234 + PAK_RESOURCE_ID_TO_STRING['inited'] = True + gen_dir = os.path.join(os.path.dirname(pak_file), 'gen') + if os.path.isdir(gen_dir): + for dirname, _dirs, files in os.walk(gen_dir): + for filename in files: + if filename.endswith('resources.h'): + LoadPakIdsFromResourceFile( + os.path.join(dirname, filename)) + return PAK_RESOURCE_ID_TO_STRING.get(resource_id, + 'Pak Resource %d' % resource_id) + def AddPakData(symbols, pak_file): - """Adds pseudo-symbols from a pak file.""" - pak_file = os.path.abspath(pak_file) - with open(pak_file, 'rb') as pak: - data = pak.read() + """Adds pseudo-symbols from a pak file.""" + pak_file = os.path.abspath(pak_file) + with open(pak_file, 'rb') as pak: + data = pak.read() - PAK_FILE_VERSION = 4 - HEADER_LENGTH = 2 * 4 + 1 # Two uint32s. (file version, number of entries) - # and one uint8 (encoding of text resources) - INDEX_ENTRY_SIZE = 2 + 4 # Each entry is a uint16 and a uint32. - version, num_entries, _encoding = struct.unpack(' 0: - # Read the index and data. - data = data[HEADER_LENGTH:] - for _ in range(num_entries): - resource_id, offset = struct.unpack(' 0: + # Read the index and data. + data = data[HEADER_LENGTH:] + for _ in range(num_entries): + resource_id, offset = struct.unpack(' 2 or major == 2 and minor > 22 + tool_output = subprocess.check_output([addr2line_binary, '--version']) + version_re = re.compile(r'^GNU [^ ]+ .* (\d+).(\d+).*?$', re.M) + parsed_output = version_re.match(tool_output) + major = int(parsed_output.group(1)) + minor = int(parsed_output.group(2)) + supports_dwarf4 = major > 2 or major == 2 and minor > 22 - if supports_dwarf4: - return + if supports_dwarf4: + return - print('Checking version of debug information in %s.' % library) - debug_info = subprocess.check_output(['readelf', '--debug-dump=info', - '--dwarf-depth=1', library]) - dwarf_version_re = re.compile(r'^\s+Version:\s+(\d+)$', re.M) - parsed_dwarf_format_output = dwarf_version_re.search(debug_info) - version = int(parsed_dwarf_format_output.group(1)) - if version > 2: - print('The supplied tools only support DWARF2 debug data but the binary\n' + - 'uses DWARF%d. Update the tools or compile the binary\n' % version + - 'with -gdwarf-2.') - sys.exit(1) + print('Checking version of debug information in %s.' % library) + debug_info = subprocess.check_output( + ['readelf', '--debug-dump=info', '--dwarf-depth=1', library]) + dwarf_version_re = re.compile(r'^\s+Version:\s+(\d+)$', re.M) + parsed_dwarf_format_output = dwarf_version_re.search(debug_info) + version = int(parsed_dwarf_format_output.group(1)) + if version > 2: + print( + 'The supplied tools only support DWARF2 debug data but the binary\n' + + 'uses DWARF%d. Update the tools or compile the binary\n' % version + + 'with -gdwarf-2.') + sys.exit(1) def main(): - usage = """%prog [options] + usage = """%prog [options] Runs a spatial analysis on a given library, looking up the source locations of its symbols and calculating how much space each directory, source file, @@ -519,148 +539,170 @@ def main(): Other options are available via '--help'. """ - parser = optparse.OptionParser(usage=usage) - parser.add_option('--nm-in', metavar='PATH', - help='if specified, use nm input from instead of ' - 'generating it. Note that source locations should be ' - 'present in the file; i.e., no addr2line symbol lookups ' - 'will be performed when this option is specified. ' - 'Mutually exclusive with --library.') - parser.add_option('--destdir', metavar='PATH', - help='write output to the specified directory. An HTML ' - 'report is generated here along with supporting files; ' - 'any existing report will be overwritten.') - parser.add_option('--library', metavar='PATH', - help='if specified, process symbols in the library at ' - 'the specified path. Mutually exclusive with --nm-in.') - parser.add_option('--pak', metavar='PATH', - help='if specified, includes the contents of the ' - 'specified *.pak file in the output.') - parser.add_option('--nm-binary', - help='use the specified nm binary to analyze library. ' - 'This is to be used when the nm in the path is not for ' - 'the right architecture or of the right version.') - parser.add_option('--addr2line-binary', - help='use the specified addr2line binary to analyze ' - 'library. This is to be used when the addr2line in ' - 'the path is not for the right architecture or ' - 'of the right version.') - parser.add_option('--jobs', type='int', - help='number of jobs to use for the parallel ' - 'addr2line processing pool; defaults to 1. More ' - 'jobs greatly improve throughput but eat RAM like ' - 'popcorn, and take several gigabytes each. Start low ' - 'and ramp this number up until your machine begins to ' - 'struggle with RAM. ' - 'This argument is only valid when using --library.') - parser.add_option('-v', '--verbose', dest='verbose', action='store_true', - help='be verbose, printing lots of status information.') - parser.add_option('--nm-out', metavar='PATH', - help='(deprecated) No-op. nm.out is stored in --destdir.') - parser.add_option('--no-nm-out', action='store_true', - help='do not keep the nm output file. This file is useful ' - 'if you want to see the fully processed nm output after ' - 'the symbols have been mapped to source locations, or if ' - 'you plan to run explain_binary_size_delta.py. By default ' - 'the file \'nm.out\' is placed alongside the generated ' - 'report. The nm.out file is only created when using ' - '--library.') - parser.add_option('--disable-disambiguation', action='store_true', - help='disables the disambiguation process altogether,' - ' NOTE: this may, depending on your toolchain, produce' - ' output with some symbols at the top layer if addr2line' - ' could not get the entire source path.') - parser.add_option('--source-path', default='./', - help='the path to the source code of the output binary, ' - 'default set to current directory. Used in the' - ' disambiguation process.') - opts, _args = parser.parse_args() + parser = optparse.OptionParser(usage=usage) + parser.add_option( + '--nm-in', + metavar='PATH', + help='if specified, use nm input from instead of ' + 'generating it. Note that source locations should be ' + 'present in the file; i.e., no addr2line symbol lookups ' + 'will be performed when this option is specified. ' + 'Mutually exclusive with --library.') + parser.add_option( + '--destdir', + metavar='PATH', + help='write output to the specified directory. An HTML ' + 'report is generated here along with supporting files; ' + 'any existing report will be overwritten.') + parser.add_option( + '--library', + metavar='PATH', + help='if specified, process symbols in the library at ' + 'the specified path. Mutually exclusive with --nm-in.') + parser.add_option( + '--pak', + metavar='PATH', + help='if specified, includes the contents of the ' + 'specified *.pak file in the output.') + parser.add_option( + '--nm-binary', + help='use the specified nm binary to analyze library. ' + 'This is to be used when the nm in the path is not for ' + 'the right architecture or of the right version.') + parser.add_option( + '--addr2line-binary', + help='use the specified addr2line binary to analyze ' + 'library. This is to be used when the addr2line in ' + 'the path is not for the right architecture or ' + 'of the right version.') + parser.add_option( + '--jobs', + type='int', + help='number of jobs to use for the parallel ' + 'addr2line processing pool; defaults to 1. More ' + 'jobs greatly improve throughput but eat RAM like ' + 'popcorn, and take several gigabytes each. Start low ' + 'and ramp this number up until your machine begins to ' + 'struggle with RAM. ' + 'This argument is only valid when using --library.') + parser.add_option( + '-v', + '--verbose', + dest='verbose', + action='store_true', + help='be verbose, printing lots of status information.') + parser.add_option( + '--nm-out', + metavar='PATH', + help='(deprecated) No-op. nm.out is stored in --destdir.') + parser.add_option( + '--no-nm-out', + action='store_true', + help='do not keep the nm output file. This file is useful ' + 'if you want to see the fully processed nm output after ' + 'the symbols have been mapped to source locations, or if ' + 'you plan to run explain_binary_size_delta.py. By default ' + 'the file \'nm.out\' is placed alongside the generated ' + 'report. The nm.out file is only created when using ' + '--library.') + parser.add_option( + '--disable-disambiguation', + action='store_true', + help='disables the disambiguation process altogether,' + ' NOTE: this may, depending on your toolchain, produce' + ' output with some symbols at the top layer if addr2line' + ' could not get the entire source path.') + parser.add_option( + '--source-path', + default='./', + help='the path to the source code of the output binary, ' + 'default set to current directory. Used in the' + ' disambiguation process.') + opts, _args = parser.parse_args() - if ((not opts.library) and (not opts.nm_in)) or (opts.library and opts.nm_in): - parser.error('exactly one of --library or --nm-in is required') - if opts.nm_out: - print >> sys.stderr, ('WARNING: --nm-out is deprecated and has no effect.') - if (opts.nm_in): - if opts.jobs: - print >> sys.stderr, ('WARNING: --jobs has no effect ' - 'when used with --nm-in') - if not opts.destdir: - parser.error('--destdir is a required argument') - if not opts.jobs: - # Use the number of processors but cap between 2 and 4 since raw - # CPU power isn't the limiting factor. It's I/O limited, memory - # bus limited and available-memory-limited. Too many processes and - # the computer will run out of memory and it will be slow. - opts.jobs = max(2, min(4, str(multiprocessing.cpu_count()))) + if ((not opts.library) and + (not opts.nm_in)) or (opts.library and opts.nm_in): + parser.error('exactly one of --library or --nm-in is required') + if opts.nm_out: + print >> sys.stderr, ( + 'WARNING: --nm-out is deprecated and has no effect.') + if (opts.nm_in): + if opts.jobs: + print >> sys.stderr, ('WARNING: --jobs has no effect ' + 'when used with --nm-in') + if not opts.destdir: + parser.error('--destdir is a required argument') + if not opts.jobs: + # Use the number of processors but cap between 2 and 4 since raw + # CPU power isn't the limiting factor. It's I/O limited, memory + # bus limited and available-memory-limited. Too many processes and + # the computer will run out of memory and it will be slow. + opts.jobs = max(2, min(4, str(multiprocessing.cpu_count()))) - if opts.addr2line_binary: - assert os.path.isfile(opts.addr2line_binary) - addr2line_binary = opts.addr2line_binary - else: - addr2line_binary = _find_in_system_path('addr2line') - assert addr2line_binary, 'Unable to find addr2line in the path. '\ - 'Use --addr2line-binary to specify location.' + if opts.addr2line_binary: + assert os.path.isfile(opts.addr2line_binary) + addr2line_binary = opts.addr2line_binary + else: + addr2line_binary = _find_in_system_path('addr2line') + assert addr2line_binary, 'Unable to find addr2line in the path. '\ + 'Use --addr2line-binary to specify location.' - if opts.nm_binary: - assert os.path.isfile(opts.nm_binary) - nm_binary = opts.nm_binary - else: - nm_binary = _find_in_system_path('nm') - assert nm_binary, 'Unable to find nm in the path. Use --nm-binary '\ - 'to specify location.' + if opts.nm_binary: + assert os.path.isfile(opts.nm_binary) + nm_binary = opts.nm_binary + else: + nm_binary = _find_in_system_path('nm') + assert nm_binary, 'Unable to find nm in the path. Use --nm-binary '\ + 'to specify location.' - if opts.pak: - assert os.path.isfile(opts.pak), 'Could not find ' % opts.pak + if opts.pak: + assert os.path.isfile(opts.pak), 'Could not find ' % opts.pak - print('addr2line: %s' % addr2line_binary) - print('nm: %s' % nm_binary) + print('addr2line: %s' % addr2line_binary) + print('nm: %s' % nm_binary) - if opts.library: - CheckDebugFormatSupport(opts.library, addr2line_binary) + if opts.library: + CheckDebugFormatSupport(opts.library, addr2line_binary) - # Prepare output directory and report guts - if not os.path.exists(opts.destdir): - os.makedirs(opts.destdir, 0755) - nm_out = os.path.join(opts.destdir, 'nm.out') - if opts.no_nm_out: - nm_out = None + # Prepare output directory and report guts + if not os.path.exists(opts.destdir): + os.makedirs(opts.destdir, 0755) + nm_out = os.path.join(opts.destdir, 'nm.out') + if opts.no_nm_out: + nm_out = None - # Copy report boilerplate into output directory. This also proves that the - # output directory is safe for writing, so there should be no problems writing - # the nm.out file later. - data_js_file_name = os.path.join(opts.destdir, 'data.js') - d3_out = os.path.join(opts.destdir, 'd3') - if not os.path.exists(d3_out): - os.makedirs(d3_out, 0755) - d3_src = os.path.join(os.path.dirname(__file__), - '..', - '..', - 'd3', 'src') - template_src = os.path.join(os.path.dirname(__file__), - 'template') - shutil.copy(os.path.join(d3_src, 'LICENSE'), d3_out) - shutil.copy(os.path.join(d3_src, 'd3.js'), d3_out) - shutil.copy(os.path.join(template_src, 'index.html'), opts.destdir) - shutil.copy(os.path.join(template_src, 'D3SymbolTreeMap.js'), opts.destdir) + # Copy report boilerplate into output directory. This also proves that the + # output directory is safe for writing, so there should be no problems writing + # the nm.out file later. + data_js_file_name = os.path.join(opts.destdir, 'data.js') + d3_out = os.path.join(opts.destdir, 'd3') + if not os.path.exists(d3_out): + os.makedirs(d3_out, 0755) + d3_src = os.path.join(os.path.dirname(__file__), '..', '..', 'd3', 'src') + template_src = os.path.join(os.path.dirname(__file__), 'template') + shutil.copy(os.path.join(d3_src, 'LICENSE'), d3_out) + shutil.copy(os.path.join(d3_src, 'd3.js'), d3_out) + shutil.copy(os.path.join(template_src, 'index.html'), opts.destdir) + shutil.copy(os.path.join(template_src, 'D3SymbolTreeMap.js'), opts.destdir) - # Run nm and/or addr2line to gather the data - symbols = GetNmSymbols(opts.nm_in, nm_out, opts.library, - opts.jobs, opts.verbose is True, - addr2line_binary, nm_binary, - opts.disable_disambiguation is None, - opts.source_path) + # Run nm and/or addr2line to gather the data + symbols = GetNmSymbols(opts.nm_in, nm_out, opts.library, opts.jobs, + opts.verbose is True, addr2line_binary, nm_binary, + opts.disable_disambiguation is None, + opts.source_path) + + # Post-processing + if opts.pak: + AddPakData(symbols, opts.pak) + if opts.library: + symbol_path_origin_dir = os.path.dirname(os.path.abspath(opts.library)) + else: + # Just a guess. Hopefully all paths in the input file are absolute. + symbol_path_origin_dir = os.path.abspath(os.getcwd()) + # Dump JSON for the HTML report. + DumpCompactTree(symbols, symbol_path_origin_dir, data_js_file_name) + print 'Report saved to ' + opts.destdir + '/index.html' - # Post-processing - if opts.pak: - AddPakData(symbols, opts.pak) - if opts.library: - symbol_path_origin_dir = os.path.dirname(os.path.abspath(opts.library)) - else: - # Just a guess. Hopefully all paths in the input file are absolute. - symbol_path_origin_dir = os.path.abspath(os.getcwd()) - # Dump JSON for the HTML report. - DumpCompactTree(symbols, symbol_path_origin_dir, data_js_file_name) - print 'Report saved to ' + opts.destdir + '/index.html' if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/runtime/tools/android_finder.py b/runtime/tools/android_finder.py index 93c53fc12c0..c3d82494f54 100755 --- a/runtime/tools/android_finder.py +++ b/runtime/tools/android_finder.py @@ -3,7 +3,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Find an Android device with a given ABI. @@ -20,210 +19,222 @@ import sys import traceback import utils - DEBUG = False VERBOSE = False def BuildOptions(): - result = optparse.OptionParser() - result.add_option( - "-a", "--abi", - action="store", type="string", - help="Desired ABI. armeabi-v7a or x86.") - result.add_option( - "-b", "--bootstrap", - help='Bootstrap - create an emulator, installing SDK packages if needed.', - default=False, action="store_true") - result.add_option( - "-d", "--debug", - help='Turn on debugging diagnostics.', - default=False, action="store_true") - result.add_option( - "-v", "--verbose", - help='Verbose output.', - default=False, action="store_true") - return result + result = optparse.OptionParser() + result.add_option( + "-a", + "--abi", + action="store", + type="string", + help="Desired ABI. armeabi-v7a or x86.") + result.add_option( + "-b", + "--bootstrap", + help= + 'Bootstrap - create an emulator, installing SDK packages if needed.', + default=False, + action="store_true") + result.add_option( + "-d", + "--debug", + help='Turn on debugging diagnostics.', + default=False, + action="store_true") + result.add_option( + "-v", + "--verbose", + help='Verbose output.', + default=False, + action="store_true") + return result def ProcessOptions(options): - global DEBUG - DEBUG = options.debug - global VERBOSE - VERBOSE = options.verbose - if options.abi is None: - sys.stderr.write('--abi not specified.\n') - return False - return True + global DEBUG + DEBUG = options.debug + global VERBOSE + VERBOSE = options.verbose + if options.abi is None: + sys.stderr.write('--abi not specified.\n') + return False + return True def ParseAndroidListSdkResult(text): - """ + """ Parse the output of an 'android list sdk' command. Return list of (id-num, id-key, type, description). """ - header_regex = re.compile( - r'Packages available for installation or update: \d+\n') - packages = re.split(header_regex, text) - if len(packages) != 2: - raise utils.Error("Could not get a list of packages to install") - entry_regex = re.compile( - r'^id\: (\d+) or "([^"]*)"\n\s*Type\: ([^\n]*)\n\s*Desc\: (.*)') - entries = [] - for entry in packages[1].split('----------\n'): - match = entry_regex.match(entry) - if match == None: - continue - entries.append((int(match.group(1)), match.group(2), match.group(3), - match.group(4))) - return entries + header_regex = re.compile( + r'Packages available for installation or update: \d+\n') + packages = re.split(header_regex, text) + if len(packages) != 2: + raise utils.Error("Could not get a list of packages to install") + entry_regex = re.compile( + r'^id\: (\d+) or "([^"]*)"\n\s*Type\: ([^\n]*)\n\s*Desc\: (.*)') + entries = [] + for entry in packages[1].split('----------\n'): + match = entry_regex.match(entry) + if match == None: + continue + entries.append((int(match.group(1)), match.group(2), match.group(3), + match.group(4))) + return entries def AndroidListSdk(): - return ParseAndroidListSdkResult(utils.RunCommand( - ["android", "list", "sdk", "-a", "-e"])) + return ParseAndroidListSdkResult( + utils.RunCommand(["android", "list", "sdk", "-a", "-e"])) def AndroidSdkFindPackage(packages, key): - """ + """ Args: packages: list of (id-num, id-key, type, description). key: (id-key, type, description-prefix). """ - (key_id, key_type, key_description_prefix) = key - for package in packages: - (package_num, package_id, package_type, package_description) = package - if (package_id == key_id and package_type == key_type - and package_description.startswith(key_description_prefix)): - return package - return None + (key_id, key_type, key_description_prefix) = key + for package in packages: + (package_num, package_id, package_type, package_description) = package + if (package_id == key_id and package_type == key_type and + package_description.startswith(key_description_prefix)): + return package + return None def EnsureSdkPackageInstalled(packages, key): - """ + """ Makes sure the package with a given key is installed. key is (id-key, type, description-prefix) Returns True if the package was not already installed. """ - entry = AndroidSdkFindPackage(packages, key) - if entry is None: - raise utils.Error("Could not find a package for key %s" % key) - packageId = entry[0] - if VERBOSE: - sys.stderr.write('Checking Android SDK package %s...\n' % str(entry)) - out = utils.RunCommand( - ["android", "update", "sdk", "-a", "-u", "--filter", str(packageId)]) - return '\nInstalling Archives:\n' in out + entry = AndroidSdkFindPackage(packages, key) + if entry is None: + raise utils.Error("Could not find a package for key %s" % key) + packageId = entry[0] + if VERBOSE: + sys.stderr.write('Checking Android SDK package %s...\n' % str(entry)) + out = utils.RunCommand( + ["android", "update", "sdk", "-a", "-u", "--filter", + str(packageId)]) + return '\nInstalling Archives:\n' in out def SdkPackagesForAbi(abi): - packagesForAbi = { - 'armeabi-v7a': [ - # The platform needed to install the armeabi ABI system image: - ('android-15', 'Platform', 'Android SDK Platform 4.0.3'), - # The armeabi-v7a ABI system image: - ('sysimg-15', 'SystemImage', 'Android SDK Platform 4.0.3') - ], - 'x86': [ - # The platform needed to install the x86 ABI system image: - ('android-15', 'Platform', 'Android SDK Platform 4.0.3'), - # The x86 ABI system image: - ('sysimg-15', 'SystemImage', 'Android SDK Platform 4.0.4') - ] - } + packagesForAbi = { + 'armeabi-v7a': [ + # The platform needed to install the armeabi ABI system image: + ('android-15', 'Platform', 'Android SDK Platform 4.0.3'), + # The armeabi-v7a ABI system image: + ('sysimg-15', 'SystemImage', 'Android SDK Platform 4.0.3') + ], + 'x86': [ + # The platform needed to install the x86 ABI system image: + ('android-15', 'Platform', 'Android SDK Platform 4.0.3'), + # The x86 ABI system image: + ('sysimg-15', 'SystemImage', 'Android SDK Platform 4.0.4') + ] + } - if abi not in packagesForAbi: - raise utils.Error('Unsupported abi %s' % abi) - return packagesForAbi[abi] + if abi not in packagesForAbi: + raise utils.Error('Unsupported abi %s' % abi) + return packagesForAbi[abi] def TargetForAbi(abi): - for package in SdkPackagesForAbi(abi): - if package[1] == 'Platform': - return package[0] + for package in SdkPackagesForAbi(abi): + if package[1] == 'Platform': + return package[0] def EnsureAndroidSdkPackagesInstalled(abi): - """Return true if at least one package was not already installed.""" - abiPackageList = SdkPackagesForAbi(abi) - installedSomething = False - packages = AndroidListSdk() - for package in abiPackageList: - installedSomething |= EnsureSdkPackageInstalled(packages, package) - return installedSomething + """Return true if at least one package was not already installed.""" + abiPackageList = SdkPackagesForAbi(abi) + installedSomething = False + packages = AndroidListSdk() + for package in abiPackageList: + installedSomething |= EnsureSdkPackageInstalled(packages, package) + return installedSomething def ParseAndroidListAvdResult(text): - """ + """ Parse the output of an 'android list avd' command. Return List of {Name: Path: Target: ABI: Skin: Sdcard:} """ - text = text.split('Available Android Virtual Devices:\n')[-1] - text = text.split( - 'The following Android Virtual Devices could not be loaded:\n')[0] - result = [] - line_re = re.compile(r'^\s*([^\:]+)\:\s*(.*)$') - for chunk in text.split('\n---------\n'): - entry = {} - for line in chunk.split('\n'): - line = line.strip() - if len(line) == 0: - continue - match = line_re.match(line) - if match is None: - sys.stderr.write('Match fail %s\n' % str(line)) - continue - #raise utils.Error('Match failed') - entry[match.group(1)] = match.group(2) - if len(entry) > 0: - result.append(entry) - return result + text = text.split('Available Android Virtual Devices:\n')[-1] + text = text.split( + 'The following Android Virtual Devices could not be loaded:\n')[0] + result = [] + line_re = re.compile(r'^\s*([^\:]+)\:\s*(.*)$') + for chunk in text.split('\n---------\n'): + entry = {} + for line in chunk.split('\n'): + line = line.strip() + if len(line) == 0: + continue + match = line_re.match(line) + if match is None: + sys.stderr.write('Match fail %s\n' % str(line)) + continue + #raise utils.Error('Match failed') + entry[match.group(1)] = match.group(2) + if len(entry) > 0: + result.append(entry) + return result def AndroidListAvd(): - """Returns a list of available Android Virtual Devices.""" - return ParseAndroidListAvdResult(utils.RunCommand(["android", "list", "avd"])) + """Returns a list of available Android Virtual Devices.""" + return ParseAndroidListAvdResult( + utils.RunCommand(["android", "list", "avd"])) def FindAvd(avds, key): - for avd in avds: - if avd['Name'] == key: - return avd - return None + for avd in avds: + if avd['Name'] == key: + return avd + return None def CreateAvd(avdName, abi): - out = utils.RunCommand(["android", "create", "avd", "--name", avdName, - "--target", TargetForAbi(abi), '--abi', abi], - input="no\n") - if out.find('Created AVD ') < 0: - if VERBOSE: - sys.stderr.write('Could not create AVD:\n%s\n' % out) - raise utils.Error('Could not create AVD') + out = utils.RunCommand([ + "android", "create", "avd", "--name", avdName, "--target", + TargetForAbi(abi), '--abi', abi + ], + input="no\n") + if out.find('Created AVD ') < 0: + if VERBOSE: + sys.stderr.write('Could not create AVD:\n%s\n' % out) + raise utils.Error('Could not create AVD') def AvdExists(avdName): - avdList = AndroidListAvd() - return FindAvd(avdList, avdName) is not None + avdList = AndroidListAvd() + return FindAvd(avdList, avdName) is not None def EnsureAvdExists(avdName, abi): - if AvdExists(avdName): - return - if VERBOSE: - sys.stderr.write('Checking SDK packages...\n') - if EnsureAndroidSdkPackagesInstalled(abi): - # Installing a new package could have made a previously invalid AVD valid if AvdExists(avdName): return - CreateAvd(avdName, abi) + if VERBOSE: + sys.stderr.write('Checking SDK packages...\n') + if EnsureAndroidSdkPackagesInstalled(abi): + # Installing a new package could have made a previously invalid AVD valid + if AvdExists(avdName): + return + CreateAvd(avdName, abi) def StartEmulator(abi, avdName, pollFn): - """ + """ Start an emulator for a given abi and svdName. Echo the emulator's stderr and stdout output to our stderr. @@ -238,135 +249,144 @@ def StartEmulator(abi, avdName, pollFn): three levels of nested python scripts.) Calling the ABI-specific versions of the emulator directly works around this bug. """ - emulatorName = {'x86' : 'emulator-x86', 'armeabi-v7a': 'emulator-arm'}[abi] - command = [emulatorName, '-avd', avdName, '-no-boot-anim', '-no-window'] - utils.RunCommand(command, pollFn=pollFn, killOnEarlyReturn=False, - outStream=sys.stderr, errStream=sys.stderr) + emulatorName = {'x86': 'emulator-x86', 'armeabi-v7a': 'emulator-arm'}[abi] + command = [emulatorName, '-avd', avdName, '-no-boot-anim', '-no-window'] + utils.RunCommand( + command, + pollFn=pollFn, + killOnEarlyReturn=False, + outStream=sys.stderr, + errStream=sys.stderr) def ParseAndroidDevices(text): - """Return Dictionary [name] -> status""" - text = text.split('List of devices attached')[-1] - lines = [line.strip() for line in text.split('\n')] - lines = [line for line in lines if len(line) > 0] - devices = {} - for line in lines: - lineItems = line.split('\t') - devices[lineItems[0]] = lineItems[1] - return devices + """Return Dictionary [name] -> status""" + text = text.split('List of devices attached')[-1] + lines = [line.strip() for line in text.split('\n')] + lines = [line for line in lines if len(line) > 0] + devices = {} + for line in lines: + lineItems = line.split('\t') + devices[lineItems[0]] = lineItems[1] + return devices def GetAndroidDevices(): - return ParseAndroidDevices(utils.RunCommand(["adb", "devices"])) + return ParseAndroidDevices(utils.RunCommand(["adb", "devices"])) def FilterOfflineDevices(devices): - online = {} - for device in devices.keys(): - status = devices[device] - if status != 'offline': - online[device] = status - return online + online = {} + for device in devices.keys(): + status = devices[device] + if status != 'offline': + online[device] = status + return online def GetOnlineAndroidDevices(): - return FilterOfflineDevices(GetAndroidDevices()) + return FilterOfflineDevices(GetAndroidDevices()) def GetAndroidDeviceProperty(device, property): - return utils.RunCommand( - ["adb", "-s", device, "shell", "getprop", property]).strip() + return utils.RunCommand(["adb", "-s", device, "shell", "getprop", + property]).strip() def GetAndroidDeviceAbis(device): - abis = [] - for property in ['ro.product.cpu.abi', 'ro.product.cpu.abi2']: - out = GetAndroidDeviceProperty(device, property) - if len(out) > 0: - abis.append(out) - return abis + abis = [] + for property in ['ro.product.cpu.abi', 'ro.product.cpu.abi2']: + out = GetAndroidDeviceProperty(device, property) + if len(out) > 0: + abis.append(out) + return abis def FindAndroidRunning(abi): - for device in GetOnlineAndroidDevices().keys(): - if abi in GetAndroidDeviceAbis(device): - return device - return None + for device in GetOnlineAndroidDevices().keys(): + if abi in GetAndroidDeviceAbis(device): + return device + return None def AddSdkToolsToPath(): - script_dir = os.path.dirname(sys.argv[0]) - dart_root = os.path.realpath(os.path.join(script_dir, '..', '..')) - third_party_root = os.path.join(dart_root, 'third_party') - android_tools = os.path.join(third_party_root, 'android_tools') - android_sdk_root = os.path.join(android_tools, 'sdk') - android_sdk_tools = os.path.join(android_sdk_root, 'tools') - android_sdk_platform_tools = os.path.join(android_sdk_root, 'platform-tools') - os.environ['PATH'] = ':'.join([ - os.environ['PATH'], android_sdk_tools, android_sdk_platform_tools]) - # Remove any environment variables that would affect our build. - for i in ['ANDROID_NDK_ROOT', 'ANDROID_SDK_ROOT', 'ANDROID_TOOLCHAIN', - 'AR', 'BUILDTYPE', 'CC', 'CXX', 'GYP_DEFINES', - 'LD_LIBRARY_PATH', 'LINK', 'MAKEFLAGS', 'MAKELEVEL', - 'MAKEOVERRIDES', 'MFLAGS', 'NM']: - if i in os.environ: - del os.environ[i] + script_dir = os.path.dirname(sys.argv[0]) + dart_root = os.path.realpath(os.path.join(script_dir, '..', '..')) + third_party_root = os.path.join(dart_root, 'third_party') + android_tools = os.path.join(third_party_root, 'android_tools') + android_sdk_root = os.path.join(android_tools, 'sdk') + android_sdk_tools = os.path.join(android_sdk_root, 'tools') + android_sdk_platform_tools = os.path.join(android_sdk_root, + 'platform-tools') + os.environ['PATH'] = ':'.join( + [os.environ['PATH'], android_sdk_tools, android_sdk_platform_tools]) + # Remove any environment variables that would affect our build. + for i in [ + 'ANDROID_NDK_ROOT', 'ANDROID_SDK_ROOT', 'ANDROID_TOOLCHAIN', 'AR', + 'BUILDTYPE', 'CC', 'CXX', 'GYP_DEFINES', 'LD_LIBRARY_PATH', 'LINK', + 'MAKEFLAGS', 'MAKELEVEL', 'MAKEOVERRIDES', 'MFLAGS', 'NM' + ]: + if i in os.environ: + del os.environ[i] def FindAndroid(abi, bootstrap): - if VERBOSE: - sys.stderr.write('Looking for an Android device running abi %s...\n' % abi) - AddSdkToolsToPath() - device = FindAndroidRunning(abi) - if not device: - if bootstrap: - if VERBOSE: - sys.stderr.write("No emulator found, try to create one.\n") - avdName = 'dart-build-%s' % abi - EnsureAvdExists(avdName, abi) + if VERBOSE: + sys.stderr.write( + 'Looking for an Android device running abi %s...\n' % abi) + AddSdkToolsToPath() + device = FindAndroidRunning(abi) + if not device: + if bootstrap: + if VERBOSE: + sys.stderr.write("No emulator found, try to create one.\n") + avdName = 'dart-build-%s' % abi + EnsureAvdExists(avdName, abi) - # It takes a while to start up an emulator. - # Provide feedback while we wait. - pollResult = [None] - def pollFunction(): - if VERBOSE: - sys.stderr.write('.') - pollResult[0] = FindAndroidRunning(abi) - # Stop polling once we have our result. - return pollResult[0] != None - StartEmulator(abi, avdName, pollFunction) - device = pollResult[0] - return device + # It takes a while to start up an emulator. + # Provide feedback while we wait. + pollResult = [None] + + def pollFunction(): + if VERBOSE: + sys.stderr.write('.') + pollResult[0] = FindAndroidRunning(abi) + # Stop polling once we have our result. + return pollResult[0] != None + + StartEmulator(abi, avdName, pollFunction) + device = pollResult[0] + return device def Main(): - # Parse options. - parser = BuildOptions() - (options, args) = parser.parse_args() - if not ProcessOptions(options): - parser.print_help() - return 1 + # Parse options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 - # If there are additional arguments, report error and exit. - if args: - parser.print_help() - return 1 + # If there are additional arguments, report error and exit. + if args: + parser.print_help() + return 1 - try: - device = FindAndroid(options.abi, options.bootstrap) - if device != None: - sys.stdout.write("%s\n" % device) - return 0 - else: - if VERBOSE: - sys.stderr.write('Could not find device\n') - return 2 - except utils.Error as e: - sys.stderr.write("error: %s\n" % e) - if DEBUG: - traceback.print_exc(file=sys.stderr) - return -1 + try: + device = FindAndroid(options.abi, options.bootstrap) + if device != None: + sys.stdout.write("%s\n" % device) + return 0 + else: + if VERBOSE: + sys.stderr.write('Could not find device\n') + return 2 + except utils.Error as e: + sys.stderr.write("error: %s\n" % e) + if DEBUG: + traceback.print_exc(file=sys.stderr) + return -1 if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/runtime/tools/benchmark.py b/runtime/tools/benchmark.py index b3a587fb58b..313a0d92f0a 100755 --- a/runtime/tools/benchmark.py +++ b/runtime/tools/benchmark.py @@ -15,116 +15,133 @@ import sys import utils import re - HOST_OS = utils.GuessOS() HOST_CPUS = utils.GuessCpus() + # Returns whether 'bench' matches any element in the 'filt' list. def match(bench, filt): - bench = bench.lower(); - for element in filt: - if element.search(bench): - return True - return False + bench = bench.lower() + for element in filt: + if element.search(bench): + return True + return False + def GetBenchmarkFile(path): - benchmark_root_path = [dirname(sys.argv[0]), '..', '..'] + ['benchmarks'] - return realpath(os.path.sep.join(benchmark_root_path + path)) + benchmark_root_path = [dirname(sys.argv[0]), '..', '..'] + ['benchmarks'] + return realpath(os.path.sep.join(benchmark_root_path + path)) + def ReadBenchmarkList(mode, path, core): - filename = GetBenchmarkFile([path]) - benchmarks = dict() - execfile(filename, benchmarks) - if (mode == "release") and not core: - return benchmarks['SUPPORTED_BENCHMARKS'] - else: - return benchmarks['SUPPORTED_CORE_BENCHMARKS'] + filename = GetBenchmarkFile([path]) + benchmarks = dict() + execfile(filename, benchmarks) + if (mode == "release") and not core: + return benchmarks['SUPPORTED_BENCHMARKS'] + else: + return benchmarks['SUPPORTED_CORE_BENCHMARKS'] + def BuildOptions(): - result = optparse.OptionParser() - result.add_option("-m", "--mode", - help='Build variants (comma-separated).', - metavar='[all,debug,release]', - default='release') - result.add_option("-v", "--verbose", - help='Verbose output.', - default=False, action="store_true") - result.add_option("-c", "--core", - help='Run only core benchmarks.', - default=False, action="store_true") - result.add_option("--arch", - help='Target architectures (comma-separated).', - metavar='[all,ia32,x64,simarm,arm,dartc]', - default=utils.GuessArchitecture()) - result.add_option("--executable", - help='Virtual machine to execute.', - metavar='[dart, (path to dart binary)]', - default=None) - result.add_option("-w", "--warmup", - help='Only run the warmup period.', - default=False, action="store_true") - return result + result = optparse.OptionParser() + result.add_option( + "-m", + "--mode", + help='Build variants (comma-separated).', + metavar='[all,debug,release]', + default='release') + result.add_option( + "-v", + "--verbose", + help='Verbose output.', + default=False, + action="store_true") + result.add_option( + "-c", + "--core", + help='Run only core benchmarks.', + default=False, + action="store_true") + result.add_option( + "--arch", + help='Target architectures (comma-separated).', + metavar='[all,ia32,x64,simarm,arm,dartc]', + default=utils.GuessArchitecture()) + result.add_option( + "--executable", + help='Virtual machine to execute.', + metavar='[dart, (path to dart binary)]', + default=None) + result.add_option( + "-w", + "--warmup", + help='Only run the warmup period.', + default=False, + action="store_true") + return result def ProcessOptions(options): - if options.arch == 'all': - options.arch = 'ia32,x64,simarm,dartc' - if options.mode == 'all': - options.mode = 'debug,release' - options.mode = options.mode.split(',') - options.arch = options.arch.split(',') - for mode in options.mode: - if not mode in ['debug', 'release']: - print "Unknown mode %s" % mode - return False - for arch in options.arch: - if not arch in ['ia32', 'x64', 'simarm', 'arm', 'dartc']: - print "Unknown arch %s" % arch - return False - return True + if options.arch == 'all': + options.arch = 'ia32,x64,simarm,dartc' + if options.mode == 'all': + options.mode = 'debug,release' + options.mode = options.mode.split(',') + options.arch = options.arch.split(',') + for mode in options.mode: + if not mode in ['debug', 'release']: + print "Unknown mode %s" % mode + return False + for arch in options.arch: + if not arch in ['ia32', 'x64', 'simarm', 'arm', 'dartc']: + print "Unknown arch %s" % arch + return False + return True def GetBuildRoot(mode, arch): - return utils.GetBuildRoot(HOST_OS, mode, arch) + return utils.GetBuildRoot(HOST_OS, mode, arch) + def GetDart(mode, arch): - executable = [abspath(join(GetBuildRoot(mode, arch), 'dart'))] - return executable + executable = [abspath(join(GetBuildRoot(mode, arch), 'dart'))] + return executable + def Main(): - # Parse the options. - parser = BuildOptions() - (options, args) = parser.parse_args() - if not ProcessOptions(options): - parser.print_help() - return 1 + # Parse the options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 - chosen_benchmarks = ReadBenchmarkList(options.mode, - 'BENCHMARKS', - options.core) + chosen_benchmarks = ReadBenchmarkList(options.mode, 'BENCHMARKS', + options.core) - # Use arguments to filter the benchmarks. - if len(args) > 0: - filt = [re.compile(x.lower()) for x in args] - chosen_benchmarks = [b for b in chosen_benchmarks if match(b[0], filt)] + # Use arguments to filter the benchmarks. + if len(args) > 0: + filt = [re.compile(x.lower()) for x in args] + chosen_benchmarks = [b for b in chosen_benchmarks if match(b[0], filt)] - for mode in options.mode: - for arch in options.arch: - if options.executable is None: - # Construct the path to the dart binary. - executable = GetDart(mode, arch) - else: - executable = [options.executable] - for benchmark, vmargs, progargs in chosen_benchmarks: - command = executable - command = command + [ - GetBenchmarkFile([benchmark, 'dart', benchmark + '.dart']), - ] - if options.verbose: - print ' '.join(command) - subprocess.call(command) - return 0 + for mode in options.mode: + for arch in options.arch: + if options.executable is None: + # Construct the path to the dart binary. + executable = GetDart(mode, arch) + else: + executable = [options.executable] + for benchmark, vmargs, progargs in chosen_benchmarks: + command = executable + command = command + [ + GetBenchmarkFile([benchmark, 'dart', benchmark + '.dart']), + ] + if options.verbose: + print ' '.join(command) + subprocess.call(command) + return 0 if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/runtime/tools/bin_to_assembly.py b/runtime/tools/bin_to_assembly.py index 8bc5d8c8ad0..fc539e394b9 100755 --- a/runtime/tools/bin_to_assembly.py +++ b/runtime/tools/bin_to_assembly.py @@ -11,125 +11,124 @@ import os import sys from optparse import OptionParser + def Main(): - parser = OptionParser() - parser.add_option("--output", - action="store", type="string", - help="output assembly file name") - parser.add_option("--input", - action="store", type="string", - help="input binary blob file") - parser.add_option("--symbol_name", - action="store", type="string") - parser.add_option("--executable", - action="store_true", default=False) - parser.add_option("--target_os", - action="store", type="string") - parser.add_option("--size_symbol_name", - action="store", type="string") - parser.add_option("--target_arch", - action="store", type="string") + parser = OptionParser() + parser.add_option( + "--output", + action="store", + type="string", + help="output assembly file name") + parser.add_option( + "--input", action="store", type="string", help="input binary blob file") + parser.add_option("--symbol_name", action="store", type="string") + parser.add_option("--executable", action="store_true", default=False) + parser.add_option("--target_os", action="store", type="string") + parser.add_option("--size_symbol_name", action="store", type="string") + parser.add_option("--target_arch", action="store", type="string") - (options, args) = parser.parse_args() - if not options.output: - sys.stderr.write("--output not specified\n") - parser.print_help(); - return -1 - if not options.input: - sys.stderr.write("--input not specified\n") - parser.print_help(); - return -1 - if not os.path.isfile(options.input): - sys.stderr.write("input file does not exist: %s\n" % options.input) - parser.print_help(); - return -1 - if not options.symbol_name: - sys.stderr.write("--symbol_name not specified\n") - parser.print_help(); - return -1 - if not options.target_os: - sys.stderr.write("--target_os not specified\n") - parser.print_help(); - return -1 - - with open(options.output, "w") as output_file: - if options.target_os in ["mac", "ios"]: - if options.executable: - output_file.write(".text\n") - else: - output_file.write(".const\n") - output_file.write(".global _%s\n" % options.symbol_name) - output_file.write(".balign 32\n") - output_file.write("_%s:\n" % options.symbol_name) - elif options.target_os in ["win"]: - output_file.write("ifndef _ML64_X64\n") - output_file.write(".model flat, C\n") - output_file.write("endif\n") - if options.executable: - output_file.write(".code\n") - else: - output_file.write(".const\n") - output_file.write("public %s\n" % options.symbol_name) - output_file.write("%s label byte\n" % options.symbol_name) - else: - if options.executable: - output_file.write(".text\n") - output_file.write(".type %s STT_FUNC\n" % options.symbol_name) - else: - output_file.write(".section .rodata\n") - output_file.write(".type %s STT_OBJECT\n" % options.symbol_name) - output_file.write(".global %s\n" % options.symbol_name) - output_file.write(".balign 32\n") - output_file.write("%s:\n" % options.symbol_name) - - size = 0 - with open(options.input, "rb") as input_file: - if options.target_os in ["win"]: - for byte in input_file.read(): - output_file.write("byte %d\n" % ord(byte)) - size += 1 - else: - for byte in input_file.read(): - output_file.write(".byte %d\n" % ord(byte)) - size += 1 - - if options.target_os not in ["mac", "ios", "win"]: - output_file.write(".size {0}, .-{0}\n".format(options.symbol_name)) - - if options.size_symbol_name: - if not options.target_arch: - sys.stderr.write("--target_arch not specified\n") - parser.print_help(); + (options, args) = parser.parse_args() + if not options.output: + sys.stderr.write("--output not specified\n") + parser.print_help() + return -1 + if not options.input: + sys.stderr.write("--input not specified\n") + parser.print_help() + return -1 + if not os.path.isfile(options.input): + sys.stderr.write("input file does not exist: %s\n" % options.input) + parser.print_help() + return -1 + if not options.symbol_name: + sys.stderr.write("--symbol_name not specified\n") + parser.print_help() + return -1 + if not options.target_os: + sys.stderr.write("--target_os not specified\n") + parser.print_help() return -1 - is64bit = 0 - if options.target_arch: - if options.target_arch in ["arm64", "x64"]: - is64bit = 1 - - if options.target_os in ["win"]: - output_file.write("public %s\n" % options.size_symbol_name) - output_file.write("%s label byte\n" % options.size_symbol_name) - if (is64bit == 1): - output_file.write("qword %d\n" % size ) - else: - output_file.write("dword %d\n" % size ) - else: + with open(options.output, "w") as output_file: if options.target_os in ["mac", "ios"]: - output_file.write(".global _%s\n" % options.size_symbol_name) - output_file.write("_%s:\n" % options.size_symbol_name) + if options.executable: + output_file.write(".text\n") + else: + output_file.write(".const\n") + output_file.write(".global _%s\n" % options.symbol_name) + output_file.write(".balign 32\n") + output_file.write("_%s:\n" % options.symbol_name) + elif options.target_os in ["win"]: + output_file.write("ifndef _ML64_X64\n") + output_file.write(".model flat, C\n") + output_file.write("endif\n") + if options.executable: + output_file.write(".code\n") + else: + output_file.write(".const\n") + output_file.write("public %s\n" % options.symbol_name) + output_file.write("%s label byte\n" % options.symbol_name) else: - output_file.write(".global %s\n" % options.size_symbol_name) - output_file.write("%s:\n" % options.size_symbol_name) - if (is64bit == 1): - output_file.write(".quad %d\n" % size ) - else: - output_file.write(".long %d\n" % size ) + if options.executable: + output_file.write(".text\n") + output_file.write(".type %s STT_FUNC\n" % options.symbol_name) + else: + output_file.write(".section .rodata\n") + output_file.write(".type %s STT_OBJECT\n" % options.symbol_name) + output_file.write(".global %s\n" % options.symbol_name) + output_file.write(".balign 32\n") + output_file.write("%s:\n" % options.symbol_name) - if options.target_os in ["win"]: - output_file.write("end\n") + size = 0 + with open(options.input, "rb") as input_file: + if options.target_os in ["win"]: + for byte in input_file.read(): + output_file.write("byte %d\n" % ord(byte)) + size += 1 + else: + for byte in input_file.read(): + output_file.write(".byte %d\n" % ord(byte)) + size += 1 + + if options.target_os not in ["mac", "ios", "win"]: + output_file.write(".size {0}, .-{0}\n".format(options.symbol_name)) + + if options.size_symbol_name: + if not options.target_arch: + sys.stderr.write("--target_arch not specified\n") + parser.print_help() + return -1 + + is64bit = 0 + if options.target_arch: + if options.target_arch in ["arm64", "x64"]: + is64bit = 1 + + if options.target_os in ["win"]: + output_file.write("public %s\n" % options.size_symbol_name) + output_file.write("%s label byte\n" % options.size_symbol_name) + if (is64bit == 1): + output_file.write("qword %d\n" % size) + else: + output_file.write("dword %d\n" % size) + else: + if options.target_os in ["mac", "ios"]: + output_file.write( + ".global _%s\n" % options.size_symbol_name) + output_file.write("_%s:\n" % options.size_symbol_name) + else: + output_file.write(".global %s\n" % options.size_symbol_name) + output_file.write("%s:\n" % options.size_symbol_name) + if (is64bit == 1): + output_file.write(".quad %d\n" % size) + else: + output_file.write(".long %d\n" % size) + + if options.target_os in ["win"]: + output_file.write("end\n") + + return 0 - return 0 if __name__ == "__main__": - sys.exit(Main()) + sys.exit(Main()) diff --git a/runtime/tools/bin_to_coff.py b/runtime/tools/bin_to_coff.py index 9eb0d3dfdf2..1925bcdb9b2 100644 --- a/runtime/tools/bin_to_coff.py +++ b/runtime/tools/bin_to_coff.py @@ -9,16 +9,16 @@ from ctypes import create_string_buffer from struct import * # FILE HEADER FLAGS -FILE_HEADER_RELFLG = 0x1 # No relocation information -FILE_HEADER_EXEC = 0x2 # Executable -FILE_HEADER_LNNO = 0x4 # No line number information -FILE_HEADER_LSYMS = 0x8 # Local symbols removed / not present +FILE_HEADER_RELFLG = 0x1 # No relocation information +FILE_HEADER_EXEC = 0x2 # Executable +FILE_HEADER_LNNO = 0x4 # No line number information +FILE_HEADER_LSYMS = 0x8 # Local symbols removed / not present FILE_HEADER_AR32WR = 0x100 # File is 32-bit little endian # SECTION HEADER FLAGS SECTION_HEADER_TEXT = 0x20 # Contains executable code SECTION_HEADER_DATA = 0x40 # Contains only initialized data -SECTION_HEADER_BSS = 0x80 # Contains uninitialized data +SECTION_HEADER_BSS = 0x80 # Contains uninitialized data # FILE HEADER FORMAT # typedef struct { @@ -30,14 +30,14 @@ SECTION_HEADER_BSS = 0x80 # Contains uninitialized data # unsigned short f_opthdr; /* sizeof(optional hdr) */ # unsigned short f_flags; /* flags */ # } FILHDR; -FILE_HEADER_FORMAT = 'HHIIIHH' -FILE_HEADER_SIZE = calcsize(FILE_HEADER_FORMAT) -FILE_HEADER_MAGIC_X64 = 0x8664 -FILE_HEADER_MAGIC_IA32 = 0x014c -FILE_HEADER_NUM_SECTIONS = 1 -FILE_HEADER_TIMESTAMP = 0 +FILE_HEADER_FORMAT = 'HHIIIHH' +FILE_HEADER_SIZE = calcsize(FILE_HEADER_FORMAT) +FILE_HEADER_MAGIC_X64 = 0x8664 +FILE_HEADER_MAGIC_IA32 = 0x014c +FILE_HEADER_NUM_SECTIONS = 1 +FILE_HEADER_TIMESTAMP = 0 FILE_HEADER_SIZE_OF_OPTIONAL = 0 -FILE_HEADER_FLAGS = FILE_HEADER_LNNO +FILE_HEADER_FLAGS = FILE_HEADER_LNNO # SECTION HEADER FORMAT # typedef struct { @@ -52,18 +52,18 @@ FILE_HEADER_FLAGS = FILE_HEADER_LNNO # unsigned short s_nlnno; /* number of line number entries */ # unsigned long s_flags; /* flags */ # } SCNHDR; -SECTION_HEADER_FORMAT = '8sIIIIIIHHI' -SECTION_HEADER_SIZE = calcsize(SECTION_HEADER_FORMAT) -SECTION_NAME_RODATA = '.rodata' -SECTION_NAME_TEXT = '.text' -SECTION_PADDR = 0x0 -SECTION_VADDR = 0x0 -SECTION_RAW_DATA_PTR = (FILE_HEADER_SIZE + FILE_HEADER_NUM_SECTIONS - * SECTION_HEADER_SIZE) +SECTION_HEADER_FORMAT = '8sIIIIIIHHI' +SECTION_HEADER_SIZE = calcsize(SECTION_HEADER_FORMAT) +SECTION_NAME_RODATA = '.rodata' +SECTION_NAME_TEXT = '.text' +SECTION_PADDR = 0x0 +SECTION_VADDR = 0x0 +SECTION_RAW_DATA_PTR = ( + FILE_HEADER_SIZE + FILE_HEADER_NUM_SECTIONS * SECTION_HEADER_SIZE) SECTION_RELOCATION_PTR = 0x0 -SECTION_LINE_NUMS_PTR = 0x0 +SECTION_LINE_NUMS_PTR = 0x0 SECTION_NUM_RELOCATION = 0 -SECTION_NUM_LINE_NUMS = 0 +SECTION_NUM_LINE_NUMS = 0 # SYMBOL TABLE FORMAT # typedef struct { @@ -80,165 +80,176 @@ SECTION_NUM_LINE_NUMS = 0 # unsigned char e_sclass; # unsigned char e_numaux; # } SYMENT; -SYMBOL_TABLE_ENTRY_SHORT_LEN = 8 +SYMBOL_TABLE_ENTRY_SHORT_LEN = 8 SYMBOL_TABLE_ENTRY_FORMAT_SHORT = '8sIhHBB' -SYMBOL_TABLE_ENTRY_FORMAT_LONG = 'IIIhHBB' -SYMBOL_TABLE_ENTRY_SIZE = calcsize(SYMBOL_TABLE_ENTRY_FORMAT_SHORT) -SYMBOL_TABLE_ENTRY_ZEROS = 0x0 -SYMBOL_TABLE_ENTRY_SECTION = 1 -SYMBOL_TABLE_ENTRY_TYPE = 0 -SYMBOL_TABLE_ENTRY_CLASS = 2 # External (public) symbol. -SYMBOL_TABLE_ENTRY_NUM_AUX = 0 # Number of auxiliary entries. +SYMBOL_TABLE_ENTRY_FORMAT_LONG = 'IIIhHBB' +SYMBOL_TABLE_ENTRY_SIZE = calcsize(SYMBOL_TABLE_ENTRY_FORMAT_SHORT) +SYMBOL_TABLE_ENTRY_ZEROS = 0x0 +SYMBOL_TABLE_ENTRY_SECTION = 1 +SYMBOL_TABLE_ENTRY_TYPE = 0 +SYMBOL_TABLE_ENTRY_CLASS = 2 # External (public) symbol. +SYMBOL_TABLE_ENTRY_NUM_AUX = 0 # Number of auxiliary entries. - -STRING_TABLE_OFFSET = 0x4 # Starting offset for the string table. -SIZE_FORMAT = 'I' -SIZE_LENGTH = calcsize(SIZE_FORMAT) +STRING_TABLE_OFFSET = 0x4 # Starting offset for the string table. +SIZE_FORMAT = 'I' +SIZE_LENGTH = calcsize(SIZE_FORMAT) SIZE_SYMBOL_FORMAT_X64 = 'Q' SIZE_SYMBOL_LENGTH_X64 = calcsize(SIZE_SYMBOL_FORMAT_X64) + def main(): - parser = argparse.ArgumentParser(description='Generate a COFF file for binary data.') - parser.add_argument('--input', dest='input', help='Path of the input file.') - parser.add_argument('--output', dest='output', help='Name of the output file.') - parser.add_argument('--symbol_name', dest='symbol_name', help='Name of the symbol for the binary data') - parser.add_argument('--size_symbol_name', dest='size_name', help='Name of the symbol for the size of the binary data') - parser.add_argument('--64-bit', dest='use_64_bit', action='store_true', default=False) - parser.add_argument('--executable', dest='executable', action='store_true', default=False) + parser = argparse.ArgumentParser( + description='Generate a COFF file for binary data.') + parser.add_argument('--input', dest='input', help='Path of the input file.') + parser.add_argument( + '--output', dest='output', help='Name of the output file.') + parser.add_argument( + '--symbol_name', + dest='symbol_name', + help='Name of the symbol for the binary data') + parser.add_argument( + '--size_symbol_name', + dest='size_name', + help='Name of the symbol for the size of the binary data') + parser.add_argument( + '--64-bit', dest='use_64_bit', action='store_true', default=False) + parser.add_argument( + '--executable', dest='executable', action='store_true', default=False) - args = parser.parse_args() + args = parser.parse_args() - with open(args.input, 'rb') as f: - section_data = f.read() + with open(args.input, 'rb') as f: + section_data = f.read() - # We need to calculate the following to determine the size of our buffer: - # 1) Size of the data - # 2) Total length of the symbol strings which are over 8 characters + # We need to calculate the following to determine the size of our buffer: + # 1) Size of the data + # 2) Total length of the symbol strings which are over 8 characters - section_size = len(section_data) - includes_size_name = (args.size_name != None) + section_size = len(section_data) + includes_size_name = (args.size_name != None) - # Symbols on x86 are prefixed with '_' - symbol_prefix = '' if args.use_64_bit else '_' - num_symbols = 2 if includes_size_name else 1 - symbol_name = symbol_prefix + args.symbol_name - size_symbol_name = None - if (includes_size_name): - size_symbol = args.size_name if args.size_name else args.symbol_name + "Size" - size_symbol_name = symbol_prefix + size_symbol + # Symbols on x86 are prefixed with '_' + symbol_prefix = '' if args.use_64_bit else '_' + num_symbols = 2 if includes_size_name else 1 + symbol_name = symbol_prefix + args.symbol_name + size_symbol_name = None + if (includes_size_name): + size_symbol = args.size_name if args.size_name else args.symbol_name + "Size" + size_symbol_name = symbol_prefix + size_symbol - size_symbol_format = SIZE_SYMBOL_FORMAT_X64 if args.use_64_bit else SIZE_FORMAT - size_symbol_size = SIZE_SYMBOL_LENGTH_X64 if args.use_64_bit else SIZE_LENGTH + size_symbol_format = SIZE_SYMBOL_FORMAT_X64 if args.use_64_bit else SIZE_FORMAT + size_symbol_size = SIZE_SYMBOL_LENGTH_X64 if args.use_64_bit else SIZE_LENGTH - # The symbol table is directly after the data section - symbol_table_ptr = (FILE_HEADER_SIZE + SECTION_HEADER_SIZE + section_size + size_symbol_size) - string_table_len = 0 + # The symbol table is directly after the data section + symbol_table_ptr = (FILE_HEADER_SIZE + SECTION_HEADER_SIZE + section_size + + size_symbol_size) + string_table_len = 0 - # Symbols longer than 8 characters have their string representations stored - # in the string table. - long_symbol_name = False - long_size_symbol_name = False - if (len(symbol_name) > SYMBOL_TABLE_ENTRY_SHORT_LEN): - string_table_len += len(symbol_name) + 1 - long_symbol_name = True + # Symbols longer than 8 characters have their string representations stored + # in the string table. + long_symbol_name = False + long_size_symbol_name = False + if (len(symbol_name) > SYMBOL_TABLE_ENTRY_SHORT_LEN): + string_table_len += len(symbol_name) + 1 + long_symbol_name = True - if (includes_size_name and (len(size_symbol_name) > SYMBOL_TABLE_ENTRY_SHORT_LEN)): - string_table_len += len(size_symbol_name) + 1 - long_size_symbol_name = True + if (includes_size_name and + (len(size_symbol_name) > SYMBOL_TABLE_ENTRY_SHORT_LEN)): + string_table_len += len(size_symbol_name) + 1 + long_size_symbol_name = True - # Create the buffer and start building. - offset = 0 - buff = create_string_buffer(FILE_HEADER_SIZE + SECTION_HEADER_SIZE + - section_size + num_symbols * - SYMBOL_TABLE_ENTRY_SIZE + SIZE_LENGTH + size_symbol_size + - string_table_len) + # Create the buffer and start building. + offset = 0 + buff = create_string_buffer( + FILE_HEADER_SIZE + SECTION_HEADER_SIZE + section_size + + num_symbols * SYMBOL_TABLE_ENTRY_SIZE + SIZE_LENGTH + size_symbol_size + + string_table_len) - FILE_HEADER_MAGIC = FILE_HEADER_MAGIC_X64 if args.use_64_bit else FILE_HEADER_MAGIC_IA32 + FILE_HEADER_MAGIC = FILE_HEADER_MAGIC_X64 if args.use_64_bit else FILE_HEADER_MAGIC_IA32 - # Populate the file header. Basically constant except for the pointer to the - # beginning of the symbol table. - pack_into(FILE_HEADER_FORMAT, buff, offset, - FILE_HEADER_MAGIC, FILE_HEADER_NUM_SECTIONS, - FILE_HEADER_TIMESTAMP, symbol_table_ptr, - num_symbols, FILE_HEADER_SIZE_OF_OPTIONAL, - FILE_HEADER_FLAGS) - offset += FILE_HEADER_SIZE + # Populate the file header. Basically constant except for the pointer to the + # beginning of the symbol table. + pack_into(FILE_HEADER_FORMAT, buff, offset, FILE_HEADER_MAGIC, + FILE_HEADER_NUM_SECTIONS, FILE_HEADER_TIMESTAMP, symbol_table_ptr, + num_symbols, FILE_HEADER_SIZE_OF_OPTIONAL, FILE_HEADER_FLAGS) + offset += FILE_HEADER_SIZE - section_name = SECTION_NAME_RODATA - section_type = SECTION_HEADER_DATA - if args.executable: - section_name = SECTION_NAME_TEXT - section_type = SECTION_HEADER_TEXT + section_name = SECTION_NAME_RODATA + section_type = SECTION_HEADER_DATA + if args.executable: + section_name = SECTION_NAME_TEXT + section_type = SECTION_HEADER_TEXT - # Populate the section header for a single section. - pack_into(SECTION_HEADER_FORMAT, buff, offset, - section_name, SECTION_PADDR, SECTION_VADDR, - section_size + size_symbol_size, SECTION_RAW_DATA_PTR, SECTION_RELOCATION_PTR, - SECTION_LINE_NUMS_PTR, SECTION_NUM_RELOCATION, - SECTION_NUM_LINE_NUMS, section_type) - offset += SECTION_HEADER_SIZE + # Populate the section header for a single section. + pack_into(SECTION_HEADER_FORMAT, buff, offset, section_name, SECTION_PADDR, + SECTION_VADDR, section_size + size_symbol_size, + SECTION_RAW_DATA_PTR, SECTION_RELOCATION_PTR, + SECTION_LINE_NUMS_PTR, SECTION_NUM_RELOCATION, + SECTION_NUM_LINE_NUMS, section_type) + offset += SECTION_HEADER_SIZE - # Copy the binary data. - buff[offset:offset + section_size] = section_data - offset += section_size + # Copy the binary data. + buff[offset:offset + section_size] = section_data + offset += section_size - # Append the size of the section. - pack_into(size_symbol_format, buff, offset, section_size) - offset += size_symbol_size + # Append the size of the section. + pack_into(size_symbol_format, buff, offset, section_size) + offset += size_symbol_size - # Build the symbol table. If a symbol name is 8 characters or less, it's - # placed directly in the symbol table. If not, it's entered in the string - # table immediately after the symbol table. + # Build the symbol table. If a symbol name is 8 characters or less, it's + # placed directly in the symbol table. If not, it's entered in the string + # table immediately after the symbol table. - string_table_offset = STRING_TABLE_OFFSET - if long_symbol_name: - pack_into(SYMBOL_TABLE_ENTRY_FORMAT_LONG, buff, offset, - SYMBOL_TABLE_ENTRY_ZEROS, string_table_offset, 0x0, - SYMBOL_TABLE_ENTRY_SECTION, SYMBOL_TABLE_ENTRY_TYPE, - SYMBOL_TABLE_ENTRY_CLASS, SYMBOL_TABLE_ENTRY_NUM_AUX) - string_table_offset += len(symbol_name) + 1 - else: - pack_into(SYMBOL_TABLE_ENTRY_FORMAT_SHORT, buff, offset, - symbol_name, 0x0, - SYMBOL_TABLE_ENTRY_SECTION, SYMBOL_TABLE_ENTRY_TYPE, - SYMBOL_TABLE_ENTRY_CLASS, SYMBOL_TABLE_ENTRY_NUM_AUX) - offset += SYMBOL_TABLE_ENTRY_SIZE - - if includes_size_name: - # The size symbol table entry actually contains the value for the size. - if long_size_symbol_name: - pack_into(SYMBOL_TABLE_ENTRY_FORMAT_LONG, buff, offset, - SYMBOL_TABLE_ENTRY_ZEROS, string_table_offset, section_size, - SYMBOL_TABLE_ENTRY_SECTION, SYMBOL_TABLE_ENTRY_TYPE, - SYMBOL_TABLE_ENTRY_CLASS, SYMBOL_TABLE_ENTRY_NUM_AUX) + string_table_offset = STRING_TABLE_OFFSET + if long_symbol_name: + pack_into(SYMBOL_TABLE_ENTRY_FORMAT_LONG, buff, offset, + SYMBOL_TABLE_ENTRY_ZEROS, string_table_offset, 0x0, + SYMBOL_TABLE_ENTRY_SECTION, SYMBOL_TABLE_ENTRY_TYPE, + SYMBOL_TABLE_ENTRY_CLASS, SYMBOL_TABLE_ENTRY_NUM_AUX) + string_table_offset += len(symbol_name) + 1 else: - pack_into(SYMBOL_TABLE_ENTRY_FORMAT_SHORT, buff, offset, - symbol_name, section_size, - SYMBOL_TABLE_ENTRY_SECTION, SYMBOL_TABLE_ENTRY_TYPE, - SYMBOL_TABLE_ENTRY_CLASS, SYMBOL_TABLE_ENTRY_NUM_AUX) + pack_into(SYMBOL_TABLE_ENTRY_FORMAT_SHORT, buff, offset, symbol_name, + 0x0, SYMBOL_TABLE_ENTRY_SECTION, SYMBOL_TABLE_ENTRY_TYPE, + SYMBOL_TABLE_ENTRY_CLASS, SYMBOL_TABLE_ENTRY_NUM_AUX) offset += SYMBOL_TABLE_ENTRY_SIZE - pack_into(SIZE_FORMAT, buff, offset, string_table_len + SIZE_LENGTH) - offset += SIZE_LENGTH + if includes_size_name: + # The size symbol table entry actually contains the value for the size. + if long_size_symbol_name: + pack_into(SYMBOL_TABLE_ENTRY_FORMAT_LONG, buff, offset, + SYMBOL_TABLE_ENTRY_ZEROS, string_table_offset, + section_size, SYMBOL_TABLE_ENTRY_SECTION, + SYMBOL_TABLE_ENTRY_TYPE, SYMBOL_TABLE_ENTRY_CLASS, + SYMBOL_TABLE_ENTRY_NUM_AUX) + else: + pack_into(SYMBOL_TABLE_ENTRY_FORMAT_SHORT, buff, offset, + symbol_name, section_size, SYMBOL_TABLE_ENTRY_SECTION, + SYMBOL_TABLE_ENTRY_TYPE, SYMBOL_TABLE_ENTRY_CLASS, + SYMBOL_TABLE_ENTRY_NUM_AUX) + offset += SYMBOL_TABLE_ENTRY_SIZE - # Populate the string table for any symbols longer than 8 characters. - if long_symbol_name: - symbol_len = len(symbol_name) - buff[offset:offset + symbol_len] = symbol_name - offset += symbol_len - buff[offset] = '\0' - offset += 1 + pack_into(SIZE_FORMAT, buff, offset, string_table_len + SIZE_LENGTH) + offset += SIZE_LENGTH - if includes_size_name and long_size_symbol_name: - symbol_len = len(size_symbol_name) - buff[offset:offset + symbol_len] = size_symbol_name - offset += symbol_len - buff[offset] = '\0' - offset += 1 + # Populate the string table for any symbols longer than 8 characters. + if long_symbol_name: + symbol_len = len(symbol_name) + buff[offset:offset + symbol_len] = symbol_name + offset += symbol_len + buff[offset] = '\0' + offset += 1 + + if includes_size_name and long_size_symbol_name: + symbol_len = len(size_symbol_name) + buff[offset:offset + symbol_len] = size_symbol_name + offset += symbol_len + buff[offset] = '\0' + offset += 1 + + with open(args.output, 'wb') as f: + f.write(buff.raw) - with open(args.output, 'wb') as f: - f.write(buff.raw) if __name__ == '__main__': - main() + main() diff --git a/runtime/tools/create_archive.py b/runtime/tools/create_archive.py index aa27ca4f8b1..c0b7c896fda 100755 --- a/runtime/tools/create_archive.py +++ b/runtime/tools/create_archive.py @@ -16,169 +16,176 @@ import tarfile import tempfile import gzip + def CreateTarArchive(tar_path, client_root, compress, files): - mode_string = 'w' - tar = tarfile.open(tar_path, mode=mode_string) - for input_file_name in files: - # Chop off client_root. - archive_file_name = input_file_name[ len(client_root) : ] - # Replace back slash with forward slash. So we do not have Windows paths. - archive_file_name = archive_file_name.replace("\\", "/") - # Open input file and add it to the archive. - with open(input_file_name, 'rb') as input_file: - tarInfo = tarfile.TarInfo(name=archive_file_name) - input_file.seek(0,2) - tarInfo.size = input_file.tell() - tarInfo.mtime = 0 # For deterministic builds. - input_file.seek(0) - tar.addfile(tarInfo, fileobj=input_file) - tar.close() - if compress: - with open(tar_path, "rb") as fin: - uncompressed = fin.read() - with open(tar_path, "wb") as fout: - # mtime=0 for deterministic builds. - gz = gzip.GzipFile(fileobj=fout, mode="wb", filename="", mtime=0) - gz.write(uncompressed) - gz.close() + mode_string = 'w' + tar = tarfile.open(tar_path, mode=mode_string) + for input_file_name in files: + # Chop off client_root. + archive_file_name = input_file_name[len(client_root):] + # Replace back slash with forward slash. So we do not have Windows paths. + archive_file_name = archive_file_name.replace("\\", "/") + # Open input file and add it to the archive. + with open(input_file_name, 'rb') as input_file: + tarInfo = tarfile.TarInfo(name=archive_file_name) + input_file.seek(0, 2) + tarInfo.size = input_file.tell() + tarInfo.mtime = 0 # For deterministic builds. + input_file.seek(0) + tar.addfile(tarInfo, fileobj=input_file) + tar.close() + if compress: + with open(tar_path, "rb") as fin: + uncompressed = fin.read() + with open(tar_path, "wb") as fout: + # mtime=0 for deterministic builds. + gz = gzip.GzipFile(fileobj=fout, mode="wb", filename="", mtime=0) + gz.write(uncompressed) + gz.close() def MakeArchive(options): - if not options.client_root: - sys.stderr.write('--client_root not specified') - return -1 + if not options.client_root: + sys.stderr.write('--client_root not specified') + return -1 - files = [ ] - for dirname, dirnames, filenames in os.walk(options.client_root): - # strip out all dot files. - filenames = [f for f in filenames if not f[0] == '.'] - dirnames[:] = [d for d in dirnames if not d[0] == '.'] - for f in filenames: - src_path = os.path.join(dirname, f) - if (os.path.isdir(src_path)): - continue - files.append(src_path) + files = [] + for dirname, dirnames, filenames in os.walk(options.client_root): + # strip out all dot files. + filenames = [f for f in filenames if not f[0] == '.'] + dirnames[:] = [d for d in dirnames if not d[0] == '.'] + for f in filenames: + src_path = os.path.join(dirname, f) + if (os.path.isdir(src_path)): + continue + files.append(src_path) - # Ensure consistent file ordering for reproducible builds. - files.sort() + # Ensure consistent file ordering for reproducible builds. + files.sort() - # Write out archive. - CreateTarArchive(options.tar_output, - options.client_root, - options.compress, - files) - return 0 + # Write out archive. + CreateTarArchive(options.tar_output, options.client_root, options.compress, + files) + return 0 -def WriteCCFile(output_file, - outer_namespace, - inner_namespace, - name, - tar_archive, - ): - with open(output_file, 'w') as out: - out.write(''' +def WriteCCFile( + output_file, + outer_namespace, + inner_namespace, + name, + tar_archive, +): + with open(output_file, 'w') as out: + out.write(''' // Copyright (c) %d, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. ''' % date.today().year) - out.write(''' + out.write(''' #include ''') - out.write('namespace %s {\n' % outer_namespace) - if inner_namespace != None: - out.write('namespace %s {\n' % inner_namespace) - out.write('\n\n') - # Write the byte contents of the archive as a comma separated list of - # integers, one integer for each byte. - out.write('static const uint8_t %s_[] = {\n' % name) - line = ' ' - lineCounter = 0 - for byte in tar_archive: - line += r" %d," % ord(byte) - lineCounter += 1 - if lineCounter == 10: - out.write(line + '\n') + out.write('namespace %s {\n' % outer_namespace) + if inner_namespace != None: + out.write('namespace %s {\n' % inner_namespace) + out.write('\n\n') + # Write the byte contents of the archive as a comma separated list of + # integers, one integer for each byte. + out.write('static const uint8_t %s_[] = {\n' % name) line = ' ' lineCounter = 0 - if lineCounter != 0: - out.write(line + '\n') - out.write('};\n') - out.write('\nunsigned int %s_len = %d;\n' % (name, len(tar_archive))) - out.write('\nconst uint8_t* %s = %s_;\n\n' % (name, name)) - if inner_namespace != None: - out.write('} // namespace %s\n' % inner_namespace) - out.write('} // namespace %s\n' % outer_namespace) + for byte in tar_archive: + line += r" %d," % ord(byte) + lineCounter += 1 + if lineCounter == 10: + out.write(line + '\n') + line = ' ' + lineCounter = 0 + if lineCounter != 0: + out.write(line + '\n') + out.write('};\n') + out.write('\nunsigned int %s_len = %d;\n' % (name, len(tar_archive))) + out.write('\nconst uint8_t* %s = %s_;\n\n' % (name, name)) + if inner_namespace != None: + out.write('} // namespace %s\n' % inner_namespace) + out.write('} // namespace %s\n' % outer_namespace) + def MakeCCFile(options): - if not options.output: - sys.stderr.write('--output not specified\n') - return -1 - if not options.name: - sys.stderr.write('--name not specified\n') - return -1 - if not options.tar_input: - sys.stderr.write('--tar_input not specified\n') - return -1 + if not options.output: + sys.stderr.write('--output not specified\n') + return -1 + if not options.name: + sys.stderr.write('--name not specified\n') + return -1 + if not options.tar_input: + sys.stderr.write('--tar_input not specified\n') + return -1 - # Read it back in. - with open(options.tar_input, 'rb') as tar_file: - tar_archive = tar_file.read() + # Read it back in. + with open(options.tar_input, 'rb') as tar_file: + tar_archive = tar_file.read() - # Write CC file. - WriteCCFile(options.output, - options.outer_namespace, - options.inner_namespace, - options.name, - tar_archive) - return 0 + # Write CC file. + WriteCCFile(options.output, options.outer_namespace, + options.inner_namespace, options.name, tar_archive) + return 0 def Main(args): - try: - # Parse input. - parser = OptionParser() - parser.add_option("--output", - action="store", type="string", - help="output file name") - parser.add_option("--tar_input",\ - action="store", type="string", - help="input tar archive") - parser.add_option("--tar_output", - action="store", type="string", - help="tar output file name") - parser.add_option("--outer_namespace", - action="store", type="string", - help="outer C++ namespace", - default="dart") - parser.add_option("--inner_namespace", - action="store", type="string", - help="inner C++ namespace", - default="bin") - parser.add_option("--name", - action="store", type="string", - help="name of tar archive symbol") - parser.add_option("--compress", action="store_true", default=False) - parser.add_option("--client_root", - action="store", type="string", - help="root directory client resources") + try: + # Parse input. + parser = OptionParser() + parser.add_option( + "--output", action="store", type="string", help="output file name") + parser.add_option("--tar_input",\ + action="store", type="string", + help="input tar archive") + parser.add_option( + "--tar_output", + action="store", + type="string", + help="tar output file name") + parser.add_option( + "--outer_namespace", + action="store", + type="string", + help="outer C++ namespace", + default="dart") + parser.add_option( + "--inner_namespace", + action="store", + type="string", + help="inner C++ namespace", + default="bin") + parser.add_option( + "--name", + action="store", + type="string", + help="name of tar archive symbol") + parser.add_option("--compress", action="store_true", default=False) + parser.add_option( + "--client_root", + action="store", + type="string", + help="root directory client resources") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - if options.tar_output: - return MakeArchive(options) - else: - return MakeCCFile(options) + if options.tar_output: + return MakeArchive(options) + else: + return MakeCCFile(options) - except Exception, inst: - sys.stderr.write('create_archive.py exception\n') - sys.stderr.write(str(inst)) - sys.stderr.write('\n') - return -1 + except Exception, inst: + sys.stderr.write('create_archive.py exception\n') + sys.stderr.write(str(inst)) + sys.stderr.write('\n') + return -1 if __name__ == '__main__': - sys.exit(Main(sys.argv)) + sys.exit(Main(sys.argv)) diff --git a/runtime/tools/create_snapshot_bin.py b/runtime/tools/create_snapshot_bin.py index 88da22ed4e3..8a222b8c657 100755 --- a/runtime/tools/create_snapshot_bin.py +++ b/runtime/tools/create_snapshot_bin.py @@ -3,7 +3,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """Script to create snapshot bin file.""" import getopt @@ -15,146 +14,185 @@ import utils def BuildOptions(): - result = optparse.OptionParser() - result.add_option("--executable", - action="store", type="string", - help="path to snapshot generator executable") - result.add_option("--snapshot_kind", - action="store", type="string", - help="kind of snapshot to generate", - default="core") - result.add_option("--load_compilation_trace", - action="store", type="string", - help="path to a compilation trace to load before generating a core-jit snapshot") - result.add_option("--vm_flag", - action="append", type="string", default=[], - help="pass additional Dart VM flag") - result.add_option("--vm_output_bin", - action="store", type="string", - help="output file name into which vm isolate snapshot in binary form " + - "is generated") - result.add_option("--vm_instructions_output_bin", - action="store", type="string", - help="output file name into which vm isolate snapshot in binary form " + - "is generated") - result.add_option("--isolate_output_bin", - action="store", type="string", - help="output file name into which isolate snapshot in binary form " + - "is generated") - result.add_option("--isolate_instructions_output_bin", - action="store", type="string", - help="output file name into which isolate snapshot in binary form " + - "is generated") - result.add_option("--script", - action="store", type="string", - help="Dart script for which snapshot is to be generated") - result.add_option("--package_root", - action="store", type="string", - help="path used to resolve package: imports.") - result.add_option("--packages", - action="store", type="string", - help="package config file used to reasolve package: imports.") - result.add_option("-v", "--verbose", - help='Verbose output.', - default=False, action="store_true") - result.add_option("--timestamp_file", - action="store", type="string", - help="Path to timestamp file that will be written", - default="") - return result + result = optparse.OptionParser() + result.add_option( + "--executable", + action="store", + type="string", + help="path to snapshot generator executable") + result.add_option( + "--snapshot_kind", + action="store", + type="string", + help="kind of snapshot to generate", + default="core") + result.add_option( + "--load_compilation_trace", + action="store", + type="string", + help= + "path to a compilation trace to load before generating a core-jit snapshot" + ) + result.add_option( + "--vm_flag", + action="append", + type="string", + default=[], + help="pass additional Dart VM flag") + result.add_option( + "--vm_output_bin", + action="store", + type="string", + help="output file name into which vm isolate snapshot in binary form " + + "is generated") + result.add_option( + "--vm_instructions_output_bin", + action="store", + type="string", + help="output file name into which vm isolate snapshot in binary form " + + "is generated") + result.add_option( + "--isolate_output_bin", + action="store", + type="string", + help="output file name into which isolate snapshot in binary form " + + "is generated") + result.add_option( + "--isolate_instructions_output_bin", + action="store", + type="string", + help="output file name into which isolate snapshot in binary form " + + "is generated") + result.add_option( + "--script", + action="store", + type="string", + help="Dart script for which snapshot is to be generated") + result.add_option( + "--package_root", + action="store", + type="string", + help="path used to resolve package: imports.") + result.add_option( + "--packages", + action="store", + type="string", + help="package config file used to reasolve package: imports.") + result.add_option( + "-v", + "--verbose", + help='Verbose output.', + default=False, + action="store_true") + result.add_option( + "--timestamp_file", + action="store", + type="string", + help="Path to timestamp file that will be written", + default="") + return result def ProcessOptions(options): - if not options.executable: - sys.stderr.write('--executable not specified\n') - return False - if not options.snapshot_kind: - sys.stderr.write('--snapshot_kind not specified\n') - return False - if not options.vm_output_bin: - sys.stderr.write('--vm_output_bin not specified\n') - return False - if not options.isolate_output_bin: - sys.stderr.write('--isolate_output_bin not specified\n') - return False - if (options.snapshot_kind == 'core-jit' - and not options.vm_instructions_output_bin): - sys.stderr.write('--vm_instructions_output_bin not specified\n') - return False - if (options.snapshot_kind == 'core-jit' - and not options.isolate_instructions_output_bin): - sys.stderr.write('--isolate_instructions_output_bin not specified\n') - return False - return True + if not options.executable: + sys.stderr.write('--executable not specified\n') + return False + if not options.snapshot_kind: + sys.stderr.write('--snapshot_kind not specified\n') + return False + if not options.vm_output_bin: + sys.stderr.write('--vm_output_bin not specified\n') + return False + if not options.isolate_output_bin: + sys.stderr.write('--isolate_output_bin not specified\n') + return False + if (options.snapshot_kind == 'core-jit' and + not options.vm_instructions_output_bin): + sys.stderr.write('--vm_instructions_output_bin not specified\n') + return False + if (options.snapshot_kind == 'core-jit' and + not options.isolate_instructions_output_bin): + sys.stderr.write('--isolate_instructions_output_bin not specified\n') + return False + return True def CreateTimestampFile(options): - if options.timestamp_file != '': - dir_name = os.path.dirname(options.timestamp_file) - if not os.path.exists(dir_name): - os.mkdir(dir_name) - open(options.timestamp_file, 'w').close() + if options.timestamp_file != '': + dir_name = os.path.dirname(options.timestamp_file) + if not os.path.exists(dir_name): + os.mkdir(dir_name) + open(options.timestamp_file, 'w').close() def Main(): - # Parse options. - parser = BuildOptions() - (options, args) = parser.parse_args() - if not ProcessOptions(options): - parser.print_help() - return 1 + # Parse options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 - # If there are additional arguments, report error and exit. - if args: - parser.print_help() - return 1 + # If there are additional arguments, report error and exit. + if args: + parser.print_help() + return 1 - # Setup arguments to the snapshot generator binary. - script_args = ["--ignore_unrecognized_flags"] + # Setup arguments to the snapshot generator binary. + script_args = ["--ignore_unrecognized_flags"] - for flag in options.vm_flag: - script_args.append(flag) + for flag in options.vm_flag: + script_args.append(flag) - if options.load_compilation_trace: - script_args.append(''.join([ "--load_compilation_trace=", options.load_compilation_trace])) + if options.load_compilation_trace: + script_args.append(''.join( + ["--load_compilation_trace=", options.load_compilation_trace])) - # Pass along the package_root if there is one. - if options.package_root: - script_args.append(''.join([ "--package_root=", options.package_root])) + # Pass along the package_root if there is one. + if options.package_root: + script_args.append(''.join(["--package_root=", options.package_root])) - # Pass along the packages if there is one. - if options.packages: - script_args.append(''.join([ "--packages=", options.packages])) + # Pass along the packages if there is one. + if options.packages: + script_args.append(''.join(["--packages=", options.packages])) - # First setup the vm isolate and regular isolate snapshot output filename. - script_args.append(''.join([ "--snapshot_kind=", options.snapshot_kind ])) - script_args.append(''.join([ "--vm_snapshot_data=", options.vm_output_bin ])) - script_args.append(''.join([ "--isolate_snapshot_data=", options.isolate_output_bin ])) + # First setup the vm isolate and regular isolate snapshot output filename. + script_args.append(''.join(["--snapshot_kind=", options.snapshot_kind])) + script_args.append(''.join(["--vm_snapshot_data=", options.vm_output_bin])) + script_args.append(''.join( + ["--isolate_snapshot_data=", options.isolate_output_bin])) - if options.vm_instructions_output_bin != None: - script_args.append(''.join([ "--vm_snapshot_instructions=", - options.vm_instructions_output_bin ])) - if options.isolate_instructions_output_bin != None: - script_args.append(''.join([ "--isolate_snapshot_instructions=", - options.isolate_instructions_output_bin ])) + if options.vm_instructions_output_bin != None: + script_args.append(''.join( + ["--vm_snapshot_instructions=", + options.vm_instructions_output_bin])) + if options.isolate_instructions_output_bin != None: + script_args.append(''.join([ + "--isolate_snapshot_instructions=", + options.isolate_instructions_output_bin + ])) - # Finally append the script name if one is specified. - if options.script: - script_args.append(options.script) + # Finally append the script name if one is specified. + if options.script: + script_args.append(options.script) - # Construct command line to execute the snapshot generator binary and invoke. - command = [ options.executable ] + script_args - try: - utils.RunCommand(command, outStream=sys.stderr, errStream=sys.stderr, - verbose=options.verbose, printErrorInfo=True) - except Exception as e: - return -1 + # Construct command line to execute the snapshot generator binary and invoke. + command = [options.executable] + script_args + try: + utils.RunCommand( + command, + outStream=sys.stderr, + errStream=sys.stderr, + verbose=options.verbose, + printErrorInfo=True) + except Exception as e: + return -1 - # Success, update timestamp file. - CreateTimestampFile(options) + # Success, update timestamp file. + CreateTimestampFile(options) - return 0 + return 0 if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/runtime/tools/create_snapshot_file.py b/runtime/tools/create_snapshot_file.py index 19ea870d14c..42163662e1c 100755 --- a/runtime/tools/create_snapshot_file.py +++ b/runtime/tools/create_snapshot_file.py @@ -14,106 +14,118 @@ import subprocess import sys import utils - HOST_OS = utils.GuessOS() HOST_CPUS = utils.GuessCpus() def BuildOptions(): - result = optparse.OptionParser() - result.add_option("--vm_input_bin", - action="store", type="string", - help="input file name of the vm isolate snapshot in binary form") - result.add_option("--input_bin", - action="store", type="string", - help="input file name of the isolate snapshot in binary form") - result.add_option("--input_cc", - action="store", type="string", - help="input file name which contains the C buffer template") - result.add_option("--output", - action="store", type="string", - help="output file name into which snapshot in C buffer form is generated") - result.add_option("-v", "--verbose", - help='Verbose output.', - default=False, action="store_true") - return result + result = optparse.OptionParser() + result.add_option( + "--vm_input_bin", + action="store", + type="string", + help="input file name of the vm isolate snapshot in binary form") + result.add_option( + "--input_bin", + action="store", + type="string", + help="input file name of the isolate snapshot in binary form") + result.add_option( + "--input_cc", + action="store", + type="string", + help="input file name which contains the C buffer template") + result.add_option( + "--output", + action="store", + type="string", + help="output file name into which snapshot in C buffer form is generated" + ) + result.add_option( + "-v", + "--verbose", + help='Verbose output.', + default=False, + action="store_true") + return result def ProcessOptions(options): - if not options.vm_input_bin: - sys.stderr.write('--vm_input_bin not specified\n') - return False - if not options.input_bin: - sys.stderr.write('--input_bin not specified\n') - return False - if not options.input_cc: - sys.stderr.write('--input_cc not specified\n') - return False - if not options.output: - sys.stderr.write('--output not specified\n') - return False - return True + if not options.vm_input_bin: + sys.stderr.write('--vm_input_bin not specified\n') + return False + if not options.input_bin: + sys.stderr.write('--input_bin not specified\n') + return False + if not options.input_cc: + sys.stderr.write('--input_cc not specified\n') + return False + if not options.output: + sys.stderr.write('--output not specified\n') + return False + return True def WriteBytesAsText(out, input_file): - """Writes byte contents of the input_file into out file as text. + """Writes byte contents of the input_file into out file as text. Output is formatted as a list of comma separated integer values - one value for each byte. """ - with open(input_file, 'rb') as input: - lineCounter = 0 - line = ' ' - for byte in input.read(): - line += ' %d,' % ord(byte) - lineCounter += 1 - if lineCounter == 10: - out.write(line + '\n') - line = ' ' + with open(input_file, 'rb') as input: lineCounter = 0 - if lineCounter != 0: - out.write(line + '\n') + line = ' ' + for byte in input.read(): + line += ' %d,' % ord(byte) + lineCounter += 1 + if lineCounter == 10: + out.write(line + '\n') + line = ' ' + lineCounter = 0 + if lineCounter != 0: + out.write(line + '\n') -def GenerateFileFromTemplate(output_file, input_cc_file, - vm_isolate_input_file, isolate_input_file): - """Generates C++ file based on a input_cc_file template and two binary files +def GenerateFileFromTemplate(output_file, input_cc_file, vm_isolate_input_file, + isolate_input_file): + """Generates C++ file based on a input_cc_file template and two binary files Template is expected to have two %s placehoders which would be filled with binary contents of the given files each formatted as a comma separated list of integers. """ - snapshot_cc_text = open(input_cc_file).read() - chunks = snapshot_cc_text.split("%s") - if len(chunks) != 3: - raise Exception("Template %s should contain exactly two %%s occurrences" - % input_cc_file) + snapshot_cc_text = open(input_cc_file).read() + chunks = snapshot_cc_text.split("%s") + if len(chunks) != 3: + raise Exception("Template %s should contain exactly two %%s occurrences" + % input_cc_file) - with open(output_file, 'w') as out: - out.write(chunks[0]) - WriteBytesAsText(out, vm_isolate_input_file) - out.write(chunks[1]) - WriteBytesAsText(out, isolate_input_file) - out.write(chunks[2]) + with open(output_file, 'w') as out: + out.write(chunks[0]) + WriteBytesAsText(out, vm_isolate_input_file) + out.write(chunks[1]) + WriteBytesAsText(out, isolate_input_file) + out.write(chunks[2]) def Main(): - # Parse options. - parser = BuildOptions() - (options, args) = parser.parse_args() - if not ProcessOptions(options): - parser.print_help() - return 1 + # Parse options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 - # If there are additional arguments, report error and exit. - if args: - parser.print_help() - return 1 + # If there are additional arguments, report error and exit. + if args: + parser.print_help() + return 1 - GenerateFileFromTemplate(options.output, options.input_cc, - options.vm_input_bin, options.input_bin) + GenerateFileFromTemplate(options.output, options.input_cc, + options.vm_input_bin, options.input_bin) + + return 0 - return 0 if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/runtime/tools/create_string_literal.py b/runtime/tools/create_string_literal.py index 52a9b1013c7..b7db38840c6 100755 --- a/runtime/tools/create_string_literal.py +++ b/runtime/tools/create_string_literal.py @@ -15,85 +15,82 @@ from optparse import OptionParser def makeString(input_files): - result = ' ' - for string_file in input_files: - if string_file.endswith('dart'): - fileHandle = open(string_file, 'rb') - lineCounter = 0 - result += ' // ' + string_file + '\n ' - for byte in fileHandle.read(): - result += ' %d,' % ord(byte) - lineCounter += 1 - if lineCounter == 10: - result += '\n ' - lineCounter = 0 - if lineCounter != 0: - result += '\n ' - result += ' // Terminating null character.\n 0' - return result + result = ' ' + for string_file in input_files: + if string_file.endswith('dart'): + fileHandle = open(string_file, 'rb') + lineCounter = 0 + result += ' // ' + string_file + '\n ' + for byte in fileHandle.read(): + result += ' %d,' % ord(byte) + lineCounter += 1 + if lineCounter == 10: + result += '\n ' + lineCounter = 0 + if lineCounter != 0: + result += '\n ' + result += ' // Terminating null character.\n 0' + return result def makeFile(output_file, input_cc_file, include, var_name, input_files): - bootstrap_cc_text = open(input_cc_file).read() - bootstrap_cc_text = bootstrap_cc_text.replace("{{INCLUDE}}", include) - bootstrap_cc_text = bootstrap_cc_text.replace("{{VAR_NAME}}", var_name) - bootstrap_cc_text = bootstrap_cc_text.replace("{{DART_SOURCE}}", - makeString(input_files)) - open(output_file, 'w').write(bootstrap_cc_text) - return True + bootstrap_cc_text = open(input_cc_file).read() + bootstrap_cc_text = bootstrap_cc_text.replace("{{INCLUDE}}", include) + bootstrap_cc_text = bootstrap_cc_text.replace("{{VAR_NAME}}", var_name) + bootstrap_cc_text = bootstrap_cc_text.replace("{{DART_SOURCE}}", + makeString(input_files)) + open(output_file, 'w').write(bootstrap_cc_text) + return True def main(args): - try: - # Parse input. - parser = OptionParser() - parser.add_option("--output", - action="store", type="string", - help="output file name") - parser.add_option("--input_cc", - action="store", type="string", - help="input template file") - parser.add_option("--include", - action="store", type="string", - help="variable name") - parser.add_option("--var_name", - action="store", type="string", - help="variable name") + try: + # Parse input. + parser = OptionParser() + parser.add_option( + "--output", action="store", type="string", help="output file name") + parser.add_option( + "--input_cc", + action="store", + type="string", + help="input template file") + parser.add_option( + "--include", action="store", type="string", help="variable name") + parser.add_option( + "--var_name", action="store", type="string", help="variable name") - (options, args) = parser.parse_args() - if not options.output: - sys.stderr.write('--output not specified\n') - return -1 - if not len(options.input_cc): - sys.stderr.write('--input_cc not specified\n') - return -1 - if not len(options.include): - sys.stderr.write('--include not specified\n') - return -1 - if not len(options.var_name): - sys.stderr.write('--var_name not specified\n') - return -1 - if len(args) == 0: - sys.stderr.write('No input files specified\n') - return -1 + (options, args) = parser.parse_args() + if not options.output: + sys.stderr.write('--output not specified\n') + return -1 + if not len(options.input_cc): + sys.stderr.write('--input_cc not specified\n') + return -1 + if not len(options.include): + sys.stderr.write('--include not specified\n') + return -1 + if not len(options.var_name): + sys.stderr.write('--var_name not specified\n') + return -1 + if len(args) == 0: + sys.stderr.write('No input files specified\n') + return -1 - files = [ ] - for arg in args: - files.append(arg) + files = [] + for arg in args: + files.append(arg) - if not makeFile(options.output, - options.input_cc, - options.include, - options.var_name, - files): - return -1 + if not makeFile(options.output, options.input_cc, options.include, + options.var_name, files): + return -1 + + return 0 + except Exception, inst: + sys.stderr.write('create_string_literal.py exception\n') + sys.stderr.write(str(inst)) + sys.stderr.write('\n') + return -1 - return 0 - except Exception, inst: - sys.stderr.write('create_string_literal.py exception\n') - sys.stderr.write(str(inst)) - sys.stderr.write('\n') - return -1 if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/runtime/tools/gen_library_src_paths.py b/runtime/tools/gen_library_src_paths.py index cd1f38d5964..56d770a4a2c 100755 --- a/runtime/tools/gen_library_src_paths.py +++ b/runtime/tools/gen_library_src_paths.py @@ -17,119 +17,121 @@ HOST_OS = utils.GuessOS() def makeString(input_file, var_name): - result = 'static const char ' + var_name + '[] = {\n ' - fileHandle = open(input_file, 'rb') - lineCounter = 0 - for byte in fileHandle.read(): - result += '\'\\x%02x' % ord(byte) + '\', ' - lineCounter += 1 - if lineCounter == 19: - result += '\n ' - lineCounter = 0 - result += '0};\n' - return result + result = 'static const char ' + var_name + '[] = {\n ' + fileHandle = open(input_file, 'rb') + lineCounter = 0 + for byte in fileHandle.read(): + result += '\'\\x%02x' % ord(byte) + '\', ' + lineCounter += 1 + if lineCounter == 19: + result += '\n ' + lineCounter = 0 + result += '0};\n' + return result + def makeSourceArrays(in_files): - result = ''; - file_count = 0; - for string_file in in_files: - if string_file.endswith('.dart'): - file_count += 1 - file_string = makeString(string_file, "source_array_" + str(file_count)) - result += file_string - return result + result = '' + file_count = 0 + for string_file in in_files: + if string_file.endswith('.dart'): + file_count += 1 + file_string = makeString(string_file, + "source_array_" + str(file_count)) + result += file_string + return result + def makeFile(output_file, input_cc_file, include, var_name, lib_name, in_files): - part_index = [ ] - bootstrap_cc_text = open(input_cc_file).read() - bootstrap_cc_text = bootstrap_cc_text.replace("{{SOURCE_ARRAYS}}", makeSourceArrays(in_files)) - bootstrap_cc_text = bootstrap_cc_text.replace("{{INCLUDE}}", include) - bootstrap_cc_text = bootstrap_cc_text.replace("{{VAR_NAME}}", var_name) - main_file_found = False - file_count = 0 - for string_file in in_files: - if string_file.endswith('.dart'): - file_count += 1 - if (not main_file_found): - inpt = open(string_file, 'r') - for line in inpt: - # File with library tag is the main file. - if line.startswith('library '): - main_file_found = True - bootstrap_cc_text = bootstrap_cc_text.replace( - "{{LIBRARY_SOURCE_MAP}}", - ' "' + lib_name + '",\n' + - ' source_array_' + str(file_count) + ',\n') - inpt.close() - if (main_file_found): - continue - part_index.append(' "' + - lib_name + "/" + os.path.basename(string_file).replace('\\', '\\\\') + '",\n') - part_index.append(' source_array_' + str(file_count) + ',\n\n') - bootstrap_cc_text = bootstrap_cc_text.replace("{{LIBRARY_SOURCE_MAP}}", '') - bootstrap_cc_text = bootstrap_cc_text.replace("{{PART_SOURCE_MAP}}", - ''.join(part_index)) - open(output_file, 'w').write(bootstrap_cc_text) - return True + part_index = [] + bootstrap_cc_text = open(input_cc_file).read() + bootstrap_cc_text = bootstrap_cc_text.replace("{{SOURCE_ARRAYS}}", + makeSourceArrays(in_files)) + bootstrap_cc_text = bootstrap_cc_text.replace("{{INCLUDE}}", include) + bootstrap_cc_text = bootstrap_cc_text.replace("{{VAR_NAME}}", var_name) + main_file_found = False + file_count = 0 + for string_file in in_files: + if string_file.endswith('.dart'): + file_count += 1 + if (not main_file_found): + inpt = open(string_file, 'r') + for line in inpt: + # File with library tag is the main file. + if line.startswith('library '): + main_file_found = True + bootstrap_cc_text = bootstrap_cc_text.replace( + "{{LIBRARY_SOURCE_MAP}}", ' "' + lib_name + '",\n' + + ' source_array_' + str(file_count) + ',\n') + inpt.close() + if (main_file_found): + continue + part_index.append(' "' + lib_name + "/" + os.path.basename( + string_file).replace('\\', '\\\\') + '",\n') + part_index.append(' source_array_' + str(file_count) + ',\n\n') + bootstrap_cc_text = bootstrap_cc_text.replace("{{LIBRARY_SOURCE_MAP}}", '') + bootstrap_cc_text = bootstrap_cc_text.replace("{{PART_SOURCE_MAP}}", + ''.join(part_index)) + open(output_file, 'w').write(bootstrap_cc_text) + return True + def main(args): - try: - # Parse input. - parser = OptionParser() - parser.add_option("--output", - action="store", type="string", - help="output file name") - parser.add_option("--input_cc", - action="store", type="string", - help="input template file") - parser.add_option("--include", - action="store", type="string", - help="variable name") - parser.add_option("--library_name", - action="store", type="string", - help="library name") - parser.add_option("--var_name", - action="store", type="string", - help="variable name") + try: + # Parse input. + parser = OptionParser() + parser.add_option( + "--output", action="store", type="string", help="output file name") + parser.add_option( + "--input_cc", + action="store", + type="string", + help="input template file") + parser.add_option( + "--include", action="store", type="string", help="variable name") + parser.add_option( + "--library_name", + action="store", + type="string", + help="library name") + parser.add_option( + "--var_name", action="store", type="string", help="variable name") - (options, args) = parser.parse_args() - if not options.output: - sys.stderr.write('--output not specified\n') - return -1 - if not len(options.input_cc): - sys.stderr.write('--input_cc not specified\n') - return -1 - if not len(options.include): - sys.stderr.write('--include not specified\n') - return -1 - if not len(options.var_name): - sys.stderr.write('--var_name not specified\n') - return -1 - if not len(options.library_name): - sys.stderr.write('--library_name not specified\n') - return -1 - if len(args) == 0: - sys.stderr.write('No input files specified\n') - return -1 + (options, args) = parser.parse_args() + if not options.output: + sys.stderr.write('--output not specified\n') + return -1 + if not len(options.input_cc): + sys.stderr.write('--input_cc not specified\n') + return -1 + if not len(options.include): + sys.stderr.write('--include not specified\n') + return -1 + if not len(options.var_name): + sys.stderr.write('--var_name not specified\n') + return -1 + if not len(options.library_name): + sys.stderr.write('--library_name not specified\n') + return -1 + if len(args) == 0: + sys.stderr.write('No input files specified\n') + return -1 - files = [ ] - for arg in args: - files.append(arg) + files = [] + for arg in args: + files.append(arg) - if not makeFile(options.output, - options.input_cc, - options.include, - options.var_name, - options.library_name, - files): - return -1 + if not makeFile(options.output, options.input_cc, options.include, + options.var_name, options.library_name, files): + return -1 + + return 0 + except Exception, inst: + sys.stderr.write('gen_library_src_paths.py exception\n') + sys.stderr.write(str(inst)) + sys.stderr.write('\n') + return -1 - return 0 - except Exception, inst: - sys.stderr.write('gen_library_src_paths.py exception\n') - sys.stderr.write(str(inst)) - sys.stderr.write('\n') - return -1 if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/runtime/tools/layering_check.py b/runtime/tools/layering_check.py index 50a9b25644a..e686ce9304b 100755 --- a/runtime/tools/layering_check.py +++ b/runtime/tools/layering_check.py @@ -18,110 +18,115 @@ import sys INCLUDE_DIRECTIVE_RE = re.compile(r'^#include "(.*)"') RUNTIME_LAYER_HEADERS = [ - 'runtime/vm/isolate.h', - 'runtime/vm/object.h', - 'runtime/vm/raw_object.h', - 'runtime/vm/thread.h', + 'runtime/vm/isolate.h', + 'runtime/vm/object.h', + 'runtime/vm/raw_object.h', + 'runtime/vm/thread.h', ] SHOULD_NOT_DEPEND_ON_RUNTIME = [ - 'runtime/vm/allocation.h', - 'runtime/vm/growable_array.h', + 'runtime/vm/allocation.h', + 'runtime/vm/growable_array.h', ] + class LayeringChecker(object): - def __init__(self, root): - self.root = root - self.worklist = set() - # Mapping from header to a set of files it is included into. - self.included_into = dict() - # Set of files that were parsed to avoid double parsing. - self.loaded = set() - # Mapping from headers to their layer. - self.file_layers = {file: 'runtime' for file in RUNTIME_LAYER_HEADERS} - def Check(self): - self.AddAllSourcesToWorklist(os.path.join(self.root, 'runtime/vm')) - self.BuildIncludesGraph() - errors = self.PropagateLayers() - errors += self.CheckNotInRuntime(SHOULD_NOT_DEPEND_ON_RUNTIME) - return errors + def __init__(self, root): + self.root = root + self.worklist = set() + # Mapping from header to a set of files it is included into. + self.included_into = dict() + # Set of files that were parsed to avoid double parsing. + self.loaded = set() + # Mapping from headers to their layer. + self.file_layers = {file: 'runtime' for file in RUNTIME_LAYER_HEADERS} - def CheckNotInRuntime(self, files): - """Check that given files do not depend on runtime layer.""" - errors = [] - for file in files: - if not os.path.exists(os.path.join(self.root, file)): - errors.append('File %s does not exist.' % (file)) - if self.file_layers.get(file) is not None: - errors.append( - 'LAYERING ERROR: %s includes object.h or raw_object.h' % (file)) - return errors + def Check(self): + self.AddAllSourcesToWorklist(os.path.join(self.root, 'runtime/vm')) + self.BuildIncludesGraph() + errors = self.PropagateLayers() + errors += self.CheckNotInRuntime(SHOULD_NOT_DEPEND_ON_RUNTIME) + return errors - def BuildIncludesGraph(self): - while self.worklist: - file = self.worklist.pop() - deps = self.ExtractIncludes(file) - self.loaded.add(file) - for d in deps: - if d not in self.included_into: - self.included_into[d] = set() - self.included_into[d].add(file) - if d not in self.loaded: - self.worklist.add(d) + def CheckNotInRuntime(self, files): + """Check that given files do not depend on runtime layer.""" + errors = [] + for file in files: + if not os.path.exists(os.path.join(self.root, file)): + errors.append('File %s does not exist.' % (file)) + if self.file_layers.get(file) is not None: + errors.append( + 'LAYERING ERROR: %s includes object.h or raw_object.h' % + (file)) + return errors - def PropagateLayers(self): - """Propagate layering information through include graph. + def BuildIncludesGraph(self): + while self.worklist: + file = self.worklist.pop() + deps = self.ExtractIncludes(file) + self.loaded.add(file) + for d in deps: + if d not in self.included_into: + self.included_into[d] = set() + self.included_into[d].add(file) + if d not in self.loaded: + self.worklist.add(d) + + def PropagateLayers(self): + """Propagate layering information through include graph. If A is in layer L and A is included into B then B is in layer L. """ - errors = [] - self.worklist = set(self.file_layers.keys()) - while self.worklist: - file = self.worklist.pop() - if file not in self.included_into: - continue - file_layer = self.file_layers[file] - for tgt in self.included_into[file]: - if tgt in self.file_layers: - if self.file_layers[tgt] != file_layer: - errors.add('Layer mismatch: %s (%s) is included into %s (%s)' % ( - file, file_layer, tgt, self.file_layers[tgt])) - self.file_layers[tgt] = file_layer - self.worklist.add(tgt) - return errors + errors = [] + self.worklist = set(self.file_layers.keys()) + while self.worklist: + file = self.worklist.pop() + if file not in self.included_into: + continue + file_layer = self.file_layers[file] + for tgt in self.included_into[file]: + if tgt in self.file_layers: + if self.file_layers[tgt] != file_layer: + errors.add( + 'Layer mismatch: %s (%s) is included into %s (%s)' % + (file, file_layer, tgt, self.file_layers[tgt])) + self.file_layers[tgt] = file_layer + self.worklist.add(tgt) + return errors - def AddAllSourcesToWorklist(self, dir): - """Add all *.cc and *.h files from dir recursively into worklist.""" - for file in os.listdir(dir): - path = os.path.join(dir, file) - if os.path.isdir(path): - self.AddAllSourcesToWorklist(path) - elif path.endswith('.cc') or path.endswith('.h'): - self.worklist.add(os.path.relpath(path, self.root)) + def AddAllSourcesToWorklist(self, dir): + """Add all *.cc and *.h files from dir recursively into worklist.""" + for file in os.listdir(dir): + path = os.path.join(dir, file) + if os.path.isdir(path): + self.AddAllSourcesToWorklist(path) + elif path.endswith('.cc') or path.endswith('.h'): + self.worklist.add(os.path.relpath(path, self.root)) - def ExtractIncludes(self, file): - """Extract the list of includes from the given file.""" - deps = set() - with open(os.path.join(self.root, file)) as file: - for line in file: - if line.startswith('namespace dart {'): - break + def ExtractIncludes(self, file): + """Extract the list of includes from the given file.""" + deps = set() + with open(os.path.join(self.root, file)) as file: + for line in file: + if line.startswith('namespace dart {'): + break + + m = INCLUDE_DIRECTIVE_RE.match(line) + if m is not None: + header = os.path.join('runtime', m.group(1)) + if os.path.isfile(os.path.join(self.root, header)): + deps.add(header) + return deps - m = INCLUDE_DIRECTIVE_RE.match(line) - if m is not None: - header = os.path.join('runtime', m.group(1)) - if os.path.isfile(os.path.join(self.root,header)): - deps.add(header) - return deps def DoCheck(sdk_root): - """Run layering check at the given root folder.""" - return LayeringChecker(sdk_root).Check() + """Run layering check at the given root folder.""" + return LayeringChecker(sdk_root).Check() + if __name__ == '__main__': - errors = DoCheck('.') - print '\n'.join(errors) - if errors: - sys.exit(-1) - + errors = DoCheck('.') + print '\n'.join(errors) + if errors: + sys.exit(-1) diff --git a/runtime/tools/utils.py b/runtime/tools/utils.py index 595a9ca8edd..4148bd34566 100644 --- a/runtime/tools/utils.py +++ b/runtime/tools/utils.py @@ -19,133 +19,145 @@ import time # Try to guess the host operating system. def GuessOS(): - id = platform.system() - if id == "Linux": - return "linux" - elif id == "Darwin": - return "macos" - elif id == "Windows" or id == "Microsoft": - # On Windows Vista platform.system() can return "Microsoft" with some - # versions of Python, see http://bugs.python.org/issue1082 for details. - return "win32" - elif id == 'FreeBSD': - return 'freebsd' - elif id == 'OpenBSD': - return 'openbsd' - elif id == 'SunOS': - return 'solaris' - else: - return None + id = platform.system() + if id == "Linux": + return "linux" + elif id == "Darwin": + return "macos" + elif id == "Windows" or id == "Microsoft": + # On Windows Vista platform.system() can return "Microsoft" with some + # versions of Python, see http://bugs.python.org/issue1082 for details. + return "win32" + elif id == 'FreeBSD': + return 'freebsd' + elif id == 'OpenBSD': + return 'openbsd' + elif id == 'SunOS': + return 'solaris' + else: + return None # Try to guess the host architecture. def GuessArchitecture(): - id = platform.machine() - if id.startswith('arm'): - return 'arm' - elif (not id) or (not re.match('(x|i[3-6])86', id) is None): - return 'ia32' - elif id == 'i86pc': - return 'ia32' - else: - return None + id = platform.machine() + if id.startswith('arm'): + return 'arm' + elif (not id) or (not re.match('(x|i[3-6])86', id) is None): + return 'ia32' + elif id == 'i86pc': + return 'ia32' + else: + return None # Try to guess the number of cpus on this machine. def GuessCpus(): - if os.path.exists("/proc/cpuinfo"): - return int(commands.getoutput("GREP_OPTIONS= grep -E '^processor' /proc/cpuinfo | wc -l")) - if os.path.exists("/usr/bin/hostinfo"): - return int(commands.getoutput('/usr/bin/hostinfo | GREP_OPTIONS= grep "processors are logically available." | awk "{ print \$1 }"')) - win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS") - if win_cpu_count: - return int(win_cpu_count) - return int(os.getenv("DART_NUMBER_OF_CORES", 2)) + if os.path.exists("/proc/cpuinfo"): + return int( + commands.getoutput( + "GREP_OPTIONS= grep -E '^processor' /proc/cpuinfo | wc -l")) + if os.path.exists("/usr/bin/hostinfo"): + return int( + commands.getoutput( + '/usr/bin/hostinfo | GREP_OPTIONS= grep "processors are logically available." | awk "{ print \$1 }"' + )) + win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS") + if win_cpu_count: + return int(win_cpu_count) + return int(os.getenv("DART_NUMBER_OF_CORES", 2)) # Returns true if we're running under Windows. def IsWindows(): - return GuessOS() == 'win32' + return GuessOS() == 'win32' # Reads a text file into an array of strings - one for each # line. Strips comments in the process. def ReadLinesFrom(name): - result = [] - for line in open(name): - if '#' in line: - line = line[:line.find('#')] - line = line.strip() - if len(line) == 0: - continue - result.append(line) - return result + result = [] + for line in open(name): + if '#' in line: + line = line[:line.find('#')] + line = line.strip() + if len(line) == 0: + continue + result.append(line) + return result + # Filters out all arguments until the next '--' argument # occurs. def ListArgCallback(option, opt_str, value, parser): - if value is None: - value = [] + if value is None: + value = [] - for arg in parser.rargs: - if arg[:2].startswith('--'): - break - value.append(arg) + for arg in parser.rargs: + if arg[:2].startswith('--'): + break + value.append(arg) - del parser.rargs[:len(value)] - setattr(parser.values, option.dest, value) + del parser.rargs[:len(value)] + setattr(parser.values, option.dest, value) # Filters out all argument until the first non '-' or the # '--' argument occurs. def ListDashArgCallback(option, opt_str, value, parser): - if value is None: - value = [] + if value is None: + value = [] - for arg in parser.rargs: - if arg[:2].startswith('--') or arg[0] != '-': - break - value.append(arg) + for arg in parser.rargs: + if arg[:2].startswith('--') or arg[0] != '-': + break + value.append(arg) - del parser.rargs[:len(value)] - setattr(parser.values, option.dest, value) + del parser.rargs[:len(value)] + setattr(parser.values, option.dest, value) # Mapping table between build mode and build configuration. BUILD_MODES = { - 'debug': 'Debug', - 'release': 'Release', + 'debug': 'Debug', + 'release': 'Release', } - # Mapping table between OS and build output location. BUILD_ROOT = { - 'linux': os.path.join('out'), - 'freebsd': os.path.join('out'), - 'macos': os.path.join('xcodebuild'), + 'linux': os.path.join('out'), + 'freebsd': os.path.join('out'), + 'macos': os.path.join('xcodebuild'), } + def GetBuildMode(mode): - global BUILD_MODES - return BUILD_MODES[mode] + global BUILD_MODES + return BUILD_MODES[mode] def GetBuildConf(mode, arch): - return GetBuildMode(mode) + arch.upper() + return GetBuildMode(mode) + arch.upper() def GetBuildRoot(host_os, mode=None, arch=None): - global BUILD_ROOT - if mode: - return os.path.join(BUILD_ROOT[host_os], GetBuildConf(mode, arch)) - else: - return BUILD_ROOT[host_os] + global BUILD_ROOT + if mode: + return os.path.join(BUILD_ROOT[host_os], GetBuildConf(mode, arch)) + else: + return BUILD_ROOT[host_os] -def RunCommand(command, input=None, pollFn=None, outStream=None, errStream=None, - killOnEarlyReturn=True, verbose=False, debug=False, +def RunCommand(command, + input=None, + pollFn=None, + outStream=None, + errStream=None, + killOnEarlyReturn=True, + verbose=False, + debug=False, printErrorInfo=False): - """ + """ Run a command, with optional input and polling function. Args: @@ -171,94 +183,98 @@ def RunCommand(command, input=None, pollFn=None, outStream=None, errStream=None, Raises Error if the subprocess returns an error code. Raises ValueError if called with invalid arguments. """ - if verbose: - sys.stderr.write("command %s\n" % command) - stdin = None - if input: - stdin = subprocess.PIPE - try: - process = subprocess.Popen(args=command, - stdin=stdin, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError as e: - if not isinstance(command, basestring): - command = ' '.join(command) - if printErrorInfo: - sys.stderr.write("Command failed: '%s'\n" % command) - raise Error(e) - - def StartThread(out): - queue = Queue.Queue() - def EnqueueOutput(out, queue): - for line in iter(out.readline, b''): - queue.put(line) - out.close() - thread = threading.Thread(target=EnqueueOutput, args=(out, queue)) - thread.daemon = True - thread.start() - return queue - outQueue = StartThread(process.stdout) - errQueue = StartThread(process.stderr) - - def ReadQueue(queue, out, out2): + if verbose: + sys.stderr.write("command %s\n" % command) + stdin = None + if input: + stdin = subprocess.PIPE try: - while True: - line = queue.get(False) - out.write(line) - if out2 != None: - out2.write(line) - except Queue.Empty: - pass + process = subprocess.Popen( + args=command, + stdin=stdin, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as e: + if not isinstance(command, basestring): + command = ' '.join(command) + if printErrorInfo: + sys.stderr.write("Command failed: '%s'\n" % command) + raise Error(e) - outBuf = StringIO.StringIO() - errorBuf = StringIO.StringIO() - if input: - process.stdin.write(input) - while True: - returncode = process.poll() - if returncode != None: - break + def StartThread(out): + queue = Queue.Queue() + + def EnqueueOutput(out, queue): + for line in iter(out.readline, b''): + queue.put(line) + out.close() + + thread = threading.Thread(target=EnqueueOutput, args=(out, queue)) + thread.daemon = True + thread.start() + return queue + + outQueue = StartThread(process.stdout) + errQueue = StartThread(process.stderr) + + def ReadQueue(queue, out, out2): + try: + while True: + line = queue.get(False) + out.write(line) + if out2 != None: + out2.write(line) + except Queue.Empty: + pass + + outBuf = StringIO.StringIO() + errorBuf = StringIO.StringIO() + if input: + process.stdin.write(input) + while True: + returncode = process.poll() + if returncode != None: + break + ReadQueue(errQueue, errorBuf, errStream) + ReadQueue(outQueue, outBuf, outStream) + if pollFn != None and pollFn(): + returncode = 0 + if killOnEarlyReturn: + process.kill() + break + time.sleep(0.1) + # Drain queue ReadQueue(errQueue, errorBuf, errStream) ReadQueue(outQueue, outBuf, outStream) - if pollFn != None and pollFn(): - returncode = 0 - if killOnEarlyReturn: - process.kill() - break - time.sleep(0.1) - # Drain queue - ReadQueue(errQueue, errorBuf, errStream) - ReadQueue(outQueue, outBuf, outStream) - out = outBuf.getvalue(); - error = errorBuf.getvalue(); - if returncode: - if not isinstance(command, basestring): - command = ' '.join(command) - if printErrorInfo: - sys.stderr.write("Command failed: '%s'\n" % command) - sys.stderr.write(" stdout: '%s'\n" % out) - sys.stderr.write(" stderr: '%s'\n" % error) - sys.stderr.write(" returncode: %d\n" % returncode) - raise Error("Command failed: %s" % command) - if debug: - sys.stderr.write("output: %s\n" % out) - return out + out = outBuf.getvalue() + error = errorBuf.getvalue() + if returncode: + if not isinstance(command, basestring): + command = ' '.join(command) + if printErrorInfo: + sys.stderr.write("Command failed: '%s'\n" % command) + sys.stderr.write(" stdout: '%s'\n" % out) + sys.stderr.write(" stderr: '%s'\n" % error) + sys.stderr.write(" returncode: %d\n" % returncode) + raise Error("Command failed: %s" % command) + if debug: + sys.stderr.write("output: %s\n" % out) + return out def Main(argv): - print "GuessOS() -> ", GuessOS() - print "GuessArchitecture() -> ", GuessArchitecture() - print "GuessCpus() -> ", GuessCpus() - print "IsWindows() -> ", IsWindows() + print "GuessOS() -> ", GuessOS() + print "GuessArchitecture() -> ", GuessArchitecture() + print "GuessCpus() -> ", GuessCpus() + print "IsWindows() -> ", IsWindows() class Error(Exception): - pass + pass if __name__ == "__main__": - import sys - Main(sys.argv) + import sys + Main(sys.argv) diff --git a/runtime/tools/valgrind.py b/runtime/tools/valgrind.py index c0440f17485..f82d6d378c8 100755 --- a/runtime/tools/valgrind.py +++ b/runtime/tools/valgrind.py @@ -12,21 +12,20 @@ import sys import re VALGRIND_ARGUMENTS = [ - 'valgrind', - '--error-exitcode=1', - '--leak-check=full', - '--trace-children=yes', - '--ignore-ranges=0x000-0xFFF', # Used for implicit null checks. - '--vex-iropt-level=1' # Valgrind crashes with the default level (2). + 'valgrind', + '--error-exitcode=1', + '--leak-check=full', + '--trace-children=yes', + '--ignore-ranges=0x000-0xFFF', # Used for implicit null checks. + '--vex-iropt-level=1' # Valgrind crashes with the default level (2). ] # Compute the command line. command = VALGRIND_ARGUMENTS + sys.argv[1:] # Run Valgrind. -process = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) +process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) code = process.wait() output = process.stdout.readlines() errors = process.stderr.readlines() @@ -34,13 +33,13 @@ errors = process.stderr.readlines() # Always print the output, but leave out the 3 line banner printed # by certain versions of Valgrind. if len(output) > 0 and output[0].startswith("** VALGRIND_ROOT="): - output = output[3:] + output = output[3:] sys.stdout.writelines(output) # If Valgrind produced an error, we report that to the user. if code != 0: - sys.stderr.writelines(errors) - sys.exit(code) + sys.stderr.writelines(errors) + sys.exit(code) # Look through the leak details and make sure that we don't have # any definitely or indirectly lost bytes. We allow possibly lost @@ -50,17 +49,17 @@ LEAK_LINE_MATCHER = re.compile(LEAK_RE) LEAK_OKAY_MATCHER = re.compile(r"lost: 0 bytes in 0 blocks") leaks = [] for line in errors: - if LEAK_LINE_MATCHER.search(line): - leaks.append(line) - if not LEAK_OKAY_MATCHER.search(line): - sys.stderr.writelines(errors) - sys.exit(1) + if LEAK_LINE_MATCHER.search(line): + leaks.append(line) + if not LEAK_OKAY_MATCHER.search(line): + sys.stderr.writelines(errors) + sys.exit(1) # Make sure we found the right number of leak lines. if not len(leaks) in [0, 2, 3]: - sys.stderr.writelines(errors) - sys.stderr.write('\n\n#### Malformed Valgrind output.\n#### Exiting.\n') - sys.exit(1) + sys.stderr.writelines(errors) + sys.stderr.write('\n\n#### Malformed Valgrind output.\n#### Exiting.\n') + sys.exit(1) # Success. sys.exit(0) diff --git a/samples-dev/swarm/appengine/encoder.py b/samples-dev/swarm/appengine/encoder.py index 6aa3c6a7838..a6c0c2e0a47 100644 --- a/samples-dev/swarm/appengine/encoder.py +++ b/samples-dev/swarm/appengine/encoder.py @@ -1,7 +1,6 @@ # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - ''' This Encoder shares a lot in common with protobufs. It uses variable length ints and size-encoded strings and binary values. Other than being hugely @@ -14,47 +13,49 @@ this range, all numeric data is encoded in only 7 bits. import base64 + class Encoder: - def __init__(self): - self.data = [] - def writeInt(self, value): - '''Uses a 7-bit per byte encoding to stay UTF-8 "safe".''' - bits = value & 0x3f - value >>= 6 - while value: - self.data.append(chr(0x40|bits)) - bits = value & 0x3f - value >>= 6 - self.data.append(chr(bits)) + def __init__(self): + self.data = [] - def writeBool(self, b): - self.data.append(('F', 'T')[b]) + def writeInt(self, value): + '''Uses a 7-bit per byte encoding to stay UTF-8 "safe".''' + bits = value & 0x3f + value >>= 6 + while value: + self.data.append(chr(0x40 | bits)) + bits = value & 0x3f + value >>= 6 + self.data.append(chr(bits)) - def writeString(self, s): - if not s: s = '' - self.writeInt(len(s)) - self.data.append(s) + def writeBool(self, b): + self.data.append(('F', 'T')[b]) - def writeBinary(self, s): - '''Encode binary data using base64. This is less efficient than a 7-bit + def writeString(self, s): + if not s: s = '' + self.writeInt(len(s)) + self.data.append(s) + + def writeBinary(self, s): + '''Encode binary data using base64. This is less efficient than a 7-bit encoding would be; however, it can be decoded much faster on most browsers due to native support for the format.''' - v = base64.b64encode(s) - self.writeInt(len(v)) - self.data.append(v) + v = base64.b64encode(s) + self.writeInt(len(v)) + self.data.append(v) - def writeList(self, l): - self.writeInt(len(l)) - for i in l: - i.encode(self) + def writeList(self, l): + self.writeInt(len(l)) + for i in l: + i.encode(self) - def writeRaw(self, s): - self.data.append(s) + def writeRaw(self, s): + self.data.append(s) - def finish(self): - d = ''.join(self.data) - return _encVarInt(len(d)) + d + def finish(self): + d = ''.join(self.data) + return _encVarInt(len(d)) + d - def getRaw(self): - return ''.join(self.data) + def getRaw(self): + return ''.join(self.data) diff --git a/samples-dev/swarm/appengine/main.py b/samples-dev/swarm/appengine/main.py index 5644847cc3f..f047c471a31 100644 --- a/samples-dev/swarm/appengine/main.py +++ b/samples-dev/swarm/appengine/main.py @@ -27,713 +27,755 @@ READER_API = 'http://www.google.com/reader/api/0' MAX_SECTIONS = 5 MAX_ARTICLES = 20 -class UserData(db.Model): - credentials = CredentialsProperty() - sections = db.ListProperty(db.Key) - def getEncodedData(self, articleKeys=None): - enc = encoder.Encoder() - # TODO(jimhug): Only return initially visible section in first reply. - maxSections = min(MAX_SECTIONS, len(self.sections)) - enc.writeInt(maxSections) - for section in db.get(self.sections[:maxSections]): - section.encode(enc, articleKeys) - return enc.getRaw() +class UserData(db.Model): + credentials = CredentialsProperty() + sections = db.ListProperty(db.Key) + + def getEncodedData(self, articleKeys=None): + enc = encoder.Encoder() + # TODO(jimhug): Only return initially visible section in first reply. + maxSections = min(MAX_SECTIONS, len(self.sections)) + enc.writeInt(maxSections) + for section in db.get(self.sections[:maxSections]): + section.encode(enc, articleKeys) + return enc.getRaw() class Section(db.Model): - title = db.TextProperty() - feeds = db.ListProperty(db.Key) + title = db.TextProperty() + feeds = db.ListProperty(db.Key) - def fixedTitle(self): - return self.title.split('_')[0] + def fixedTitle(self): + return self.title.split('_')[0] + + def encode(self, enc, articleKeys=None): + # TODO(jimhug): Need to optimize format and support incremental updates. + enc.writeString(self.key().name()) + enc.writeString(self.fixedTitle()) + enc.writeInt(len(self.feeds)) + for feed in db.get(self.feeds): + feed.ensureEncodedFeed() + enc.writeRaw(feed.encodedFeed3) + if articleKeys is not None: + articleKeys.extend(feed.topArticles) - def encode(self, enc, articleKeys=None): - # TODO(jimhug): Need to optimize format and support incremental updates. - enc.writeString(self.key().name()) - enc.writeString(self.fixedTitle()) - enc.writeInt(len(self.feeds)) - for feed in db.get(self.feeds): - feed.ensureEncodedFeed() - enc.writeRaw(feed.encodedFeed3) - if articleKeys is not None: - articleKeys.extend(feed.topArticles) class Feed(db.Model): - title = db.TextProperty() - iconUrl = db.TextProperty() - lastUpdated = db.IntegerProperty() + title = db.TextProperty() + iconUrl = db.TextProperty() + lastUpdated = db.IntegerProperty() - encodedFeed3 = db.TextProperty() - topArticles = db.ListProperty(db.Key) + encodedFeed3 = db.TextProperty() + topArticles = db.ListProperty(db.Key) - def ensureEncodedFeed(self, force=False): - if force or self.encodedFeed3 is None: - enc = encoder.Encoder() - articleSet = [] - self.encode(enc, MAX_ARTICLES, articleSet) - logging.info('articleSet length is %s' % len(articleSet)) - self.topArticles = articleSet - self.encodedFeed3 = enc.getRaw() - self.put() + def ensureEncodedFeed(self, force=False): + if force or self.encodedFeed3 is None: + enc = encoder.Encoder() + articleSet = [] + self.encode(enc, MAX_ARTICLES, articleSet) + logging.info('articleSet length is %s' % len(articleSet)) + self.topArticles = articleSet + self.encodedFeed3 = enc.getRaw() + self.put() - def encode(self, enc, maxArticles, articleSet): - enc.writeString(self.key().name()) - enc.writeString(self.title) - enc.writeString(self.iconUrl) + def encode(self, enc, maxArticles, articleSet): + enc.writeString(self.key().name()) + enc.writeString(self.title) + enc.writeString(self.iconUrl) - logging.info('encoding feed: %s' % self.title) - encodedArts = [] + logging.info('encoding feed: %s' % self.title) + encodedArts = [] - for article in self.article_set.order('-date').fetch(limit=maxArticles): - encodedArts.append(article.encodeHeader()) - articleSet.append(article.key()) + for article in self.article_set.order('-date').fetch(limit=maxArticles): + encodedArts.append(article.encodeHeader()) + articleSet.append(article.key()) - enc.writeInt(len(encodedArts)) - enc.writeRaw(''.join(encodedArts)) + enc.writeInt(len(encodedArts)) + enc.writeRaw(''.join(encodedArts)) class Article(db.Model): - feed = db.ReferenceProperty(Feed) + feed = db.ReferenceProperty(Feed) - title = db.TextProperty() - author = db.TextProperty() - content = db.TextProperty() - snippet = db.TextProperty() - thumbnail = db.BlobProperty() - thumbnailSize = db.TextProperty() - srcurl = db.TextProperty() - date = db.IntegerProperty() + title = db.TextProperty() + author = db.TextProperty() + content = db.TextProperty() + snippet = db.TextProperty() + thumbnail = db.BlobProperty() + thumbnailSize = db.TextProperty() + srcurl = db.TextProperty() + date = db.IntegerProperty() - def ensureThumbnail(self): - # If our desired thumbnail size has changed, regenerate it and cache. - if self.thumbnailSize != str(THUMB_SIZE): - self.thumbnail = makeThumbnail(self.content) - self.thumbnailSize = str(THUMB_SIZE) - self.put() + def ensureThumbnail(self): + # If our desired thumbnail size has changed, regenerate it and cache. + if self.thumbnailSize != str(THUMB_SIZE): + self.thumbnail = makeThumbnail(self.content) + self.thumbnailSize = str(THUMB_SIZE) + self.put() + + def encodeHeader(self): + # TODO(jmesserly): for now always unescape until the crawler catches up + enc = encoder.Encoder() + enc.writeString(self.key().name()) + enc.writeString(unescape(self.title)) + enc.writeString(self.srcurl) + enc.writeBool(self.thumbnail is not None) + enc.writeString(self.author) + enc.writeInt(self.date) + enc.writeString(unescape(self.snippet)) + return enc.getRaw() - def encodeHeader(self): - # TODO(jmesserly): for now always unescape until the crawler catches up - enc = encoder.Encoder() - enc.writeString(self.key().name()) - enc.writeString(unescape(self.title)) - enc.writeString(self.srcurl) - enc.writeBool(self.thumbnail is not None) - enc.writeString(self.author) - enc.writeInt(self.date) - enc.writeString(unescape(self.snippet)) - return enc.getRaw() class HtmlFile(db.Model): - content = db.BlobProperty() - compressed = db.BooleanProperty() - filename = db.StringProperty() - author = db.UserProperty(auto_current_user=True) - date = db.DateTimeProperty(auto_now_add=True) + content = db.BlobProperty() + compressed = db.BooleanProperty() + filename = db.StringProperty() + author = db.UserProperty(auto_current_user=True) + date = db.DateTimeProperty(auto_now_add=True) class UpdateHtml(webapp.RequestHandler): - def post(self): - upload_files = self.request.POST.multi.__dict__['_items'] - version = self.request.get('version') - logging.info('files: %r' % upload_files) - for data in upload_files: - if data[0] != 'files': continue - file = data[1] - filename = file.filename - if version: - filename = '%s-%s' % (version, filename) - logging.info('upload: %r' % filename) - htmlFile = HtmlFile.get_or_insert(filename) - htmlFile.filename = filename + def post(self): + upload_files = self.request.POST.multi.__dict__['_items'] + version = self.request.get('version') + logging.info('files: %r' % upload_files) + for data in upload_files: + if data[0] != 'files': continue + file = data[1] + filename = file.filename + if version: + filename = '%s-%s' % (version, filename) + logging.info('upload: %r' % filename) - # If text > (1MB - 1KB) then gzip text to fit in 1MB space - text = file.value - if len(text) > 1024*1023: - data = StringIO.StringIO() - gz = gzip.GzipFile(str(filename), 'wb', fileobj=data) - gz.write(text) - gz.close() - htmlFile.content = data.getvalue() - htmlFile.compressed = True - else: - htmlFile.content = text - htmlFile.compressed = False + htmlFile = HtmlFile.get_or_insert(filename) + htmlFile.filename = filename - htmlFile.put() + # If text > (1MB - 1KB) then gzip text to fit in 1MB space + text = file.value + if len(text) > 1024 * 1023: + data = StringIO.StringIO() + gz = gzip.GzipFile(str(filename), 'wb', fileobj=data) + gz.write(text) + gz.close() + htmlFile.content = data.getvalue() + htmlFile.compressed = True + else: + htmlFile.content = text + htmlFile.compressed = False + + htmlFile.put() + + self.redirect('/') - self.redirect('/') class TopHandler(webapp.RequestHandler): - @login_required - def get(self): - user = users.get_current_user() - prefs = UserData.get_by_key_name(user.user_id()) - if prefs is None: - self.redirect('/update/user') - return - params = {'files': HtmlFile.all().order('-date').fetch(limit=30)} - self.response.out.write(template.render('top.html', params)) + @login_required + def get(self): + user = users.get_current_user() + prefs = UserData.get_by_key_name(user.user_id()) + if prefs is None: + self.redirect('/update/user') + return + + params = {'files': HtmlFile.all().order('-date').fetch(limit=30)} + self.response.out.write(template.render('top.html', params)) class MainHandler(webapp.RequestHandler): - @login_required - def get(self, name): - if name == 'dev': - return self.handleDev() + @login_required + def get(self, name): + if name == 'dev': + return self.handleDev() - elif name == 'login': - return self.handleLogin() + elif name == 'login': + return self.handleLogin() - elif name == 'upload': - return self.handleUpload() + elif name == 'upload': + return self.handleUpload() - user = users.get_current_user() - prefs = UserData.get_by_key_name(user.user_id()) - if prefs is None: - return self.handleLogin() + user = users.get_current_user() + prefs = UserData.get_by_key_name(user.user_id()) + if prefs is None: + return self.handleLogin() - html = HtmlFile.get_by_key_name(name) - if html is None: - self.error(404) - return + html = HtmlFile.get_by_key_name(name) + if html is None: + self.error(404) + return - self.response.headers['Content-Type'] = 'text/html' + self.response.headers['Content-Type'] = 'text/html' - if html.compressed: - # TODO(jimhug): This slightly sucks ;-) - # Can we write directly to the response.out? - gz = gzip.GzipFile(name, 'rb', fileobj=StringIO.StringIO(html.content)) - self.response.out.write(gz.read()) - gz.close() - else: - self.response.out.write(html.content) + if html.compressed: + # TODO(jimhug): This slightly sucks ;-) + # Can we write directly to the response.out? + gz = gzip.GzipFile( + name, 'rb', fileobj=StringIO.StringIO(html.content)) + self.response.out.write(gz.read()) + gz.close() + else: + self.response.out.write(html.content) - # TODO(jimhug): Include first data packet with html. + # TODO(jimhug): Include first data packet with html. - def handleLogin(self): - user = users.get_current_user() - # TODO(jimhug): Manage secrets for dart.googleplex.com better. - # TODO(jimhug): Confirm that we need client_secret. - flow = OAuth2WebServerFlow( - client_id='267793340506.apps.googleusercontent.com', - client_secret='5m8H-zyamfTYg5vnpYu1uGMU', - scope=READER_API, - user_agent='swarm') + def handleLogin(self): + user = users.get_current_user() + # TODO(jimhug): Manage secrets for dart.googleplex.com better. + # TODO(jimhug): Confirm that we need client_secret. + flow = OAuth2WebServerFlow( + client_id='267793340506.apps.googleusercontent.com', + client_secret='5m8H-zyamfTYg5vnpYu1uGMU', + scope=READER_API, + user_agent='swarm') - callback = self.request.relative_url('/oauth2callback') - authorize_url = flow.step1_get_authorize_url(callback) + callback = self.request.relative_url('/oauth2callback') + authorize_url = flow.step1_get_authorize_url(callback) - memcache.set(user.user_id(), pickle.dumps(flow)) + memcache.set(user.user_id(), pickle.dumps(flow)) - content = template.render('login.html', {'authorize': authorize_url}) - self.response.out.write(content) + content = template.render('login.html', {'authorize': authorize_url}) + self.response.out.write(content) - def handleDev(self): - user = users.get_current_user() - content = template.render('dev.html', {'user': user}) - self.response.out.write(content) + def handleDev(self): + user = users.get_current_user() + content = template.render('dev.html', {'user': user}) + self.response.out.write(content) - def handleUpload(self): - user = users.get_current_user() - content = template.render('upload.html', {'user': user}) - self.response.out.write(content) + def handleUpload(self): + user = users.get_current_user() + content = template.render('upload.html', {'user': user}) + self.response.out.write(content) class UploadFeed(webapp.RequestHandler): - def post(self): - upload_files = self.request.POST.multi.__dict__['_items'] - version = self.request.get('version') - logging.info('files: %r' % upload_files) - for data in upload_files: - if data[0] != 'files': continue - file = data[1] - logging.info('upload feed: %r' % file.filename) - data = json.loads(file.value) + def post(self): + upload_files = self.request.POST.multi.__dict__['_items'] + version = self.request.get('version') + logging.info('files: %r' % upload_files) + for data in upload_files: + if data[0] != 'files': continue + file = data[1] + logging.info('upload feed: %r' % file.filename) - feedId = file.filename - feed = Feed.get_or_insert(feedId) + data = json.loads(file.value) - # Find the section to add it to. - sectionTitle = data['section'] - section = findSectionByTitle(sectionTitle) - if section != None: - if feed.key() in section.feeds: - logging.warn('Already contains feed %s, replacing' % feedId) - section.feeds.remove(feed.key()) + feedId = file.filename + feed = Feed.get_or_insert(feedId) - # Add the feed to the section. - section.feeds.insert(0, feed.key()) - section.put() + # Find the section to add it to. + sectionTitle = data['section'] + section = findSectionByTitle(sectionTitle) + if section != None: + if feed.key() in section.feeds: + logging.warn('Already contains feed %s, replacing' % feedId) + section.feeds.remove(feed.key()) - # Add the articles. - collectFeed(feed, data) + # Add the feed to the section. + section.feeds.insert(0, feed.key()) + section.put() - else: - logging.error('Could not find section %s to add the feed to' % - sectionTitle) + # Add the articles. + collectFeed(feed, data) + + else: + logging.error('Could not find section %s to add the feed to' % + sectionTitle) + + self.redirect('/') - self.redirect('/') # TODO(jimhug): Batch these up and request them more aggressively. class DataHandler(webapp.RequestHandler): - def get(self, name): - if name.endswith('.jpg'): - # Must be a thumbnail - key = urllib2.unquote(name[:-len('.jpg')]) - article = Article.get_by_key_name(key) - self.response.headers['Content-Type'] = 'image/jpeg' - # cache images for 10 hours - self.response.headers['Cache-Control'] = 'public,max-age=36000' - article.ensureThumbnail() - self.response.out.write(article.thumbnail) - elif name.endswith('.html'): - # Must be article content - key = urllib2.unquote(name[:-len('.html')]) - article = Article.get_by_key_name(key) - self.response.headers['Content-Type'] = 'text/html' - if article is None: - content = '

Missing article

' - else: - content = article.content - # cache article content for 10 hours - self.response.headers['Cache-Control'] = 'public,max-age=36000' - self.response.out.write(content) - elif name == 'user.data': - self.response.out.write(self.getUserData()) - elif name == 'CannedData.dart': - self.canData() - elif name == 'CannedData.zip': - self.canDataZip() - else: - self.error(404) - def getUserData(self, articleKeys=None): - user = users.get_current_user() - user_id = user.user_id() + def get(self, name): + if name.endswith('.jpg'): + # Must be a thumbnail + key = urllib2.unquote(name[:-len('.jpg')]) + article = Article.get_by_key_name(key) + self.response.headers['Content-Type'] = 'image/jpeg' + # cache images for 10 hours + self.response.headers['Cache-Control'] = 'public,max-age=36000' + article.ensureThumbnail() + self.response.out.write(article.thumbnail) + elif name.endswith('.html'): + # Must be article content + key = urllib2.unquote(name[:-len('.html')]) + article = Article.get_by_key_name(key) + self.response.headers['Content-Type'] = 'text/html' + if article is None: + content = '

Missing article

' + else: + content = article.content + # cache article content for 10 hours + self.response.headers['Cache-Control'] = 'public,max-age=36000' + self.response.out.write(content) + elif name == 'user.data': + self.response.out.write(self.getUserData()) + elif name == 'CannedData.dart': + self.canData() + elif name == 'CannedData.zip': + self.canDataZip() + else: + self.error(404) - key = 'data_' + user_id - # need to flush memcache fairly frequently... - data = memcache.get(key) - if data is None: - prefs = UserData.get_or_insert(user_id) - if prefs is None: - # TODO(jimhug): Graceful failure for unknown users. - pass - data = prefs.getEncodedData(articleKeys) - # TODO(jimhug): memcache.set(key, data) + def getUserData(self, articleKeys=None): + user = users.get_current_user() + user_id = user.user_id() - return data + key = 'data_' + user_id + # need to flush memcache fairly frequently... + data = memcache.get(key) + if data is None: + prefs = UserData.get_or_insert(user_id) + if prefs is None: + # TODO(jimhug): Graceful failure for unknown users. + pass + data = prefs.getEncodedData(articleKeys) + # TODO(jimhug): memcache.set(key, data) - def canData(self): - def makeDartSafe(data): - return repr(unicode(data))[1:].replace('$', '\\$') + return data - lines = ['// TODO(jimhug): Work out correct copyright for this file.', - 'class CannedData {'] + def canData(self): - user = users.get_current_user() - prefs = UserData.get_by_key_name(user.user_id()) - articleKeys = [] - data = prefs.getEncodedData(articleKeys) - lines.append(' static const Map data = const {') - for article in db.get(articleKeys): - key = makeDartSafe(urllib.quote(article.key().name())+'.html') - lines.append(' %s:%s, ' % (key, makeDartSafe(article.content))) + def makeDartSafe(data): + return repr(unicode(data))[1:].replace('$', '\\$') - lines.append(' "user.data":%s' % makeDartSafe(data)) + lines = [ + '// TODO(jimhug): Work out correct copyright for this file.', + 'class CannedData {' + ] - lines.append(' };') + user = users.get_current_user() + prefs = UserData.get_by_key_name(user.user_id()) + articleKeys = [] + data = prefs.getEncodedData(articleKeys) + lines.append(' static const Map data = const {') + for article in db.get(articleKeys): + key = makeDartSafe(urllib.quote(article.key().name()) + '.html') + lines.append(' %s:%s, ' % (key, makeDartSafe(article.content))) - lines.append('}') - self.response.headers['Content-Type'] = 'application/dart' - self.response.out.write('\n'.join(lines)) + lines.append(' "user.data":%s' % makeDartSafe(data)) - # Get canned static data - def canDataZip(self): - # We need to zip into an in-memory buffer to get the right string encoding - # behavior. - data = StringIO.StringIO() - result = zipfile.ZipFile(data, 'w') + lines.append(' };') - articleKeys = [] - result.writestr('data/user.data', - self.getUserData(articleKeys).encode('utf-8')) - logging.info(' adding articles %s' % len(articleKeys)) - images = [] - for article in db.get(articleKeys): - article.ensureThumbnail() - path = 'data/' + article.key().name() + '.html' - result.writestr(path.encode('utf-8'), article.content.encode('utf-8')) - if article.thumbnail: - path = 'data/' + article.key().name() + '.jpg' - result.writestr(path.encode('utf-8'), article.thumbnail) + lines.append('}') + self.response.headers['Content-Type'] = 'application/dart' + self.response.out.write('\n'.join(lines)) - result.close() - logging.info('writing CannedData.zip') - self.response.headers['Content-Type'] = 'multipart/x-zip' - disposition = 'attachment; filename=CannedData.zip' - self.response.headers['Content-Disposition'] = disposition - self.response.out.write(data.getvalue()) - data.close() + # Get canned static data + def canDataZip(self): + # We need to zip into an in-memory buffer to get the right string encoding + # behavior. + data = StringIO.StringIO() + result = zipfile.ZipFile(data, 'w') + + articleKeys = [] + result.writestr('data/user.data', + self.getUserData(articleKeys).encode('utf-8')) + logging.info(' adding articles %s' % len(articleKeys)) + images = [] + for article in db.get(articleKeys): + article.ensureThumbnail() + path = 'data/' + article.key().name() + '.html' + result.writestr( + path.encode('utf-8'), article.content.encode('utf-8')) + if article.thumbnail: + path = 'data/' + article.key().name() + '.jpg' + result.writestr(path.encode('utf-8'), article.thumbnail) + + result.close() + logging.info('writing CannedData.zip') + self.response.headers['Content-Type'] = 'multipart/x-zip' + disposition = 'attachment; filename=CannedData.zip' + self.response.headers['Content-Disposition'] = disposition + self.response.out.write(data.getvalue()) + data.close() class SetDefaultFeeds(webapp.RequestHandler): - @login_required - def get(self): - user = users.get_current_user() - prefs = UserData.get_or_insert(user.user_id()) - prefs.sections = [ - db.Key.from_path('Section', 'user/17857667084667353155/label/Top'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Design'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Eco'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Geek'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Google'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Seattle'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Tech'), - db.Key.from_path('Section', 'user/17857667084667353155/label/Web')] + @login_required + def get(self): + user = users.get_current_user() + prefs = UserData.get_or_insert(user.user_id()) - prefs.put() + prefs.sections = [ + db.Key.from_path('Section', 'user/17857667084667353155/label/Top'), + db.Key.from_path('Section', + 'user/17857667084667353155/label/Design'), + db.Key.from_path('Section', 'user/17857667084667353155/label/Eco'), + db.Key.from_path('Section', 'user/17857667084667353155/label/Geek'), + db.Key.from_path('Section', + 'user/17857667084667353155/label/Google'), + db.Key.from_path('Section', + 'user/17857667084667353155/label/Seattle'), + db.Key.from_path('Section', 'user/17857667084667353155/label/Tech'), + db.Key.from_path('Section', 'user/17857667084667353155/label/Web') + ] + + prefs.put() + + self.redirect('/') - self.redirect('/') class SetTestFeeds(webapp.RequestHandler): - @login_required - def get(self): - user = users.get_current_user() - prefs = UserData.get_or_insert(user.user_id()) - sections = [] - for i in range(3): - s1 = Section.get_or_insert('Test%d' % i) - s1.title = 'Section %d' % (i+1) + @login_required + def get(self): + user = users.get_current_user() + prefs = UserData.get_or_insert(user.user_id()) - feeds = [] - for j in range(4): - label = '%d_%d' % (i, j) - f1 = Feed.get_or_insert('Test%s' % label) - f1.title = 'Feed %s' % label - f1.iconUrl = getFeedIcon('http://google.com') - f1.lastUpdated = 0 - f1.put() - feeds.append(f1.key()) + sections = [] + for i in range(3): + s1 = Section.get_or_insert('Test%d' % i) + s1.title = 'Section %d' % (i + 1) - for k in range(8): - label = '%d_%d_%d' % (i, j, k) - a1 = Article.get_or_insert('Test%s' % label) - if a1.title is None: - a1.feed = f1 - a1.title = 'Article %s' % label - a1.author = 'anon' - a1.content = 'Lorem ipsum something or other...' - a1.snippet = 'Lorem ipsum something or other...' - a1.thumbnail = None - a1.srcurl = '' - a1.date = 0 + feeds = [] + for j in range(4): + label = '%d_%d' % (i, j) + f1 = Feed.get_or_insert('Test%s' % label) + f1.title = 'Feed %s' % label + f1.iconUrl = getFeedIcon('http://google.com') + f1.lastUpdated = 0 + f1.put() + feeds.append(f1.key()) - s1.feeds = feeds - s1.put() - sections.append(s1.key()) + for k in range(8): + label = '%d_%d_%d' % (i, j, k) + a1 = Article.get_or_insert('Test%s' % label) + if a1.title is None: + a1.feed = f1 + a1.title = 'Article %s' % label + a1.author = 'anon' + a1.content = 'Lorem ipsum something or other...' + a1.snippet = 'Lorem ipsum something or other...' + a1.thumbnail = None + a1.srcurl = '' + a1.date = 0 - prefs.sections = sections - prefs.put() + s1.feeds = feeds + s1.put() + sections.append(s1.key()) - self.redirect('/') + prefs.sections = sections + prefs.put() + + self.redirect('/') class UserLoginHandler(webapp.RequestHandler): - @login_required - def get(self): - user = users.get_current_user() - prefs = UserData.get_or_insert(user.user_id()) - if prefs.credentials: - http = prefs.credentials.authorize(httplib2.Http()) - response, content = http.request('%s/subscription/list?output=json' % - READER_API) - self.collectFeeds(prefs, content) - self.redirect('/') - else: - self.redirect('/login') + @login_required + def get(self): + user = users.get_current_user() + prefs = UserData.get_or_insert(user.user_id()) + if prefs.credentials: + http = prefs.credentials.authorize(httplib2.Http()) + response, content = http.request( + '%s/subscription/list?output=json' % READER_API) + self.collectFeeds(prefs, content) + self.redirect('/') + else: + self.redirect('/login') - def collectFeeds(self, prefs, content): - data = json.loads(content) + def collectFeeds(self, prefs, content): + data = json.loads(content) - queue_name = self.request.get('queue_name', 'priority-queue') - sections = {} - for feedData in data['subscriptions']: - feed = Feed.get_or_insert(feedData['id']) - feed.put() - category = feedData['categories'][0] - categoryId = category['id'] - if not sections.has_key(categoryId): - sections[categoryId] = (category['label'], []) + queue_name = self.request.get('queue_name', 'priority-queue') + sections = {} + for feedData in data['subscriptions']: + feed = Feed.get_or_insert(feedData['id']) + feed.put() + category = feedData['categories'][0] + categoryId = category['id'] + if not sections.has_key(categoryId): + sections[categoryId] = (category['label'], []) - # TODO(jimhug): Use Reader preferences to sort feeds in a section. - sections[categoryId][1].append(feed.key()) + # TODO(jimhug): Use Reader preferences to sort feeds in a section. + sections[categoryId][1].append(feed.key()) - # Kick off a high priority feed update - taskqueue.add(url='/update/feed', queue_name=queue_name, - params={'id': feed.key().name()}) + # Kick off a high priority feed update + taskqueue.add( + url='/update/feed', + queue_name=queue_name, + params={'id': feed.key().name()}) - sectionKeys = [] - for name, (title, feeds) in sections.items(): - section = Section.get_or_insert(name) - section.feeds = feeds - section.title = title - section.put() - # Forces Top to be the first section - if title == 'Top': title = '0Top' - sectionKeys.append( (title, section.key()) ) + sectionKeys = [] + for name, (title, feeds) in sections.items(): + section = Section.get_or_insert(name) + section.feeds = feeds + section.title = title + section.put() + # Forces Top to be the first section + if title == 'Top': title = '0Top' + sectionKeys.append((title, section.key())) - # TODO(jimhug): Use Reader preferences API to get users true sort order. - prefs.sections = [key for t, key in sorted(sectionKeys)] - prefs.put() + # TODO(jimhug): Use Reader preferences API to get users true sort order. + prefs.sections = [key for t, key in sorted(sectionKeys)] + prefs.put() class AllFeedsCollector(webapp.RequestHandler): - '''Ensures that a given feed object is locally up to date.''' - def post(self): return self.get() + '''Ensures that a given feed object is locally up to date.''' + + def post(self): + return self.get() + + def get(self): + queue_name = self.request.get('queue_name', 'background') + for feed in Feed.all(): + taskqueue.add( + url='/update/feed', + queue_name=queue_name, + params={'id': feed.key().name()}) + + +UPDATE_COUNT = 4 # The number of articles to request on periodic updates. +INITIAL_COUNT = 40 # The number of articles to get first for a new queue. +SNIPPET_SIZE = 180 # The length of plain-text snippet to extract. - def get(self): - queue_name = self.request.get('queue_name', 'background') - for feed in Feed.all(): - taskqueue.add(url='/update/feed', queue_name=queue_name, - params={'id': feed.key().name()}) -UPDATE_COUNT = 4 # The number of articles to request on periodic updates. -INITIAL_COUNT = 40 # The number of articles to get first for a new queue. -SNIPPET_SIZE = 180 # The length of plain-text snippet to extract. class FeedCollector(webapp.RequestHandler): - def post(self): return self.get() - def get(self): - feedId = self.request.get('id') - feed = Feed.get_or_insert(feedId) + def post(self): + return self.get() - if feed.lastUpdated is None: - self.fetchn(feed, feedId, INITIAL_COUNT) - else: - self.fetchn(feed, feedId, UPDATE_COUNT) + def get(self): + feedId = self.request.get('id') + feed = Feed.get_or_insert(feedId) - self.response.headers['Content-Type'] = "text/plain" + if feed.lastUpdated is None: + self.fetchn(feed, feedId, INITIAL_COUNT) + else: + self.fetchn(feed, feedId, UPDATE_COUNT) - def fetchn(self, feed, feedId, n, continuation=None): - # basic pattern is to read by ARTICLE_COUNT until we hit existing. - if continuation is None: - apiUrl = '%s/stream/contents/%s?n=%d' % ( - READER_API, feedId, n) - else: - apiUrl = '%s/stream/contents/%s?n=%d&c=%s' % ( - READER_API, feedId, n, continuation) + self.response.headers['Content-Type'] = "text/plain" - logging.info('fetching: %s' % apiUrl) - result = urlfetch.fetch(apiUrl) + def fetchn(self, feed, feedId, n, continuation=None): + # basic pattern is to read by ARTICLE_COUNT until we hit existing. + if continuation is None: + apiUrl = '%s/stream/contents/%s?n=%d' % (READER_API, feedId, n) + else: + apiUrl = '%s/stream/contents/%s?n=%d&c=%s' % (READER_API, feedId, n, + continuation) + + logging.info('fetching: %s' % apiUrl) + result = urlfetch.fetch(apiUrl) + + if result.status_code == 200: + data = json.loads(result.content) + collectFeed(feed, data, continuation) + elif result.status_code == 401: + self.response.out.write('
%s
' % result.content) + else: + self.response.out.write(result.status_code) - if result.status_code == 200: - data = json.loads(result.content) - collectFeed(feed, data, continuation) - elif result.status_code == 401: - self.response.out.write( '
%s
' % result.content) - else: - self.response.out.write(result.status_code) def findSectionByTitle(title): - for section in Section.all(): - if section.fixedTitle() == title: - return section - return None + for section in Section.all(): + if section.fixedTitle() == title: + return section + return None + def collectFeed(feed, data, continuation=None): - ''' + ''' Reads a feed from the given JSON object and populates the given feed object in the datastore with its data. ''' - if continuation is None: - if 'alternate' in data: - feed.iconUrl = getFeedIcon(data['alternate'][0]['href']) - feed.title = data['title'] - feed.lastUpdated = data['updated'] + if continuation is None: + if 'alternate' in data: + feed.iconUrl = getFeedIcon(data['alternate'][0]['href']) + feed.title = data['title'] + feed.lastUpdated = data['updated'] - articles = data['items'] - logging.info('%d new articles for %s' % (len(articles), feed.title)) + articles = data['items'] + logging.info('%d new articles for %s' % (len(articles), feed.title)) - for articleData in articles: - if not collectArticle(feed, articleData): - feed.put() - return False + for articleData in articles: + if not collectArticle(feed, articleData): + feed.put() + return False - if len(articles) > 0 and data.has_key('continuation'): - logging.info('would have looked for more articles') - # TODO(jimhug): Enable this continuation check when more robust - #self.fetchn(feed, feedId, data['continuation']) + if len(articles) > 0 and data.has_key('continuation'): + logging.info('would have looked for more articles') + # TODO(jimhug): Enable this continuation check when more robust + #self.fetchn(feed, feedId, data['continuation']) + + feed.ensureEncodedFeed(force=True) + feed.put() + return True - feed.ensureEncodedFeed(force=True) - feed.put() - return True def collectArticle(feed, data): - ''' + ''' Reads an article from the given JSON object and populates the datastore with it. ''' - if not 'title' in data: - # Skip this articles without titles + if not 'title' in data: + # Skip this articles without titles + return True + + articleId = data['id'] + article = Article.get_or_insert(articleId) + # TODO(jimhug): This aborts too early - at lease for one adafruit case. + if article.date == data['published']: + logging.info( + 'found existing, aborting: %r, %r' % (articleId, article.date)) + return False + + if data.has_key('content'): + content = data['content']['content'] + elif data.has_key('summary'): + content = data['summary']['content'] + else: + content = '' + #TODO(jimhug): better summary? + article.content = content + article.date = data['published'] + article.title = unescape(data['title']) + article.snippet = unescape(strip_tags(content)[:SNIPPET_SIZE]) + + article.feed = feed + + # TODO(jimhug): make this canonical so UX can change for this state + article.author = data.get('author', 'anonymous') + + article.ensureThumbnail() + + article.srcurl = '' + if data.has_key('alternate'): + for alt in data['alternate']: + if alt.has_key('href'): + article.srcurl = alt['href'] return True - articleId = data['id'] - article = Article.get_or_insert(articleId) - # TODO(jimhug): This aborts too early - at lease for one adafruit case. - if article.date == data['published']: - logging.info('found existing, aborting: %r, %r' % - (articleId, article.date)) - return False - - if data.has_key('content'): - content = data['content']['content'] - elif data.has_key('summary'): - content = data['summary']['content'] - else: - content = '' - #TODO(jimhug): better summary? - article.content = content - article.date = data['published'] - article.title = unescape(data['title']) - article.snippet = unescape(strip_tags(content)[:SNIPPET_SIZE]) - - article.feed = feed - - # TODO(jimhug): make this canonical so UX can change for this state - article.author = data.get('author', 'anonymous') - - article.ensureThumbnail() - - article.srcurl = '' - if data.has_key('alternate'): - for alt in data['alternate']: - if alt.has_key('href'): - article.srcurl = alt['href'] - return True def unescape(html): - "Inverse of Django's utils.html.escape function" - if not isinstance(html, basestring): - html = str(html) - html = html.replace(''', "'").replace('"', '"') - return html.replace('>', '>').replace('<', '<').replace('&', '&') + "Inverse of Django's utils.html.escape function" + if not isinstance(html, basestring): + html = str(html) + html = html.replace(''', "'").replace('"', '"') + return html.replace('>', '>').replace('<', '<').replace('&', '&') + def getFeedIcon(url): - url = urlparse.urlparse(url).netloc - return 'http://s2.googleusercontent.com/s2/favicons?domain=%s&alt=feed' % url + url = urlparse.urlparse(url).netloc + return 'http://s2.googleusercontent.com/s2/favicons?domain=%s&alt=feed' % url + def findImage(text): - img = findImgTag(text, 'jpg|jpeg|png') - if img is not None: + img = findImgTag(text, 'jpg|jpeg|png') + if img is not None: + return img + + img = findVideoTag(text) + if img is not None: + return img + + img = findImgTag(text, 'gif') return img - img = findVideoTag(text) - if img is not None: - return img - - img = findImgTag(text, 'gif') - return img def findImgTag(text, extensions): - m = re.search(r'src="(http://\S+\.(%s))(\?.*)?"' % extensions, text) - if m is None: - return None - return m.group(1) + m = re.search(r'src="(http://\S+\.(%s))(\?.*)?"' % extensions, text) + if m is None: + return None + return m.group(1) + def findVideoTag(text): - # TODO(jimhug): Add other videos beyond youtube. - m = re.search(r'src="http://www.youtube.com/(\S+)/(\S+)[/|"]', text) - if m is None: - return None + # TODO(jimhug): Add other videos beyond youtube. + m = re.search(r'src="http://www.youtube.com/(\S+)/(\S+)[/|"]', text) + if m is None: + return None + + return 'http://img.youtube.com/vi/%s/0.jpg' % m.group(2) - return 'http://img.youtube.com/vi/%s/0.jpg' % m.group(2) def makeThumbnail(text): - url = None - try: - url = findImage(text) - if url is None: - return None - return generateThumbnail(url) - except: - logging.info('error decoding: %s' % (url or text)) - return None + url = None + try: + url = findImage(text) + if url is None: + return None + return generateThumbnail(url) + except: + logging.info('error decoding: %s' % (url or text)) + return None + def generateThumbnail(url): - logging.info('generating thumbnail: %s' % url) - thumbWidth, thumbHeight = THUMB_SIZE + logging.info('generating thumbnail: %s' % url) + thumbWidth, thumbHeight = THUMB_SIZE - result = urlfetch.fetch(url) - img = images.Image(result.content) + result = urlfetch.fetch(url) + img = images.Image(result.content) - w, h = img.width, img.height + w, h = img.width, img.height - aspect = float(w) / h - thumbAspect = float(thumbWidth) / thumbHeight + aspect = float(w) / h + thumbAspect = float(thumbWidth) / thumbHeight - if aspect > thumbAspect: - # Too wide, so crop on the sides. - normalizedCrop = (w - h * thumbAspect) / (2.0 * w) - img.crop(normalizedCrop, 0., 1. - normalizedCrop, 1. ) - elif aspect < thumbAspect: - # Too tall, so crop out the bottom. - normalizedCrop = (h - w / thumbAspect) / h - img.crop(0., 0., 1., 1. - normalizedCrop) + if aspect > thumbAspect: + # Too wide, so crop on the sides. + normalizedCrop = (w - h * thumbAspect) / (2.0 * w) + img.crop(normalizedCrop, 0., 1. - normalizedCrop, 1.) + elif aspect < thumbAspect: + # Too tall, so crop out the bottom. + normalizedCrop = (h - w / thumbAspect) / h + img.crop(0., 0., 1., 1. - normalizedCrop) - img.resize(thumbWidth, thumbHeight) + img.resize(thumbWidth, thumbHeight) - # Chose JPEG encoding because informal experiments showed it generated - # the best size to quality ratio for thumbnail images. - nimg = img.execute_transforms(output_encoding=images.JPEG) - logging.info(' finished thumbnail: %s' % url) + # Chose JPEG encoding because informal experiments showed it generated + # the best size to quality ratio for thumbnail images. + nimg = img.execute_transforms(output_encoding=images.JPEG) + logging.info(' finished thumbnail: %s' % url) + + return nimg - return nimg class OAuthHandler(webapp.RequestHandler): - @login_required - def get(self): - user = users.get_current_user() - flow = pickle.loads(memcache.get(user.user_id())) - if flow: - prefs = UserData.get_or_insert(user.user_id()) - prefs.credentials = flow.step2_exchange(self.request.params) - prefs.put() - self.redirect('/update/user') - else: - pass + @login_required + def get(self): + user = users.get_current_user() + flow = pickle.loads(memcache.get(user.user_id())) + if flow: + prefs = UserData.get_or_insert(user.user_id()) + prefs.credentials = flow.step2_exchange(self.request.params) + prefs.put() + self.redirect('/update/user') + else: + pass def main(): - application = webapp.WSGIApplication( - [ - ('/data/(.*)', DataHandler), + application = webapp.WSGIApplication( + [ + ('/data/(.*)', DataHandler), - # This is called periodically from cron.yaml. - ('/update/allFeeds', AllFeedsCollector), - ('/update/feed', FeedCollector), - ('/update/user', UserLoginHandler), - ('/update/defaultFeeds', SetDefaultFeeds), - ('/update/testFeeds', SetTestFeeds), - ('/update/html', UpdateHtml), - ('/update/upload', UploadFeed), - ('/oauth2callback', OAuthHandler), + # This is called periodically from cron.yaml. + ('/update/allFeeds', AllFeedsCollector), + ('/update/feed', FeedCollector), + ('/update/user', UserLoginHandler), + ('/update/defaultFeeds', SetDefaultFeeds), + ('/update/testFeeds', SetTestFeeds), + ('/update/html', UpdateHtml), + ('/update/upload', UploadFeed), + ('/oauth2callback', OAuthHandler), + ('/', TopHandler), + ('/(.*)', MainHandler), + ], + debug=True) + webapp.util.run_wsgi_app(application) - ('/', TopHandler), - ('/(.*)', MainHandler), - ], - debug=True) - webapp.util.run_wsgi_app(application) if __name__ == '__main__': - main() + main() diff --git a/samples-dev/swarm/buildapp.py b/samples-dev/swarm/buildapp.py index d3ef028432b..12d2a6afb01 100755 --- a/samples-dev/swarm/buildapp.py +++ b/samples-dev/swarm/buildapp.py @@ -18,55 +18,54 @@ CLIENT_PATH = os.path.normpath(DART_PATH + '/client') sys.path.append(os.path.abspath(DART_PATH + '/tools')) import utils -buildRoot = CLIENT_PATH + '/' + utils.GetBuildRoot( - utils.GuessOS(), 'debug', 'dartc') +buildRoot = CLIENT_PATH + '/' + utils.GetBuildRoot(utils.GuessOS(), 'debug', + 'dartc') + def execute(*command): - ''' + ''' Executes the given command in a new process. If the command fails (returns non-zero) halts the script and returns that exit code. ''' - exitcode = subprocess.call(command) - if exitcode != 0: - sys.exit(exitcode) + exitcode = subprocess.call(command) + if exitcode != 0: + sys.exit(exitcode) + def createChromeApp(buildRoot, antTarget, resultFile): - buildDir = os.path.join(buildRoot, 'war') + buildDir = os.path.join(buildRoot, 'war') - # Use ant to create the 'war' directory - # TODO(jmesserly): we should factor out as much as possible from the ant file - # It's not really doing anything useful for us besides compiling Dart code - # with DartC and copying files. But for now, it helps us share code with - # our appengine update.py, which is good. - execute( - DART_PATH + '/third_party/apache_ant/v1_7_1/bin/ant', - '-f', 'build-appengine.xml', - '-Dbuild.dir=' + buildRoot, - antTarget) + # Use ant to create the 'war' directory + # TODO(jmesserly): we should factor out as much as possible from the ant file + # It's not really doing anything useful for us besides compiling Dart code + # with DartC and copying files. But for now, it helps us share code with + # our appengine update.py, which is good. + execute(DART_PATH + '/third_party/apache_ant/v1_7_1/bin/ant', '-f', + 'build-appengine.xml', '-Dbuild.dir=' + buildRoot, antTarget) - # Call Dartium (could be any Chrome--but we know Dartium will be there) and - # ask it to create the .crx file for us using the checked in developer key. - chrome = CLIENT_PATH + '/tests/drt/chrome' + # Call Dartium (could be any Chrome--but we know Dartium will be there) and + # ask it to create the .crx file for us using the checked in developer key. + chrome = CLIENT_PATH + '/tests/drt/chrome' - # On Mac Chrome is under a .app folder - if platform.system() == 'Darwin': - chrome = CLIENT_PATH + '/tests/drt/Chromium.app/Contents/MacOS/Chromium' + # On Mac Chrome is under a .app folder + if platform.system() == 'Darwin': + chrome = CLIENT_PATH + '/tests/drt/Chromium.app/Contents/MacOS/Chromium' - keyFile = DART_PATH + '/samples/swarm/swarm-dev.pem' - execute(chrome, '--pack-extension=' + buildDir, - '--pack-extension-key=' + keyFile) + keyFile = DART_PATH + '/samples/swarm/swarm-dev.pem' + execute(chrome, '--pack-extension=' + buildDir, + '--pack-extension-key=' + keyFile) - resultFile = os.path.join(buildRoot, resultFile) - os.rename(buildDir + '.crx', resultFile) - return os.path.abspath(resultFile) + resultFile = os.path.join(buildRoot, resultFile) + os.rename(buildDir + '.crx', resultFile) + return os.path.abspath(resultFile) def main(): - # Create a DartC and Dartium app - dartiumResult = createChromeApp(buildRoot, 'build_dart_app', 'swarm.crx') - dartCResult = createChromeApp(buildRoot, 'build_js_app', 'swarm-js.crx') + # Create a DartC and Dartium app + dartiumResult = createChromeApp(buildRoot, 'build_dart_app', 'swarm.crx') + dartCResult = createChromeApp(buildRoot, 'build_js_app', 'swarm-js.crx') - print ''' + print ''' Successfully created Chrome apps! Dartium: file://%s @@ -74,7 +73,8 @@ Successfully created Chrome apps! To install, open this URL in Chrome and select Continue at the bottom. ''' % (dartiumResult, dartCResult) - return 0 + return 0 + if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/samples-dev/swarm/cacheimages.py b/samples-dev/swarm/cacheimages.py index 68fbc87c6d5..0a8d1d31327 100755 --- a/samples-dev/swarm/cacheimages.py +++ b/samples-dev/swarm/cacheimages.py @@ -18,53 +18,60 @@ sys.path.append(CLIENT_TOOLS_PATH) import htmlconverter converter = CLIENT_TOOLS_PATH + '/htmlconverter.py' + # This has to be a top level function to use with multiprocessing def convertImgs(infile): - global options - try: - htmlconverter.convertForOffline( - infile, infile, - verbose=options.verbose, - encode_images=options.inline_images) - print 'Converted ' + infile - except BaseException, e: - print 'Caught error: %s' % e + global options + try: + htmlconverter.convertForOffline( + infile, + infile, + verbose=options.verbose, + encode_images=options.inline_images) + print 'Converted ' + infile + except BaseException, e: + print 'Caught error: %s' % e + def Flags(): - """ Constructs a parser for extracting flags from the command line. """ - parser = optparse.OptionParser() - parser.add_option("--inline_images", - help=("Encode img payloads as data:// URLs rather than local files."), - default=False, - action='store_true') - parser.add_option("--verbose", - help="Print verbose output", - default=False, - action="store_true") - return parser + """ Constructs a parser for extracting flags from the command line. """ + parser = optparse.OptionParser() + parser.add_option( + "--inline_images", + help=("Encode img payloads as data:// URLs rather than local files."), + default=False, + action='store_true') + parser.add_option( + "--verbose", + help="Print verbose output", + default=False, + action="store_true") + return parser + def main(): - global options - parser = Flags() - options, args = parser.parse_args() - print "args: %s" % args - if len(args) < 1 or 'help' in args[0]: - print 'Usage: %s DIRECTORY' % basename(sys.argv[0]) - return 1 + global options + parser = Flags() + options, args = parser.parse_args() + print "args: %s" % args + if len(args) < 1 or 'help' in args[0]: + print 'Usage: %s DIRECTORY' % basename(sys.argv[0]) + return 1 - dirname = args[0] - print 'Searching directory ' + dirname + dirname = args[0] + print 'Searching directory ' + dirname - files = [] - for root, dirs, fnames in os.walk(dirname): - for fname in fnames: - if fname.endswith('.html'): - files.append(join(root, fname)) + files = [] + for root, dirs, fnames in os.walk(dirname): + for fname in fnames: + if fname.endswith('.html'): + files.append(join(root, fname)) + + count = 4 * multiprocessing.cpu_count() + pool = multiprocessing.Pool(processes=count) + # Note: need a timeout to get keyboard interrupt due to a Python bug + pool.map_async(convertImgs, files).get(3600) # one hour - count = 4 * multiprocessing.cpu_count() - pool = multiprocessing.Pool(processes=count) - # Note: need a timeout to get keyboard interrupt due to a Python bug - pool.map_async(convertImgs, files).get(3600) # one hour if __name__ == '__main__': - main() + main() diff --git a/samples-dev/swarm/gen_manifest.py b/samples-dev/swarm/gen_manifest.py index 6ab3ef8988d..4cc1280b013 100755 --- a/samples-dev/swarm/gen_manifest.py +++ b/samples-dev/swarm/gen_manifest.py @@ -4,7 +4,6 @@ #!/usr/bin/python2.6 # - """ Usage: gen_manifest.py DIRECTORY EXTENSIONS CACHE-FILE HTML-FILES... @@ -34,17 +33,21 @@ os.chdir(cacheDir) print "Generating manifest from root path: " + cacheDir patterns = extensions + htmlFiles + + def matches(file): - for pattern in patterns: - if fnmatch.fnmatch(file, pattern): - return True - return False + for pattern in patterns: + if fnmatch.fnmatch(file, pattern): + return True + return False + def findFiles(rootDir): - for root, dirs, files in os.walk(rootDir): - for f in files: - # yields this file relative to the given directory - yield os.path.join(root, f)[(len(rootDir) + 1):] + for root, dirs, files in os.walk(rootDir): + for f in files: + # yields this file relative to the given directory + yield os.path.join(root, f)[(len(rootDir) + 1):] + manifest = [] manifest.append("CACHE MANIFEST") @@ -63,16 +66,16 @@ manifest.append("NETWORK:") manifest.append("*") with open(manifestName, 'w') as f: - f.writelines(m + '\n' for m in manifest) + f.writelines(m + '\n' for m in manifest) print "Created manifest file: " + manifestName for htmlFile in htmlFiles: - cachedHtmlFile = htmlFile.replace('.html', '-cache.html') - text = open(htmlFile, 'r').read() - text = text.replace('', '' % manifestName, 1) - with open(cachedHtmlFile, 'w') as output: - output.write(text) - print "Processed html file: %s -> %s" % (htmlFile, cachedHtmlFile) + cachedHtmlFile = htmlFile.replace('.html', '-cache.html') + text = open(htmlFile, 'r').read() + text = text.replace('', '' % manifestName, 1) + with open(cachedHtmlFile, 'w') as output: + output.write(text) + print "Processed html file: %s -> %s" % (htmlFile, cachedHtmlFile) print "Successfully generated manifest and html files" diff --git a/samples-dev/swarm/update.py b/samples-dev/swarm/update.py index f73eec6e197..33526b0d0b1 100755 --- a/samples-dev/swarm/update.py +++ b/samples-dev/swarm/update.py @@ -20,64 +20,69 @@ CLIENT_PATH = dirname(CLIENT_TOOLS_PATH) sys.path.append(CLIENT_TOOLS_PATH) import htmlconverter -def convertOne(infile, options): - outDirBase = 'outcode' - outfile = join(outDirBase, infile) - print 'converting %s to %s' % (infile, outfile) - if 'dart' in options.target: - htmlconverter.convertForDartium( - infile, - outDirBase, - outfile.replace('.html', '-dart.html'), - options.verbose) - if 'js' in options.target: - htmlconverter.convertForChromium( - infile, options.dartc_extra_flags, - outfile.replace('.html', '-js.html'), - options.verbose) +def convertOne(infile, options): + outDirBase = 'outcode' + outfile = join(outDirBase, infile) + print 'converting %s to %s' % (infile, outfile) + + if 'dart' in options.target: + htmlconverter.convertForDartium(infile, outDirBase, + outfile.replace('.html', '-dart.html'), + options.verbose) + if 'js' in options.target: + htmlconverter.convertForChromium(infile, options.dartc_extra_flags, + outfile.replace('.html', '-js.html'), + options.verbose) def Flags(): - """ Consturcts a parser for extracting flags from the command line. """ - result = optparse.OptionParser() - result.add_option("-t", "--target", - help="The target html to generate", - metavar="[js,dart]", - default='js,dart') - result.add_option("--verbose", - help="Print verbose output", - default=False, - action="store_true") - result.add_option("--dartc_extra_flags", - help="Additional flag text to pass to dartc", - default="", - action="store") - #result.set_usage("update.py input.html -o OUTDIR -t chromium,dartium") - return result + """ Consturcts a parser for extracting flags from the command line. """ + result = optparse.OptionParser() + result.add_option( + "-t", + "--target", + help="The target html to generate", + metavar="[js,dart]", + default='js,dart') + result.add_option( + "--verbose", + help="Print verbose output", + default=False, + action="store_true") + result.add_option( + "--dartc_extra_flags", + help="Additional flag text to pass to dartc", + default="", + action="store") + #result.set_usage("update.py input.html -o OUTDIR -t chromium,dartium") + return result + def getAllHtmlFiles(): - htmlFiles = [] - for filename in os.listdir(APP_PATH): - fName, fExt = os.path.splitext(filename) - if fExt.lower() == '.html': - htmlFiles.append(filename) + htmlFiles = [] + for filename in os.listdir(APP_PATH): + fName, fExt = os.path.splitext(filename) + if fExt.lower() == '.html': + htmlFiles.append(filename) + + return htmlFiles - return htmlFiles def main(): - os.chdir(CLIENT_PATH) # TODO(jimhug): I don't like chdir's in scripts... + os.chdir(CLIENT_PATH) # TODO(jimhug): I don't like chdir's in scripts... - parser = Flags() - options, args = parser.parse_args() - #if len(args) < 1 or not options.out or not options.target: - # parser.print_help() - # return 1 + parser = Flags() + options, args = parser.parse_args() + #if len(args) < 1 or not options.out or not options.target: + # parser.print_help() + # return 1 + + REL_APP_PATH = relpath(APP_PATH) + for file in getAllHtmlFiles(): + infile = join(REL_APP_PATH, file) + convertOne(infile, options) - REL_APP_PATH = relpath(APP_PATH) - for file in getAllHtmlFiles(): - infile = join(REL_APP_PATH, file) - convertOne(infile, options) if __name__ == '__main__': - main() + main() diff --git a/tools/android/download_android_tools.py b/tools/android/download_android_tools.py index e16989f3146..b264e58c8a3 100644 --- a/tools/android/download_android_tools.py +++ b/tools/android/download_android_tools.py @@ -25,80 +25,83 @@ import find_depot_tools DEPOT_PATH = find_depot_tools.add_depot_tools_to_path() GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py') -def RunCommand(command): - """Run command and return success (True) or failure.""" - print 'Running %s' % (str(command)) - if subprocess.call(command, shell=False) == 0: - return True - print 'Failed.' - return False +def RunCommand(command): + """Run command and return success (True) or failure.""" + + print 'Running %s' % (str(command)) + if subprocess.call(command, shell=False) == 0: + return True + print 'Failed.' + return False def GetInstalledVersion(version_stamp): - version_file = os.path.join(INSTALL_DIR, version_stamp) - if not os.path.exists(version_file): - return None - with open(version_file) as f: - return f.read().strip() + version_file = os.path.join(INSTALL_DIR, version_stamp) + if not os.path.exists(version_file): + return None + with open(version_file) as f: + return f.read().strip() def VersionStampName(tools_name): - if sys.platform.startswith('linux'): - return 'VERSION_LINUX_' + tools_name.upper() - elif sys.platform == 'darwin': - return 'VERSION_MACOSX_' + tools_name.upper() - else: - print('NOTE: Will not download android tools. Unsupported platform: ' + sys.platform) - sys.exit(0) + if sys.platform.startswith('linux'): + return 'VERSION_LINUX_' + tools_name.upper() + elif sys.platform == 'darwin': + return 'VERSION_MACOSX_' + tools_name.upper() + else: + print('NOTE: Will not download android tools. Unsupported platform: ' + + sys.platform) + sys.exit(0) def UpdateTools(tools_name): - """Downloads zipped tools from Google Cloud Storage and extracts them, + """Downloads zipped tools from Google Cloud Storage and extracts them, stamping current version.""" - # Read latest version. - version_stamp = VersionStampName(tools_name) - version = '' - with open(os.path.join(THIS_DIR, version_stamp)) as f: - version = f.read().strip() - # Return if installed binaries are up to date. - if version == GetInstalledVersion(version_stamp): - return + # Read latest version. + version_stamp = VersionStampName(tools_name) + version = '' + with open(os.path.join(THIS_DIR, version_stamp)) as f: + version = f.read().strip() + # Return if installed binaries are up to date. + if version == GetInstalledVersion(version_stamp): + return - # Remove the old install directory checked out from git. - if os.path.exists(os.path.join(INSTALL_DIR, '.git')): - shutil.rmtree(INSTALL_DIR) - # Make sure that the install directory exists. - if not os.path.exists(INSTALL_DIR): - os.mkdir(INSTALL_DIR) - # Remove current installation. - tools_root = os.path.join(INSTALL_DIR, tools_name) - if os.path.exists(tools_root): - shutil.rmtree(tools_root) + # Remove the old install directory checked out from git. + if os.path.exists(os.path.join(INSTALL_DIR, '.git')): + shutil.rmtree(INSTALL_DIR) + # Make sure that the install directory exists. + if not os.path.exists(INSTALL_DIR): + os.mkdir(INSTALL_DIR) + # Remove current installation. + tools_root = os.path.join(INSTALL_DIR, tools_name) + if os.path.exists(tools_root): + shutil.rmtree(tools_root) - # Download tools from GCS. - archive_path = os.path.join(INSTALL_DIR, tools_name + '.tar.gz') - download_cmd = ['python', GSUTIL_PATH, 'cp', - 'gs://mojo/android/tool/%s.tar.gz' % version, - archive_path] - if not RunCommand(download_cmd): - print ('WARNING: Failed to download Android tools.') - return + # Download tools from GCS. + archive_path = os.path.join(INSTALL_DIR, tools_name + '.tar.gz') + download_cmd = [ + 'python', GSUTIL_PATH, 'cp', + 'gs://mojo/android/tool/%s.tar.gz' % version, archive_path + ] + if not RunCommand(download_cmd): + print('WARNING: Failed to download Android tools.') + return - print "Extracting Android tools (" + tools_name + ")" - with tarfile.open(archive_path) as arch: - arch.extractall(INSTALL_DIR) - os.remove(archive_path) - # Write version as the last step. - with open(os.path.join(INSTALL_DIR, version_stamp), 'w+') as f: - f.write('%s\n' % version) + print "Extracting Android tools (" + tools_name + ")" + with tarfile.open(archive_path) as arch: + arch.extractall(INSTALL_DIR) + os.remove(archive_path) + # Write version as the last step. + with open(os.path.join(INSTALL_DIR, version_stamp), 'w+') as f: + f.write('%s\n' % version) def main(): - UpdateTools('sdk') - UpdateTools('ndk') + UpdateTools('sdk') + UpdateTools('ndk') if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/tools/archive_crash.py b/tools/archive_crash.py index ef08b343791..f266b2b9713 100755 --- a/tools/archive_crash.py +++ b/tools/archive_crash.py @@ -20,55 +20,60 @@ import uuid from glob import glob GCS_FOLDER = 'dart-temp-crash-archive' -GSUTIL='/b/build/scripts/slave/gsutil' +GSUTIL = '/b/build/scripts/slave/gsutil' + def CreateTarball(input_dir, tarname): - print 'Creating tar file: %s' % tarname - tar = tarfile.open(tarname, mode='w:gz') - tar.add(input_dir) - tar.close() + print 'Creating tar file: %s' % tarname + tar = tarfile.open(tarname, mode='w:gz') + tar.add(input_dir) + tar.close() + def CopyToGCS(filename): - gs_location = 'gs://%s/%s/' % (GCS_FOLDER, uuid.uuid4()) - cmd = [GSUTIL, 'cp', filename, gs_location] - print 'Running command: %s' % cmd - subprocess.check_call(cmd) - archived_filename = '%s%s' % (gs_location, filename.split('/').pop()) - print 'Dump now available in %s' % archived_filename + gs_location = 'gs://%s/%s/' % (GCS_FOLDER, uuid.uuid4()) + cmd = [GSUTIL, 'cp', filename, gs_location] + print 'Running command: %s' % cmd + subprocess.check_call(cmd) + archived_filename = '%s%s' % (gs_location, filename.split('/').pop()) + print 'Dump now available in %s' % archived_filename + def TEMPArchiveBuild(): - if not 'PWD' in os.environ: - return - pwd = os.environ['PWD'] - print pwd - if not 'vm-' in pwd: - return - if 'win' in pwd or 'release' in pwd: - return - files = glob('%s/out/Debug*/dart' % pwd) - files.extend(glob('%s/xcodebuild/Debug*/dart' % pwd)) - print('Archiving: %s' % files) - for f in files: - CopyToGCS(f) + if not 'PWD' in os.environ: + return + pwd = os.environ['PWD'] + print pwd + if not 'vm-' in pwd: + return + if 'win' in pwd or 'release' in pwd: + return + files = glob('%s/out/Debug*/dart' % pwd) + files.extend(glob('%s/xcodebuild/Debug*/dart' % pwd)) + print('Archiving: %s' % files) + for f in files: + CopyToGCS(f) + def Main(): - TEMPArchiveBuild() - if utils.GuessOS() != 'linux': - print 'Currently only archiving crash dumps on linux' - return 0 - print 'Looking for crash dumps' - num_dumps = 0 - for v in os.listdir('/tmp'): - if v.startswith('coredump'): - fullpath = '/tmp/%s' % v - if os.path.isdir(fullpath): - num_dumps += 1 - tarname = '%s.tar.gz' % fullpath - CreateTarball(fullpath, tarname) - CopyToGCS(tarname) - os.unlink(tarname) - shutil.rmtree(fullpath) - print 'Found %s core dumps' % num_dumps + TEMPArchiveBuild() + if utils.GuessOS() != 'linux': + print 'Currently only archiving crash dumps on linux' + return 0 + print 'Looking for crash dumps' + num_dumps = 0 + for v in os.listdir('/tmp'): + if v.startswith('coredump'): + fullpath = '/tmp/%s' % v + if os.path.isdir(fullpath): + num_dumps += 1 + tarname = '%s.tar.gz' % fullpath + CreateTarball(fullpath, tarname) + CopyToGCS(tarname) + os.unlink(tarname) + shutil.rmtree(fullpath) + print 'Found %s core dumps' % num_dumps + if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/tools/bots/__init__.py b/tools/bots/__init__.py index 3479377bd9a..91994d14a0c 100644 --- a/tools/bots/__init__.py +++ b/tools/bots/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - diff --git a/tools/bots/android.py b/tools/bots/android.py index 21b69780834..9ee7d813731 100644 --- a/tools/bots/android.py +++ b/tools/bots/android.py @@ -3,7 +3,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Android buildbot steps. """ @@ -17,47 +16,50 @@ import bot ANDROID_BUILDER = r'vm-android-(linux|mac|win)' + def AndroidConfig(name, is_buildbot): - """Returns info for the current buildbot based on the name of the builder. + """Returns info for the current buildbot based on the name of the builder. Currently, this is just: - mode: always "release" (for now) - system: "linux", "mac", or "win" """ - android_pattern = re.match(ANDROID_BUILDER, name) - if not android_pattern: - return None + android_pattern = re.match(ANDROID_BUILDER, name) + if not android_pattern: + return None - system = android_pattern.group(1) - if system == 'win': system = 'windows' + system = android_pattern.group(1) + if system == 'win': system = 'windows' - return bot.BuildInfo('none', 'vm', 'release', system, checked=True) + return bot.BuildInfo('none', 'vm', 'release', system, checked=True) def AndroidSteps(build_info): - # TODO(efortuna): Here's where we'll run tests. - #bot.RunTest('android', build_info, ['android']) - pass + # TODO(efortuna): Here's where we'll run tests. + #bot.RunTest('android', build_info, ['android']) + pass + def BuildAndroid(build_info): - """ + """ Builds the android target. - build_info: the buildInfo object, containing information about what sort of build and test to be run. """ - with bot.BuildStep('Build Android'): - # TODO(vsm): A temporary hack until we figure out why incremental builds are - # broken on Android. - if os.path.exists('./out/lastHooksTargetOS.txt'): - os.remove('./out/lastHooksTargetOS.txt') - targets = ['runtime'] - args = [sys.executable, './tools/build.py', - '--arch=' + build_info.arch, - '--mode=' + build_info.mode, - '--os=android'] + targets - print 'Building Android: %s' % (' '.join(args)) - bot.RunProcess(args) + with bot.BuildStep('Build Android'): + # TODO(vsm): A temporary hack until we figure out why incremental builds are + # broken on Android. + if os.path.exists('./out/lastHooksTargetOS.txt'): + os.remove('./out/lastHooksTargetOS.txt') + targets = ['runtime'] + args = [ + sys.executable, './tools/build.py', '--arch=' + build_info.arch, + '--mode=' + build_info.mode, '--os=android' + ] + targets + print 'Building Android: %s' % (' '.join(args)) + bot.RunProcess(args) + if __name__ == '__main__': - bot.RunBot(AndroidConfig, AndroidSteps, build_step=BuildAndroid) + bot.RunBot(AndroidConfig, AndroidSteps, build_step=BuildAndroid) diff --git a/tools/bots/bot.py b/tools/bots/bot.py index ff8edc30a10..e3712398942 100644 --- a/tools/bots/bot.py +++ b/tools/bots/bot.py @@ -3,7 +3,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Shared code for use in the buildbot scripts. """ @@ -26,7 +25,7 @@ BUILDER_CLOBBER = 'BUILDBOT_CLOBBER' class BuildInfo(object): - """ + """ Encapsulation of build information. - compiler: None or 'dart2js' @@ -47,47 +46,61 @@ class BuildInfo(object): - builder_tag: A tag indicating a special builder setup. - cps_ir: Run the compiler with the cps based backend """ - def __init__(self, compiler, runtime, mode, system, checked=False, - host_checked=False, minified=False, shard_index=None, - total_shards=None, is_buildbot=False, test_set=None, - csp=None, arch=None, dart2js_full=False, builder_tag=None, - batch=False, cps_ir=False): - self.compiler = compiler - self.runtime = runtime - self.mode = mode - self.system = system - self.checked = checked - self.host_checked = host_checked - self.minified = minified - self.shard_index = shard_index - self.total_shards = total_shards - self.is_buildbot = is_buildbot - self.test_set = test_set - self.csp = csp - self.dart2js_full = dart2js_full - self.builder_tag = builder_tag - self.batch = batch - self.cps_ir = cps_ir - if (arch == None): - self.arch = 'ia32' - else: - self.arch = arch - def PrintBuildInfo(self): - shard_description = "" - if self.shard_index: - shard_description = " shard %s of %s" % (self.shard_index, - self.total_shards) - print ("compiler: %s, runtime: %s mode: %s, system: %s," - " checked: %s, host-checked: %s, minified: %s, test-set: %s" - " arch: %s%s" - ) % (self.compiler, self.runtime, self.mode, self.system, - self.checked, self.host_checked, self.minified, self.test_set, - self.arch, shard_description) + def __init__(self, + compiler, + runtime, + mode, + system, + checked=False, + host_checked=False, + minified=False, + shard_index=None, + total_shards=None, + is_buildbot=False, + test_set=None, + csp=None, + arch=None, + dart2js_full=False, + builder_tag=None, + batch=False, + cps_ir=False): + self.compiler = compiler + self.runtime = runtime + self.mode = mode + self.system = system + self.checked = checked + self.host_checked = host_checked + self.minified = minified + self.shard_index = shard_index + self.total_shards = total_shards + self.is_buildbot = is_buildbot + self.test_set = test_set + self.csp = csp + self.dart2js_full = dart2js_full + self.builder_tag = builder_tag + self.batch = batch + self.cps_ir = cps_ir + if (arch == None): + self.arch = 'ia32' + else: + self.arch = arch + + def PrintBuildInfo(self): + shard_description = "" + if self.shard_index: + shard_description = " shard %s of %s" % (self.shard_index, + self.total_shards) + print("compiler: %s, runtime: %s mode: %s, system: %s," + " checked: %s, host-checked: %s, minified: %s, test-set: %s" + " arch: %s%s") % (self.compiler, self.runtime, self.mode, + self.system, self.checked, self.host_checked, + self.minified, self.test_set, self.arch, + shard_description) class BuildStep(object): - """ + """ A context manager for handling build steps. When the context manager is entered, it prints the "@@@BUILD_STEP __@@@" @@ -97,38 +110,41 @@ class BuildStep(object): If swallow_error is True, then this will catch and discard any OSError that is thrown. This lets you run later BuildSteps if the current one fails. """ - def __init__(self, name, swallow_error=False): - self.name = name - self.swallow_error = swallow_error - def __enter__(self): - print '@@@BUILD_STEP %s@@@' % self.name - sys.stdout.flush() + def __init__(self, name, swallow_error=False): + self.name = name + self.swallow_error = swallow_error - def __exit__(self, type, value, traceback): - if value: - print '@@@STEP_FAILURE@@@' - sys.stdout.flush() - if self.swallow_error and isinstance(value, OSError): - return True + def __enter__(self): + print '@@@BUILD_STEP %s@@@' % self.name + sys.stdout.flush() + + def __exit__(self, type, value, traceback): + if value: + print '@@@STEP_FAILURE@@@' + sys.stdout.flush() + if self.swallow_error and isinstance(value, OSError): + return True def BuildSDK(build_info): - """ + """ Builds the SDK. - build_info: the buildInfo object, containing information about what sort of build and test to be run. """ - with BuildStep('Build SDK'): - args = [sys.executable, './tools/build.py', '--mode=' + build_info.mode, - '--arch=' + build_info.arch, 'create_sdk'] - print 'Building SDK: %s' % (' '.join(args)) - RunProcess(args) + with BuildStep('Build SDK'): + args = [ + sys.executable, './tools/build.py', '--mode=' + build_info.mode, + '--arch=' + build_info.arch, 'create_sdk' + ] + print 'Building SDK: %s' % (' '.join(args)) + RunProcess(args) def RunBot(parse_name, custom_steps, build_step=BuildSDK): - """ + """ The main function for running a buildbot. A buildbot script should invoke this once. The parse_name function will be @@ -141,155 +157,156 @@ def RunBot(parse_name, custom_steps, build_step=BuildSDK): This function will not return. It will call sys.exit() with an appropriate exit code. """ - if len(sys.argv) == 0: - print 'Script pathname not known, giving up.' - sys.exit(1) + if len(sys.argv) == 0: + print 'Script pathname not known, giving up.' + sys.exit(1) - name, is_buildbot = GetBotName() - build_info = parse_name(name, is_buildbot) - if not build_info: - print 'Could not handle unfamiliar bot name "%s".' % name - sys.exit(1) + name, is_buildbot = GetBotName() + build_info = parse_name(name, is_buildbot) + if not build_info: + print 'Could not handle unfamiliar bot name "%s".' % name + sys.exit(1) - # Print out the buildinfo for easy debugging. - build_info.PrintBuildInfo() + # Print out the buildinfo for easy debugging. + build_info.PrintBuildInfo() - # Make sure we are in the dart directory - os.chdir(bot_utils.DART_DIR) + # Make sure we are in the dart directory + os.chdir(bot_utils.DART_DIR) - try: - Clobber() - if build_step: - build_step(build_info) + try: + Clobber() + if build_step: + build_step(build_info) - custom_steps(build_info) - except OSError as e: - sys.exit(e.errno) + custom_steps(build_info) + except OSError as e: + sys.exit(e.errno) - sys.exit(0) + sys.exit(0) def GetBotName(): - """ + """ Gets the name of the current buildbot. Returns a tuple of the buildbot name and a flag to indicate if we are actually a buildbot (True), or just a user pretending to be one (False). """ - # For testing the bot locally, allow the user to pass in a buildbot name. - parser = optparse.OptionParser() - parser.add_option('-n', '--name', dest='name', help='The name of the build' - 'bot you would like to emulate (ex: vm-mac-debug)', default=None) - args, _ = parser.parse_args() + # For testing the bot locally, allow the user to pass in a buildbot name. + parser = optparse.OptionParser() + parser.add_option( + '-n', + '--name', + dest='name', + help='The name of the build' + 'bot you would like to emulate (ex: vm-mac-debug)', + default=None) + args, _ = parser.parse_args() - if args.name: - return args.name, False + if args.name: + return args.name, False - name = os.environ.get(BUILDER_NAME) - if not name: - print 'Use -n $BUILDBOT_NAME for the bot you would like to emulate.' - sys.exit(1) + name = os.environ.get(BUILDER_NAME) + if not name: + print 'Use -n $BUILDBOT_NAME for the bot you would like to emulate.' + sys.exit(1) - return name, True + return name, True def Clobber(force=None): - """ + """ Clobbers the builder before we do the build, if appropriate. - mode: either 'debug' or 'release' """ - if os.environ.get(BUILDER_CLOBBER) != "1" and not force: - return - clobber_string = 'Clobber' - if force: - clobber_string = 'Clobber(always)' + if os.environ.get(BUILDER_CLOBBER) != "1" and not force: + return + clobber_string = 'Clobber' + if force: + clobber_string = 'Clobber(always)' - with BuildStep(clobber_string): - cmd = [sys.executable, - './tools/clean_output_directory.py'] - print 'Clobbering %s' % (' '.join(cmd)) - RunProcess(cmd) + with BuildStep(clobber_string): + cmd = [sys.executable, './tools/clean_output_directory.py'] + print 'Clobbering %s' % (' '.join(cmd)) + RunProcess(cmd) def RunTest(name, build_info, targets, flags=None, swallow_error=False): - """ + """ Runs test.py with the given settings. """ - if not flags: - flags = [] + if not flags: + flags = [] - step_name = GetStepName(name, flags) - with BuildStep(step_name, swallow_error=swallow_error): - sys.stdout.flush() + step_name = GetStepName(name, flags) + with BuildStep(step_name, swallow_error=swallow_error): + sys.stdout.flush() - cmd = [ - sys.executable, os.path.join(os.curdir, 'tools', 'test.py'), - '--step_name=' + step_name, - '--mode=' + build_info.mode, - '--compiler=' + build_info.compiler, - '--runtime=' + build_info.runtime, - '--arch=' + build_info.arch, - '--progress=buildbot', - '--write-result-log', - '-v', '--time', '--use-sdk', '--report' - ] + cmd = [ + sys.executable, + os.path.join(os.curdir, 'tools', + 'test.py'), '--step_name=' + step_name, + '--mode=' + build_info.mode, '--compiler=' + build_info.compiler, + '--runtime=' + build_info.runtime, '--arch=' + build_info.arch, + '--progress=buildbot', '--write-result-log', '-v', '--time', + '--use-sdk', '--report' + ] - if build_info.checked: - cmd.append('--checked') + if build_info.checked: + cmd.append('--checked') - cmd.extend(flags) - cmd.extend(targets) + cmd.extend(flags) + cmd.extend(targets) - print 'Running: %s' % (' '.join(map(lambda arg: '"%s"' % arg, cmd))) - sys.stdout.flush() - RunProcess(cmd) + print 'Running: %s' % (' '.join(map(lambda arg: '"%s"' % arg, cmd))) + sys.stdout.flush() + RunProcess(cmd) def RunTestRunner(build_info, path): - """ + """ Runs the test package's runner on the package at 'path'. """ - sdk_bin = os.path.join( - bot_utils.DART_DIR, - utils.GetBuildSdkBin(BUILD_OS, build_info.mode, build_info.arch)) + sdk_bin = os.path.join( + bot_utils.DART_DIR, + utils.GetBuildSdkBin(BUILD_OS, build_info.mode, build_info.arch)) - build_root = utils.GetBuildRoot( - BUILD_OS, build_info.mode, build_info.arch) + build_root = utils.GetBuildRoot(BUILD_OS, build_info.mode, build_info.arch) - dart_name = 'dart.exe' if build_info.system == 'windows' else 'dart' - dart_bin = os.path.join(sdk_bin, dart_name) + dart_name = 'dart.exe' if build_info.system == 'windows' else 'dart' + dart_bin = os.path.join(sdk_bin, dart_name) - test_bin = os.path.abspath( - os.path.join('third_party', 'pkg', 'test', 'bin', 'test.dart')) + test_bin = os.path.abspath( + os.path.join('third_party', 'pkg', 'test', 'bin', 'test.dart')) - with utils.ChangedWorkingDirectory(path): - args = [dart_bin, test_bin, '--reporter', 'expanded', '--no-color'] - print("Running %s" % ' '.join(args)) - RunProcess(args) + with utils.ChangedWorkingDirectory(path): + args = [dart_bin, test_bin, '--reporter', 'expanded', '--no-color'] + print("Running %s" % ' '.join(args)) + RunProcess(args) def RunProcess(command, env=None): - """ + """ Runs command. If a non-zero exit code is returned, raises an OSError with errno as the exit code. """ - if env is None: - no_color_env = dict(os.environ) - else: - no_color_env = env - no_color_env['TERM'] = 'nocolor' + if env is None: + no_color_env = dict(os.environ) + else: + no_color_env = env + no_color_env['TERM'] = 'nocolor' - exit_code = subprocess.call(command, env=no_color_env) - if exit_code != 0: - raise OSError(exit_code) + exit_code = subprocess.call(command, env=no_color_env) + if exit_code != 0: + raise OSError(exit_code) def GetStepName(name, flags): - """ + """ Filters out flags with '=' as this breaks the /stats feature of the buildbot. """ - flags = [x for x in flags if not '=' in x] - return ('%s tests %s' % (name, ' '.join(flags))).strip() + flags = [x for x in flags if not '=' in x] + return ('%s tests %s' % (name, ' '.join(flags))).strip() diff --git a/tools/bots/bot_utils.py b/tools/bots/bot_utils.py index 4a8b23a417b..fc840f33aa6 100755 --- a/tools/bots/bot_utils.py +++ b/tools/bots/bot_utils.py @@ -15,53 +15,57 @@ import sys DART_DIR = os.path.abspath( os.path.normpath(os.path.join(__file__, '..', '..', '..'))) + def GetUtils(): - '''Dynamically load the tools/utils.py python module.''' - return imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py')) + '''Dynamically load the tools/utils.py python module.''' + return imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py')) + SYSTEM_RENAMES = { - 'win32': 'windows', - 'windows': 'windows', - 'win': 'windows', - - 'linux': 'linux', - 'linux2': 'linux', - 'lucid32': 'linux', - 'lucid64': 'linux', - - 'darwin': 'macos', - 'mac': 'macos', - 'macos': 'macos', + 'win32': 'windows', + 'windows': 'windows', + 'win': 'windows', + 'linux': 'linux', + 'linux2': 'linux', + 'lucid32': 'linux', + 'lucid64': 'linux', + 'darwin': 'macos', + 'mac': 'macos', + 'macos': 'macos', } ARCH_RENAMES = { - 'ia32': 'ia32', - 'x64': 'x64', - 'arm': 'arm', - 'arm64': 'arm64', + 'ia32': 'ia32', + 'x64': 'x64', + 'arm': 'arm', + 'arm64': 'arm64', } + class Channel(object): - BLEEDING_EDGE = 'be' - DEV = 'dev' - STABLE = 'stable' - TRY = 'try' - INTEGRATION = 'integration' - ALL_CHANNELS = [BLEEDING_EDGE, DEV, STABLE, TRY, INTEGRATION] + BLEEDING_EDGE = 'be' + DEV = 'dev' + STABLE = 'stable' + TRY = 'try' + INTEGRATION = 'integration' + ALL_CHANNELS = [BLEEDING_EDGE, DEV, STABLE, TRY, INTEGRATION] + class ReleaseType(object): - RAW = 'raw' - SIGNED = 'signed' - RELEASE = 'release' - ALL_TYPES = [RAW, SIGNED, RELEASE] + RAW = 'raw' + SIGNED = 'signed' + RELEASE = 'release' + ALL_TYPES = [RAW, SIGNED, RELEASE] + class Mode(object): - RELEASE = 'release' - DEBUG = 'debug' - ALL_MODES = [RELEASE, DEBUG] + RELEASE = 'release' + DEBUG = 'debug' + ALL_MODES = [RELEASE, DEBUG] + class GCSNamer(object): - """ + """ This class is used for naming objects in our "gs://dart-archive/" GoogleCloudStorage bucket. It's structure is as follows: @@ -80,281 +84,316 @@ class GCSNamer(object): - /editor-eclipse-update /{index.html,features/,plugins/,artifacts.jar,content.jar} """ - def __init__(self, channel=Channel.BLEEDING_EDGE, - release_type=ReleaseType.RAW, internal=False): - assert channel in Channel.ALL_CHANNELS - assert release_type in ReleaseType.ALL_TYPES - self.channel = channel - self.release_type = release_type - if internal: - self.bucket = 'gs://dart-archive-internal' - else: - self.bucket = 'gs://dart-archive' + def __init__(self, + channel=Channel.BLEEDING_EDGE, + release_type=ReleaseType.RAW, + internal=False): + assert channel in Channel.ALL_CHANNELS + assert release_type in ReleaseType.ALL_TYPES - # Functions for quering complete gs:// filepaths + self.channel = channel + self.release_type = release_type + if internal: + self.bucket = 'gs://dart-archive-internal' + else: + self.bucket = 'gs://dart-archive' - def version_filepath(self, revision): - return '%s/channels/%s/%s/%s/VERSION' % (self.bucket, self.channel, - self.release_type, revision) + # Functions for quering complete gs:// filepaths - def editor_zipfilepath(self, revision, system, arch): - return '/'.join([self.editor_directory(revision), - self.editor_zipfilename(system, arch)]) + def version_filepath(self, revision): + return '%s/channels/%s/%s/%s/VERSION' % (self.bucket, self.channel, + self.release_type, revision) - def editor_installer_filepath(self, revision, system, arch, extension): - return '/'.join([self.editor_directory(revision), - self.editor_installer_filename(system, arch, extension)]) + def editor_zipfilepath(self, revision, system, arch): + return '/'.join([ + self.editor_directory(revision), + self.editor_zipfilename(system, arch) + ]) - def editor_android_zipfilepath(self, revision): - return '/'.join([self.editor_directory(revision), - self.editor_android_zipfilename()]) + def editor_installer_filepath(self, revision, system, arch, extension): + return '/'.join([ + self.editor_directory(revision), + self.editor_installer_filename(system, arch, extension) + ]) - def sdk_zipfilepath(self, revision, system, arch, mode): - return '/'.join([self.sdk_directory(revision), - self.sdk_zipfilename(system, arch, mode)]) + def editor_android_zipfilepath(self, revision): + return '/'.join([ + self.editor_directory(revision), + self.editor_android_zipfilename() + ]) - def unstripped_filepath(self, revision, system, arch): - return '/'.join([self._variant_directory('unstripped', revision), - system, - arch, - self.unstripped_filename(system)]) + def sdk_zipfilepath(self, revision, system, arch, mode): + return '/'.join([ + self.sdk_directory(revision), + self.sdk_zipfilename(system, arch, mode) + ]) - def apidocs_zipfilepath(self, revision): - return '/'.join([self.apidocs_directory(revision), - self.dartdocs_zipfilename()]) + def unstripped_filepath(self, revision, system, arch): + return '/'.join([ + self._variant_directory('unstripped', revision), system, arch, + self.unstripped_filename(system) + ]) - # Functions for querying gs:// directories + def apidocs_zipfilepath(self, revision): + return '/'.join( + [self.apidocs_directory(revision), + self.dartdocs_zipfilename()]) - def sdk_directory(self, revision): - return self._variant_directory('sdk', revision) + # Functions for querying gs:// directories - def linux_packages_directory(self, revision): - return '/'.join([self._variant_directory('linux_packages', revision)]) + def sdk_directory(self, revision): + return self._variant_directory('sdk', revision) - def src_directory(self, revision): - return self._variant_directory('src', revision) + def linux_packages_directory(self, revision): + return '/'.join([self._variant_directory('linux_packages', revision)]) - def editor_directory(self, revision): - return self._variant_directory('editor', revision) + def src_directory(self, revision): + return self._variant_directory('src', revision) - def editor_eclipse_update_directory(self, revision): - return self._variant_directory('editor-eclipse-update', revision) + def editor_directory(self, revision): + return self._variant_directory('editor', revision) - def apidocs_directory(self, revision): - return self._variant_directory('api-docs', revision) + def editor_eclipse_update_directory(self, revision): + return self._variant_directory('editor-eclipse-update', revision) - def misc_directory(self, revision): - return self._variant_directory('misc', revision) + def apidocs_directory(self, revision): + return self._variant_directory('api-docs', revision) - def _variant_directory(self, name, revision): - return '%s/channels/%s/%s/%s/%s' % (self.bucket, self.channel, - self.release_type, revision, name) + def misc_directory(self, revision): + return self._variant_directory('misc', revision) - # Functions for quering filenames + def _variant_directory(self, name, revision): + return '%s/channels/%s/%s/%s/%s' % (self.bucket, self.channel, + self.release_type, revision, name) - def dartdocs_zipfilename(self): - return 'dartdocs-gen-api.zip' + # Functions for quering filenames - def editor_zipfilename(self, system, arch): - return 'darteditor-%s-%s.zip' % ( - SYSTEM_RENAMES[system], ARCH_RENAMES[arch]) + def dartdocs_zipfilename(self): + return 'dartdocs-gen-api.zip' - def editor_android_zipfilename(self): - return 'android.zip' + def editor_zipfilename(self, system, arch): + return 'darteditor-%s-%s.zip' % (SYSTEM_RENAMES[system], + ARCH_RENAMES[arch]) - def editor_installer_filename(self, system, arch, extension): - assert extension in ['dmg', 'msi'] - return 'darteditor-installer-%s-%s.%s' % ( - SYSTEM_RENAMES[system], ARCH_RENAMES[arch], extension) + def editor_android_zipfilename(self): + return 'android.zip' - def sdk_zipfilename(self, system, arch, mode): - assert mode in Mode.ALL_MODES - return 'dartsdk-%s-%s-%s.zip' % ( - SYSTEM_RENAMES[system], ARCH_RENAMES[arch], mode) + def editor_installer_filename(self, system, arch, extension): + assert extension in ['dmg', 'msi'] + return 'darteditor-installer-%s-%s.%s' % (SYSTEM_RENAMES[system], + ARCH_RENAMES[arch], extension) + + def sdk_zipfilename(self, system, arch, mode): + assert mode in Mode.ALL_MODES + return 'dartsdk-%s-%s-%s.zip' % (SYSTEM_RENAMES[system], + ARCH_RENAMES[arch], mode) + + def unstripped_filename(self, system): + return 'dart.exe' if system.startswith('win') else 'dart' - def unstripped_filename(self, system): - return 'dart.exe' if system.startswith('win') else 'dart' class GCSNamerApiDocs(object): - def __init__(self, channel=Channel.BLEEDING_EDGE): - assert channel in Channel.ALL_CHANNELS - self.channel = channel - self.bucket = 'gs://dartlang-api-docs' + def __init__(self, channel=Channel.BLEEDING_EDGE): + assert channel in Channel.ALL_CHANNELS - def dartdocs_dirpath(self, revision): - assert len('%s' % revision) > 0 - if self.channel == Channel.BLEEDING_EDGE: - return '%s/gen-dartdocs/builds/%s' % (self.bucket, revision) - return '%s/gen-dartdocs/%s/%s' % (self.bucket, self.channel, revision) + self.channel = channel + self.bucket = 'gs://dartlang-api-docs' + + def dartdocs_dirpath(self, revision): + assert len('%s' % revision) > 0 + if self.channel == Channel.BLEEDING_EDGE: + return '%s/gen-dartdocs/builds/%s' % (self.bucket, revision) + return '%s/gen-dartdocs/%s/%s' % (self.bucket, self.channel, revision) + + def docs_latestpath(self, revision): + assert len('%s' % revision) > 0 + return '%s/channels/%s/latest.txt' % (self.bucket, self.channel) - def docs_latestpath(self, revision): - assert len('%s' % revision) > 0 - return '%s/channels/%s/latest.txt' % (self.bucket, self.channel) def run(command, env=None, shell=False, throw_on_error=True): - print "Running command: ", command + print "Running command: ", command + + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + shell=shell) + (stdout, stderr) = p.communicate() + if throw_on_error and p.returncode != 0: + print >> sys.stderr, "Failed to execute '%s'. Exit code: %s." % ( + command, p.returncode) + print >> sys.stderr, "stdout: ", stdout + print >> sys.stderr, "stderr: ", stderr + raise Exception("Failed to execute %s." % command) + return (stdout, stderr, p.returncode) - p = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env, shell=shell) - (stdout, stderr) = p.communicate() - if throw_on_error and p.returncode != 0: - print >> sys.stderr, "Failed to execute '%s'. Exit code: %s." % ( - command, p.returncode) - print >> sys.stderr, "stdout: ", stdout - print >> sys.stderr, "stderr: ", stderr - raise Exception("Failed to execute %s." % command) - return (stdout, stderr, p.returncode) class GSUtil(object): - GSUTIL_IS_SHELL_SCRIPT = False - GSUTIL_PATH = None - USE_DART_REPO_VERSION = False + GSUTIL_IS_SHELL_SCRIPT = False + GSUTIL_PATH = None + USE_DART_REPO_VERSION = False - def _layzCalculateGSUtilPath(self): - if not GSUtil.GSUTIL_PATH: - buildbot_gsutil = '/b/build/scripts/slave/gsutil' - if platform.system() == 'Windows': - buildbot_gsutil = 'e:\\\\b\\build\\scripts\\slave\\gsutil' - if os.path.isfile(buildbot_gsutil) and not GSUtil.USE_DART_REPO_VERSION: - GSUtil.GSUTIL_IS_SHELL_SCRIPT = True - GSUtil.GSUTIL_PATH = buildbot_gsutil - else: - dart_gsutil = os.path.join(DART_DIR, 'third_party', 'gsutil', 'gsutil') - if os.path.isfile(dart_gsutil): - GSUtil.GSUTIL_IS_SHELL_SCRIPT = False - GSUtil.GSUTIL_PATH = dart_gsutil - elif GSUtil.USE_DART_REPO_VERSION: - raise Exception("Dart repository version of gsutil required, " - "but not found.") + def _layzCalculateGSUtilPath(self): + if not GSUtil.GSUTIL_PATH: + buildbot_gsutil = '/b/build/scripts/slave/gsutil' + if platform.system() == 'Windows': + buildbot_gsutil = 'e:\\\\b\\build\\scripts\\slave\\gsutil' + if os.path.isfile( + buildbot_gsutil) and not GSUtil.USE_DART_REPO_VERSION: + GSUtil.GSUTIL_IS_SHELL_SCRIPT = True + GSUtil.GSUTIL_PATH = buildbot_gsutil + else: + dart_gsutil = os.path.join(DART_DIR, 'third_party', 'gsutil', + 'gsutil') + if os.path.isfile(dart_gsutil): + GSUtil.GSUTIL_IS_SHELL_SCRIPT = False + GSUtil.GSUTIL_PATH = dart_gsutil + elif GSUtil.USE_DART_REPO_VERSION: + raise Exception( + "Dart repository version of gsutil required, " + "but not found.") + else: + # We did not find gsutil, look in path + possible_locations = list(os.environ['PATH'].split( + os.pathsep)) + for directory in possible_locations: + location = os.path.join(directory, 'gsutil') + if os.path.isfile(location): + GSUtil.GSUTIL_IS_SHELL_SCRIPT = False + GSUtil.GSUTIL_PATH = location + break + assert GSUtil.GSUTIL_PATH + + def execute(self, gsutil_args): + self._layzCalculateGSUtilPath() + + if GSUtil.GSUTIL_IS_SHELL_SCRIPT: + gsutil_command = [GSUtil.GSUTIL_PATH] else: - # We did not find gsutil, look in path - possible_locations = list(os.environ['PATH'].split(os.pathsep)) - for directory in possible_locations: - location = os.path.join(directory, 'gsutil') - if os.path.isfile(location): - GSUtil.GSUTIL_IS_SHELL_SCRIPT = False - GSUtil.GSUTIL_PATH = location - break - assert GSUtil.GSUTIL_PATH + gsutil_command = [sys.executable, GSUtil.GSUTIL_PATH] - def execute(self, gsutil_args): - self._layzCalculateGSUtilPath() + return run( + gsutil_command + gsutil_args, + shell=(GSUtil.GSUTIL_IS_SHELL_SCRIPT and sys.platform == 'win32')) - if GSUtil.GSUTIL_IS_SHELL_SCRIPT: - gsutil_command = [GSUtil.GSUTIL_PATH] - else: - gsutil_command = [sys.executable, GSUtil.GSUTIL_PATH] + def upload(self, + local_path, + remote_path, + recursive=False, + public=False, + multithread=False): + assert remote_path.startswith('gs://') - return run(gsutil_command + gsutil_args, - shell=(GSUtil.GSUTIL_IS_SHELL_SCRIPT and - sys.platform == 'win32')) + if multithread: + args = ['-m', 'cp'] + else: + args = ['cp'] + if public: + args += ['-a', 'public-read'] + if recursive: + args += ['-R'] + args += [local_path, remote_path] + self.execute(args) - def upload(self, local_path, remote_path, recursive=False, - public=False, multithread=False): - assert remote_path.startswith('gs://') + def cat(self, remote_path): + assert remote_path.startswith('gs://') - if multithread: - args = ['-m', 'cp'] - else: - args = ['cp'] - if public: - args += ['-a', 'public-read'] - if recursive: - args += ['-R'] - args += [local_path, remote_path] - self.execute(args) + args = ['cat', remote_path] + (stdout, _, _) = self.execute(args) + return stdout - def cat(self, remote_path): - assert remote_path.startswith('gs://') + def setGroupReadACL(self, remote_path, group): + args = ['acl', 'ch', '-g', '%s:R' % group, remote_path] + self.execute(args) - args = ['cat', remote_path] - (stdout, _, _) = self.execute(args) - return stdout + def setContentType(self, remote_path, content_type): + args = ['setmeta', '-h', 'Content-Type:%s' % content_type, remote_path] + self.execute(args) - def setGroupReadACL(self, remote_path, group): - args = ['acl', 'ch', '-g', '%s:R' % group, remote_path] - self.execute(args) + def remove(self, remote_path, recursive=False): + assert remote_path.startswith('gs://') - def setContentType(self, remote_path, content_type): - args = ['setmeta', '-h', 'Content-Type:%s' % content_type, remote_path] - self.execute(args) + args = ['rm'] + if recursive: + args += ['-R'] + args += [remote_path] + self.execute(args) - def remove(self, remote_path, recursive=False): - assert remote_path.startswith('gs://') - - args = ['rm'] - if recursive: - args += ['-R'] - args += [remote_path] - self.execute(args) def CalculateMD5Checksum(filename): - """Calculate the MD5 checksum for filename.""" + """Calculate the MD5 checksum for filename.""" - md5 = hashlib.md5() + md5 = hashlib.md5() - with open(filename, 'rb') as f: - data = f.read(65536) - while len(data) > 0: - md5.update(data) - data = f.read(65536) + with open(filename, 'rb') as f: + data = f.read(65536) + while len(data) > 0: + md5.update(data) + data = f.read(65536) + + return md5.hexdigest() - return md5.hexdigest() def CalculateSha256Checksum(filename): - """Calculate the sha256 checksum for filename.""" + """Calculate the sha256 checksum for filename.""" - sha = hashlib.sha256() + sha = hashlib.sha256() - with open(filename, 'rb') as f: - data = f.read(65536) - while len(data) > 0: - sha.update(data) - data = f.read(65536) + with open(filename, 'rb') as f: + data = f.read(65536) + while len(data) > 0: + sha.update(data) + data = f.read(65536) + + return sha.hexdigest() - return sha.hexdigest() def CreateMD5ChecksumFile(filename, mangled_filename=None): - """Create and upload an MD5 checksum file for filename.""" - if not mangled_filename: - mangled_filename = os.path.basename(filename) + """Create and upload an MD5 checksum file for filename.""" + if not mangled_filename: + mangled_filename = os.path.basename(filename) - checksum = CalculateMD5Checksum(filename) - checksum_filename = '%s.md5sum' % filename + checksum = CalculateMD5Checksum(filename) + checksum_filename = '%s.md5sum' % filename - with open(checksum_filename, 'w') as f: - f.write('%s *%s' % (checksum, mangled_filename)) + with open(checksum_filename, 'w') as f: + f.write('%s *%s' % (checksum, mangled_filename)) + + print "MD5 checksum of %s is %s" % (filename, checksum) + return checksum_filename - print "MD5 checksum of %s is %s" % (filename, checksum) - return checksum_filename def CreateSha256ChecksumFile(filename, mangled_filename=None): - """Create and upload an sha256 checksum file for filename.""" - if not mangled_filename: - mangled_filename = os.path.basename(filename) + """Create and upload an sha256 checksum file for filename.""" + if not mangled_filename: + mangled_filename = os.path.basename(filename) - checksum = CalculateSha256Checksum(filename) - checksum_filename = '%s.sha256sum' % filename + checksum = CalculateSha256Checksum(filename) + checksum_filename = '%s.sha256sum' % filename - with open(checksum_filename, 'w') as f: - f.write('%s *%s' % (checksum, mangled_filename)) + with open(checksum_filename, 'w') as f: + f.write('%s *%s' % (checksum, mangled_filename)) + + print "SHA256 checksum of %s is %s" % (filename, checksum) + return checksum_filename - print "SHA256 checksum of %s is %s" % (filename, checksum) - return checksum_filename def GetChannelFromName(name): - """Get the channel from the name. Bleeding edge builders don't + """Get the channel from the name. Bleeding edge builders don't have a suffix.""" - channel_name = string.split(name, '-').pop() - if channel_name in Channel.ALL_CHANNELS: - return channel_name - return Channel.BLEEDING_EDGE + channel_name = string.split(name, '-').pop() + if channel_name in Channel.ALL_CHANNELS: + return channel_name + return Channel.BLEEDING_EDGE + def GetSystemFromName(name): - """Get the system from the name.""" - for part in string.split(name, '-'): - if part in SYSTEM_RENAMES: return SYSTEM_RENAMES[part] + """Get the system from the name.""" + for part in string.split(name, '-'): + if part in SYSTEM_RENAMES: return SYSTEM_RENAMES[part] - raise ValueError("Bot name '{}' does not have a system name in it.".format(name)) + raise ValueError( + "Bot name '{}' does not have a system name in it.".format(name)) diff --git a/tools/bots/dart2js_dump_info.py b/tools/bots/dart2js_dump_info.py index eb863858b8b..e6e7115427f 100644 --- a/tools/bots/dart2js_dump_info.py +++ b/tools/bots/dart2js_dump_info.py @@ -3,7 +3,6 @@ # Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Buildbot steps for testing dart2js with --dump-info turned on """ @@ -16,69 +15,68 @@ import bot_utils utils = bot_utils.GetUtils() HOST_OS = utils.GuessOS() + def DumpConfig(name, is_buildbot): - """Returns info for the current buildbot. + """Returns info for the current buildbot. We only run this bot on linux, so all of this is just hard coded. """ - return bot.BuildInfo('none', 'none', 'release', 'linux') + return bot.BuildInfo('none', 'none', 'release', 'linux') + def Run(args): - print "Running: %s" % ' '.join(args) - sys.stdout.flush() - bot.RunProcess(args) + print "Running: %s" % ' '.join(args) + sys.stdout.flush() + bot.RunProcess(args) + def DumpSteps(build_info): - build_root = utils.GetBuildRoot(HOST_OS, mode='release', arch='ia32') - compilations_dir = os.path.join(bot_utils.DART_DIR, - build_root, - 'generated_compilations') - tests = ['html', 'samples'] + build_root = utils.GetBuildRoot(HOST_OS, mode='release', arch='ia32') + compilations_dir = os.path.join(bot_utils.DART_DIR, build_root, + 'generated_compilations') + tests = ['html', 'samples'] - with bot.BuildStep('Cleaning out old compilations'): - print "Cleaning out %s" % compilations_dir - shutil.rmtree(compilations_dir, ignore_errors=True) + with bot.BuildStep('Cleaning out old compilations'): + print "Cleaning out %s" % compilations_dir + shutil.rmtree(compilations_dir, ignore_errors=True) - with utils.TempDir() as temp_dir: - normal_compilations = os.path.join(temp_dir, 'normal') - dump_compilations = os.path.join(temp_dir, 'dump') - normal_compilation_command = [sys.executable, - './tools/test.py', - '--arch=ia32', - '--mode=%s' % build_info.mode, - '-cdart2js', - '-rnone', - '--time', - '--use-sdk', - '--report', - '--progress=buildbot', - '-v' - ] + tests - with bot.BuildStep('Compiling without dump info'): - Run(normal_compilation_command) - pass + with utils.TempDir() as temp_dir: + normal_compilations = os.path.join(temp_dir, 'normal') + dump_compilations = os.path.join(temp_dir, 'dump') + normal_compilation_command = [ + sys.executable, './tools/test.py', '--arch=ia32', + '--mode=%s' % build_info.mode, '-cdart2js', '-rnone', '--time', + '--use-sdk', '--report', '--progress=buildbot', '-v' + ] + tests + with bot.BuildStep('Compiling without dump info'): + Run(normal_compilation_command) + pass - with bot.BuildStep('Store normal compilation artifacts'): - args = ['mv', compilations_dir, normal_compilations] - Run(args) + with bot.BuildStep('Store normal compilation artifacts'): + args = ['mv', compilations_dir, normal_compilations] + Run(args) - with bot.BuildStep('Compiling with dump info'): - args = normal_compilation_command + ['--dart2js-options=--dump-info'] - Run(args) + with bot.BuildStep('Compiling with dump info'): + args = normal_compilation_command + [ + '--dart2js-options=--dump-info' + ] + Run(args) - with bot.BuildStep('Store normal compilation artifacts'): - args = ['mv', compilations_dir, dump_compilations] - Run(args) + with bot.BuildStep('Store normal compilation artifacts'): + args = ['mv', compilations_dir, dump_compilations] + Run(args) - with bot.BuildStep('Compare outputs'): - args = ['diff', '-rq', '-x', '*\.json', - normal_compilations, dump_compilations] - # Diff will return non zero and we will throw if there are any differences - Run(args) + with bot.BuildStep('Compare outputs'): + args = [ + 'diff', '-rq', '-x', '*\.json', normal_compilations, + dump_compilations + ] + # Diff will return non zero and we will throw if there are any differences + Run(args) + + with bot.BuildStep('Validate dump files'): + # Do whatever you like :-), files are in dump_compilations + pass - with bot.BuildStep('Validate dump files'): - # Do whatever you like :-), files are in dump_compilations - pass if __name__ == '__main__': - bot.RunBot(DumpConfig, DumpSteps) - + bot.RunBot(DumpConfig, DumpSteps) diff --git a/tools/bots/dart_sdk.py b/tools/bots/dart_sdk.py index 48564d8a861..ac919883d57 100755 --- a/tools/bots/dart_sdk.py +++ b/tools/bots/dart_sdk.py @@ -21,214 +21,247 @@ BUILD_ARCHITECTURE = utils.GuessArchitecture() (bot_name, _) = bot.GetBotName() CHANNEL = bot_utils.GetChannelFromName(bot_name) + def BuildArchitectures(): - if BUILD_OS == 'linux': - return ['ia32', 'x64', 'arm', 'arm64'] - else: - return ['ia32', 'x64'] + if BUILD_OS == 'linux': + return ['ia32', 'x64', 'arm', 'arm64'] + else: + return ['ia32', 'x64'] + def BuildRootPath(path, arch=BUILD_ARCHITECTURE, build_mode='release'): - return os.path.join(bot_utils.DART_DIR, - utils.GetBuildRoot(BUILD_OS, build_mode, arch), path) + return os.path.join(bot_utils.DART_DIR, + utils.GetBuildRoot(BUILD_OS, build_mode, arch), path) + def BuildDartdocAPIDocs(dirname): - dart_sdk = BuildRootPath('dart-sdk') - dart_exe = os.path.join(dart_sdk, 'bin', 'dart') - dartdoc_dart = os.path.join(bot_utils.DART_DIR, - 'third_party', 'pkg', 'dartdoc', 'bin', - 'dartdoc.dart') - footer_file = os.path.join(bot_utils.DART_DIR, - 'tools', 'bots', 'dartdoc_footer.html') - url = 'https://api.dartlang.org/stable' - with bot.BuildStep('Build API docs by dartdoc'): - bot_utils.run([dart_exe, dartdoc_dart, - '--sdk-docs', '--output', dirname, '--footer', footer_file, - '--rel-canonical-prefix=' + url]) + dart_sdk = BuildRootPath('dart-sdk') + dart_exe = os.path.join(dart_sdk, 'bin', 'dart') + dartdoc_dart = os.path.join(bot_utils.DART_DIR, 'third_party', 'pkg', + 'dartdoc', 'bin', 'dartdoc.dart') + footer_file = os.path.join(bot_utils.DART_DIR, 'tools', 'bots', + 'dartdoc_footer.html') + url = 'https://api.dartlang.org/stable' + with bot.BuildStep('Build API docs by dartdoc'): + bot_utils.run([ + dart_exe, dartdoc_dart, '--sdk-docs', '--output', dirname, + '--footer', footer_file, '--rel-canonical-prefix=' + url + ]) + def CreateUploadVersionFile(): - file_path = BuildRootPath('VERSION') - with open(file_path, 'w') as fd: - fd.write(utils.GetVersionFileContent()) - DartArchiveUploadVersionFile(file_path) + file_path = BuildRootPath('VERSION') + with open(file_path, 'w') as fd: + fd.write(utils.GetVersionFileContent()) + DartArchiveUploadVersionFile(file_path) + def DartArchiveUploadVersionFile(version_file): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - revision = utils.GetArchiveVersion() - for revision in [revision, 'latest']: - destination = namer.version_filepath(revision) - DartArchiveFile(version_file, destination, checksum_files=False) + namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) + revision = utils.GetArchiveVersion() + for revision in [revision, 'latest']: + destination = namer.version_filepath(revision) + DartArchiveFile(version_file, destination, checksum_files=False) + def CreateAndUploadSDKZip(arch, sdk_path): - sdk_zip = BuildRootPath('dartsdk-%s-%s.zip' % (BUILD_OS, arch), arch=arch) - FileDelete(sdk_zip) - CreateZip(sdk_path, sdk_zip) - DartArchiveUploadSDKs(BUILD_OS, arch, sdk_zip) + sdk_zip = BuildRootPath('dartsdk-%s-%s.zip' % (BUILD_OS, arch), arch=arch) + FileDelete(sdk_zip) + CreateZip(sdk_path, sdk_zip) + DartArchiveUploadSDKs(BUILD_OS, arch, sdk_zip) + def CopyAotBinaries(arch, sdk_path): - product_sdk_path = BuildRootPath('dart-sdk', arch=arch, build_mode='product') - # We don't support precompilation on ia32. - if arch != 'ia32': - with bot.BuildStep('Patching in PRODUCT built AOT binaries'): - CopyBetween(product_sdk_path, sdk_path, 'bin', 'utils', GuessExtension('gen_snapshot')) - CopyBetween(product_sdk_path, sdk_path, 'bin', GuessExtension('dartaotruntime')) + product_sdk_path = BuildRootPath( + 'dart-sdk', arch=arch, build_mode='product') + # We don't support precompilation on ia32. + if arch != 'ia32': + with bot.BuildStep('Patching in PRODUCT built AOT binaries'): + CopyBetween(product_sdk_path, sdk_path, 'bin', 'utils', + GuessExtension('gen_snapshot')) + CopyBetween(product_sdk_path, sdk_path, 'bin', + GuessExtension('dartaotruntime')) + def DartArchiveUploadSDKs(system, arch, sdk_zip): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - git_number = utils.GetArchiveVersion() - git_hash = 'hash/%s' % utils.GetGitRevision() - for revision in [git_number, git_hash, 'latest']: - path = namer.sdk_zipfilepath(revision, system, arch, 'release') - DartArchiveFile(sdk_zip, path, checksum_files=True) + namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) + git_number = utils.GetArchiveVersion() + git_hash = 'hash/%s' % utils.GetGitRevision() + for revision in [git_number, git_hash, 'latest']: + path = namer.sdk_zipfilepath(revision, system, arch, 'release') + DartArchiveFile(sdk_zip, path, checksum_files=True) + def DartArchiveUnstrippedBinaries(): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - revision = utils.GetArchiveVersion() - binary = namer.unstripped_filename(BUILD_OS) - for arch in BuildArchitectures(): - binary = BuildRootPath(binary, arch=arch) - gs_path = namer.unstripped_filepath(revision, BUILD_OS, arch) - DartArchiveFile(binary, gs_path) + namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) + revision = utils.GetArchiveVersion() + binary = namer.unstripped_filename(BUILD_OS) + for arch in BuildArchitectures(): + binary = BuildRootPath(binary, arch=arch) + gs_path = namer.unstripped_filepath(revision, BUILD_OS, arch) + DartArchiveFile(binary, gs_path) + def CreateUploadAPIDocs(): - dartdoc_dir = BuildRootPath('gen-dartdocs') - dartdoc_zip = BuildRootPath('dartdocs-api.zip') - if CHANNEL == bot_utils.Channel.TRY: - BuildDartdocAPIDocs(dartdoc_dir) - else: - UploadApiLatestFile() - BuildDartdocAPIDocs(dartdoc_dir) - UploadDartdocApiDocs(dartdoc_dir) - CreateZip(dartdoc_dir, dartdoc_zip) - DartArchiveUploadDartdocAPIDocs(dartdoc_zip) + dartdoc_dir = BuildRootPath('gen-dartdocs') + dartdoc_zip = BuildRootPath('dartdocs-api.zip') + if CHANNEL == bot_utils.Channel.TRY: + BuildDartdocAPIDocs(dartdoc_dir) + else: + UploadApiLatestFile() + BuildDartdocAPIDocs(dartdoc_dir) + UploadDartdocApiDocs(dartdoc_dir) + CreateZip(dartdoc_dir, dartdoc_zip) + DartArchiveUploadDartdocAPIDocs(dartdoc_zip) + def DartArchiveUploadDartdocAPIDocs(api_zip): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - revision = utils.GetArchiveVersion() - for revision in [revision, 'latest']: - destination = (namer.apidocs_directory(revision) + '/' + - namer.dartdocs_zipfilename()) - DartArchiveFile(api_zip, destination, checksum_files=False) + namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) + revision = utils.GetArchiveVersion() + for revision in [revision, 'latest']: + destination = (namer.apidocs_directory(revision) + '/' + + namer.dartdocs_zipfilename()) + DartArchiveFile(api_zip, destination, checksum_files=False) + def UploadDartdocApiDocs(dir_name): - apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL) - revision = utils.GetArchiveVersion() - dartdocs_destination_gcsdir = apidocs_namer.dartdocs_dirpath(revision) + apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL) + revision = utils.GetArchiveVersion() + dartdocs_destination_gcsdir = apidocs_namer.dartdocs_dirpath(revision) - # Return early if the documents have already been uploaded. - # This can happen if a build was forced, or a commit had no changes in the - # dart repository (e.g. DEPS file update). - if GsutilExists(dartdocs_destination_gcsdir): - print ("Not uploading api docs, since %s is already present." - % dartdocs_destination_gcsdir) - return + # Return early if the documents have already been uploaded. + # This can happen if a build was forced, or a commit had no changes in the + # dart repository (e.g. DEPS file update). + if GsutilExists(dartdocs_destination_gcsdir): + print("Not uploading api docs, since %s is already present." % + dartdocs_destination_gcsdir) + return + + # Upload everything inside the built apidocs directory. + gsutil = bot_utils.GSUtil() + gsutil.upload( + dir_name, + dartdocs_destination_gcsdir, + recursive=True, + public=True, + multithread=True) - # Upload everything inside the built apidocs directory. - gsutil = bot_utils.GSUtil() - gsutil.upload(dir_name, dartdocs_destination_gcsdir, recursive=True, - public=True, multithread=True) def UploadApiLatestFile(): - apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL) - revision = utils.GetArchiveVersion() - apidocs_destination_latestfile = apidocs_namer.docs_latestpath(revision) - # Update latest.txt to contain the newest revision. - with utils.TempDir('latest_file') as temp_dir: - latest_file = os.path.join(temp_dir, 'latest.txt') - with open(latest_file, 'w') as f: - f.write('%s' % revision) - DartArchiveFile(latest_file, apidocs_destination_latestfile) + apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL) + revision = utils.GetArchiveVersion() + apidocs_destination_latestfile = apidocs_namer.docs_latestpath(revision) + # Update latest.txt to contain the newest revision. + with utils.TempDir('latest_file') as temp_dir: + latest_file = os.path.join(temp_dir, 'latest.txt') + with open(latest_file, 'w') as f: + f.write('%s' % revision) + DartArchiveFile(latest_file, apidocs_destination_latestfile) + def GsutilExists(gsu_path): - # This is a little hackish, but it is basically a one off doing very - # specialized check that we don't use elsewhere. - gsutilTool = os.path.join(bot_utils.DART_DIR, - 'third_party', 'gsutil', 'gsutil') - (_, stderr, returncode) = bot_utils.run( - [gsutilTool, 'ls', gsu_path], - throw_on_error=False) - # If the returncode is nonzero and we can find a specific error message, - # we know there are no objects with a prefix of [gsu_path]. - missing = (returncode and 'CommandException: One or more URLs matched no objects.' in stderr) - # Either the returncode has to be zero or the object must be missing, - # otherwise throw an exception. - if not missing and returncode: - raise Exception("Failed to determine whether %s exists" % gsu_path) - return not missing + # This is a little hackish, but it is basically a one off doing very + # specialized check that we don't use elsewhere. + gsutilTool = os.path.join(bot_utils.DART_DIR, 'third_party', 'gsutil', + 'gsutil') + (_, stderr, returncode) = bot_utils.run([gsutilTool, 'ls', gsu_path], + throw_on_error=False) + # If the returncode is nonzero and we can find a specific error message, + # we know there are no objects with a prefix of [gsu_path]. + missing = ( + returncode and + 'CommandException: One or more URLs matched no objects.' in stderr) + # Either the returncode has to be zero or the object must be missing, + # otherwise throw an exception. + if not missing and returncode: + raise Exception("Failed to determine whether %s exists" % gsu_path) + return not missing def CreateZip(directory, target_file): - if 'win' in BUILD_OS: - CreateZipWindows(directory, target_file) - else: - CreateZipPosix(directory, target_file) + if 'win' in BUILD_OS: + CreateZipWindows(directory, target_file) + else: + CreateZipPosix(directory, target_file) + def CreateZipPosix(directory, target_file): - with utils.ChangedWorkingDirectory(os.path.dirname(directory)): - command = ['zip', '-yrq9', target_file, os.path.basename(directory)] - Run(command) + with utils.ChangedWorkingDirectory(os.path.dirname(directory)): + command = ['zip', '-yrq9', target_file, os.path.basename(directory)] + Run(command) + def CreateZipWindows(directory, target_file): - with utils.ChangedWorkingDirectory(os.path.dirname(directory)): - zip_win = os.path.join(bot_utils.DART_DIR, 'third_party', '7zip', '7za') - command = [zip_win, 'a', '-tzip', target_file, os.path.basename(directory)] - Run(command) + with utils.ChangedWorkingDirectory(os.path.dirname(directory)): + zip_win = os.path.join(bot_utils.DART_DIR, 'third_party', '7zip', '7za') + command = [ + zip_win, 'a', '-tzip', target_file, + os.path.basename(directory) + ] + Run(command) + def FileDelete(f): - if os.path.exists(f): - os.remove(f) + if os.path.exists(f): + os.remove(f) + def CopyBetween(src_path, dst_path, *relatives): - try: - os.makedirs(os.path.join(dst_path, *relatives[:-1])) - except OSError: - # This is fine. - pass - shutil.copy2( - os.path.join(src_path, *relatives), - os.path.join(dst_path, *relatives[:-1])) + try: + os.makedirs(os.path.join(dst_path, *relatives[:-1])) + except OSError: + # This is fine. + pass + shutil.copy2( + os.path.join(src_path, *relatives), + os.path.join(dst_path, *relatives[:-1])) + def GuessExtension(binary): - if 'win' in BUILD_OS: - return binary + '.exe' - return binary + if 'win' in BUILD_OS: + return binary + '.exe' + return binary + def DartArchiveFile(local_path, remote_path, checksum_files=False): - gsutil = bot_utils.GSUtil() - gsutil.upload(local_path, remote_path, public=True) - if checksum_files: - # 'local_path' may have a different filename than 'remote_path'. So we need - # to make sure the *.md5sum file contains the correct name. - assert '/' in remote_path and not remote_path.endswith('/') + gsutil = bot_utils.GSUtil() + gsutil.upload(local_path, remote_path, public=True) + if checksum_files: + # 'local_path' may have a different filename than 'remote_path'. So we need + # to make sure the *.md5sum file contains the correct name. + assert '/' in remote_path and not remote_path.endswith('/') + + mangled_filename = remote_path[remote_path.rfind('/') + 1:] + local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path, + mangled_filename) + gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True) + local_sha256 = bot_utils.CreateSha256ChecksumFile( + local_path, mangled_filename) + gsutil.upload(local_sha256, remote_path + '.sha256sum', public=True) - mangled_filename = remote_path[remote_path.rfind('/') + 1:] - local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path, - mangled_filename) - gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True) - local_sha256 = bot_utils.CreateSha256ChecksumFile(local_path, - mangled_filename) - gsutil.upload(local_sha256, remote_path + '.sha256sum', public=True) def Run(command, env=None): - print "Running %s" % ' '.join(command) - print "Environment %s" % env - return bot.RunProcess(command, env=env) + print "Running %s" % ' '.join(command) + print "Environment %s" % env + return bot.RunProcess(command, env=env) + if __name__ == '__main__': - if len(sys.argv) > 1 and sys.argv[1] == 'api_docs': - if BUILD_OS == 'linux': - CreateUploadAPIDocs() - elif CHANNEL != bot_utils.Channel.TRY: - for arch in BuildArchitectures(): - sdk_path = BuildRootPath('dart-sdk', arch=arch) - # Patch in all the PRODUCT built AOT binaries. - CopyAotBinaries(arch, sdk_path) - with bot.BuildStep('Create and upload sdk zip for ' + arch): - CreateAndUploadSDKZip(arch, sdk_path) - DartArchiveUnstrippedBinaries() - if BUILD_OS == 'linux': - CreateUploadVersionFile() - else: # CHANNEL == bot_utils.Channel.TRY - # Patch in all the PRODUCT built AOT binaries. - for arch in BuildArchitectures(): - sdk_path = BuildRootPath('dart-sdk', arch=arch) - CopyAotBinaries(arch, sdk_path) + if len(sys.argv) > 1 and sys.argv[1] == 'api_docs': + if BUILD_OS == 'linux': + CreateUploadAPIDocs() + elif CHANNEL != bot_utils.Channel.TRY: + for arch in BuildArchitectures(): + sdk_path = BuildRootPath('dart-sdk', arch=arch) + # Patch in all the PRODUCT built AOT binaries. + CopyAotBinaries(arch, sdk_path) + with bot.BuildStep('Create and upload sdk zip for ' + arch): + CreateAndUploadSDKZip(arch, sdk_path) + DartArchiveUnstrippedBinaries() + if BUILD_OS == 'linux': + CreateUploadVersionFile() + else: # CHANNEL == bot_utils.Channel.TRY + # Patch in all the PRODUCT built AOT binaries. + for arch in BuildArchitectures(): + sdk_path = BuildRootPath('dart-sdk', arch=arch) + CopyAotBinaries(arch, sdk_path) diff --git a/tools/bots/ddc_tests.py b/tools/bots/ddc_tests.py index e00602ba197..ad7a131ab85 100644 --- a/tools/bots/ddc_tests.py +++ b/tools/bots/ddc_tests.py @@ -13,30 +13,33 @@ import subprocess import bot import bot_utils -TARGETS = [ - 'language_2', - 'corelib_2', - 'lib_2' -] +TARGETS = ['language_2', 'corelib_2', 'lib_2'] -FLAGS = [ - '--strong' -] +FLAGS = ['--strong'] if __name__ == '__main__': - with bot.BuildStep('Build SDK and dartdevc test packages'): - bot.RunProcess([sys.executable, './tools/build.py', '--mode=release', - '--arch=x64', 'dartdevc_test']) + with bot.BuildStep('Build SDK and dartdevc test packages'): + bot.RunProcess([ + sys.executable, './tools/build.py', '--mode=release', '--arch=x64', + 'dartdevc_test' + ]) - with bot.BuildStep('Run tests'): - (bot_name, _) = bot.GetBotName() - system = bot_utils.GetSystemFromName(bot_name) - if system == 'linux': - bot.RunProcess([ - 'xvfb-run', sys.executable, './tools/test.py', '--strong', '-mrelease', - '-cdartdevc', '-rchrome', '-ax64', '--report', '--time', '--checked', - '--progress=buildbot', '--write-result-log'] + TARGETS ) - else: - info = bot.BuildInfo('dartdevc', 'chrome', 'release', system, - arch='x64', checked=True) - bot.RunTest('dartdevc', info, TARGETS, flags=FLAGS) + with bot.BuildStep('Run tests'): + (bot_name, _) = bot.GetBotName() + system = bot_utils.GetSystemFromName(bot_name) + if system == 'linux': + bot.RunProcess([ + 'xvfb-run', sys.executable, './tools/test.py', '--strong', + '-mrelease', '-cdartdevc', '-rchrome', '-ax64', '--report', + '--time', '--checked', '--progress=buildbot', + '--write-result-log' + ] + TARGETS) + else: + info = bot.BuildInfo( + 'dartdevc', + 'chrome', + 'release', + system, + arch='x64', + checked=True) + bot.RunTest('dartdevc', info, TARGETS, flags=FLAGS) diff --git a/tools/bots/linux_distribution_support.py b/tools/bots/linux_distribution_support.py index 28c26c7124d..9f2bb043067 100644 --- a/tools/bots/linux_distribution_support.py +++ b/tools/bots/linux_distribution_support.py @@ -3,7 +3,6 @@ # Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Buildbot steps for src tarball generation and debian package generation @@ -23,127 +22,138 @@ utils = bot_utils.GetUtils() HOST_OS = utils.GuessOS() SRC_BUILDER = r'debianpackage-linux' + def SrcConfig(name, is_buildbot): - """Returns info for the current buildbot based on the name of the builder. + """Returns info for the current buildbot based on the name of the builder. Currently, since we only run this on linux, this is just: - mode: always "release" - system: always "linux" """ - src_pattern = re.match(SRC_BUILDER, name) - if not src_pattern: - return None - return bot.BuildInfo('none', 'none', 'release', 'linux') + src_pattern = re.match(SRC_BUILDER, name) + if not src_pattern: + return None + return bot.BuildInfo('none', 'none', 'release', 'linux') + def InstallFromDep(builddir): - for entry in os.listdir(builddir): - if entry.endswith("_amd64.deb"): - path = os.path.join(builddir, entry) - Run(['dpkg', '-i', path]) + for entry in os.listdir(builddir): + if entry.endswith("_amd64.deb"): + path = os.path.join(builddir, entry) + Run(['dpkg', '-i', path]) + def UninstallDart(): - Run(['dpkg', '-r', 'dart']) + Run(['dpkg', '-r', 'dart']) + def CreateDartTestFile(tempdir): - filename = os.path.join(tempdir, 'test.dart') - with open(filename, 'w') as f: - f.write('import "dart:collection";\n\n') - f.write('void main() {\n') - f.write(' print("Hello world");\n') - f.write('}') - return filename + filename = os.path.join(tempdir, 'test.dart') + with open(filename, 'w') as f: + f.write('import "dart:collection";\n\n') + f.write('void main() {\n') + f.write(' print("Hello world");\n') + f.write('}') + return filename + def Run(args): - print "Running: %s" % ' '.join(args) - sys.stdout.flush() - bot.RunProcess(args) + print "Running: %s" % ' '.join(args) + sys.stdout.flush() + bot.RunProcess(args) + def TestInstallation(assume_installed=True): - paths = ['/usr/bin/dart'] - for tool in ['dart2js', 'pub', 'dart', 'dartanalyzer']: - paths.append(os.path.join('/usr/lib/dart/bin', tool)) - for path in paths: - if os.path.exists(path): - if not assume_installed: - print 'Assumed not installed, found %s' % path - sys.exit(1) - else: - if assume_installed: - print 'Assumed installed, but could not find %s' % path - sys.exit(1) + paths = ['/usr/bin/dart'] + for tool in ['dart2js', 'pub', 'dart', 'dartanalyzer']: + paths.append(os.path.join('/usr/lib/dart/bin', tool)) + for path in paths: + if os.path.exists(path): + if not assume_installed: + print 'Assumed not installed, found %s' % path + sys.exit(1) + else: + if assume_installed: + print 'Assumed installed, but could not find %s' % path + sys.exit(1) + def SrcSteps(build_info): - # We always clobber the bot, to not leave old tarballs and packages - # floating around the out dir. - bot.Clobber(force=True) + # We always clobber the bot, to not leave old tarballs and packages + # floating around the out dir. + bot.Clobber(force=True) - version = utils.GetVersion() - builddir = os.path.join(bot_utils.DART_DIR, - utils.GetBuildDir(HOST_OS), - 'src_and_installation') + version = utils.GetVersion() + builddir = os.path.join(bot_utils.DART_DIR, utils.GetBuildDir(HOST_OS), + 'src_and_installation') - if not os.path.exists(builddir): - os.makedirs(builddir) - tarfilename = 'dart-%s.tar.gz' % version - tarfile = os.path.join(builddir, tarfilename) + if not os.path.exists(builddir): + os.makedirs(builddir) + tarfilename = 'dart-%s.tar.gz' % version + tarfile = os.path.join(builddir, tarfilename) - with bot.BuildStep('Validating linux system'): - print 'Validating that we are on debian jessie' - args = ['cat', '/etc/os-release'] - (stdout, stderr, exitcode) = bot_utils.run(args) - if exitcode != 0: - print "Could not find linux system, exiting" - sys.exit(1) - if not "jessie" in stdout: - print "Trying to build debian bits but not on debian Jessie" - print "You can't fix this, please contact whesse@" - sys.exit(1) + with bot.BuildStep('Validating linux system'): + print 'Validating that we are on debian jessie' + args = ['cat', '/etc/os-release'] + (stdout, stderr, exitcode) = bot_utils.run(args) + if exitcode != 0: + print "Could not find linux system, exiting" + sys.exit(1) + if not "jessie" in stdout: + print "Trying to build debian bits but not on debian Jessie" + print "You can't fix this, please contact whesse@" + sys.exit(1) - with bot.BuildStep('Create src tarball'): - print 'Building src tarball' - Run([sys.executable, './tools/create_tarball.py', - '--tar_filename', tarfile]) + with bot.BuildStep('Create src tarball'): + print 'Building src tarball' + Run([ + sys.executable, './tools/create_tarball.py', '--tar_filename', + tarfile + ]) - print 'Building Debian packages' - Run([sys.executable, './tools/create_debian_packages.py', - '--tar_filename', tarfile, - '--out_dir', builddir]) + print 'Building Debian packages' + Run([ + sys.executable, './tools/create_debian_packages.py', + '--tar_filename', tarfile, '--out_dir', builddir + ]) - with bot.BuildStep('Sanity check installation'): - if os.path.exists('/usr/bin/dart') or os.path.exists( - '/usr/lib/dart/bin/dart2js'): - print "Dart already installed, removing" - UninstallDart() - TestInstallation(assume_installed=False) + with bot.BuildStep('Sanity check installation'): + if os.path.exists('/usr/bin/dart') or os.path.exists( + '/usr/lib/dart/bin/dart2js'): + print "Dart already installed, removing" + UninstallDart() + TestInstallation(assume_installed=False) - InstallFromDep(builddir) - TestInstallation(assume_installed=True) + InstallFromDep(builddir) + TestInstallation(assume_installed=True) - # We build the runtime target to get everything we need to test the - # standalone target. - Run([sys.executable, './tools/build.py', '-mrelease', '-ax64', 'runtime']) - # Copy in the installed binary to avoid poluting /usr/bin (and having to - # run as root) - Run(['cp', '/usr/bin/dart', 'out/ReleaseX64/dart']) + # We build the runtime target to get everything we need to test the + # standalone target. + Run([ + sys.executable, './tools/build.py', '-mrelease', '-ax64', 'runtime' + ]) + # Copy in the installed binary to avoid poluting /usr/bin (and having to + # run as root) + Run(['cp', '/usr/bin/dart', 'out/ReleaseX64/dart']) - # We currently can't run the testing script on wheezy since the checked in - # binary is built on precise, see issue 18742 - # TODO(18742): Run './tools/test.py' '-mrelease' 'standalone' + # We currently can't run the testing script on wheezy since the checked in + # binary is built on precise, see issue 18742 + # TODO(18742): Run './tools/test.py' '-mrelease' 'standalone' - # Sanity check dart2js and the analyzer against a hello world program - with utils.TempDir() as temp_dir: - test_file = CreateDartTestFile(temp_dir) - Run(['/usr/lib/dart/bin/dart2js', test_file]) - Run(['/usr/lib/dart/bin/dartanalyzer', test_file]) - Run(['/usr/lib/dart/bin/dart', test_file]) + # Sanity check dart2js and the analyzer against a hello world program + with utils.TempDir() as temp_dir: + test_file = CreateDartTestFile(temp_dir) + Run(['/usr/lib/dart/bin/dart2js', test_file]) + Run(['/usr/lib/dart/bin/dartanalyzer', test_file]) + Run(['/usr/lib/dart/bin/dart', test_file]) - # Sanity check that pub can start up and print the version - Run(['/usr/lib/dart/bin/pub', '--version']) + # Sanity check that pub can start up and print the version + Run(['/usr/lib/dart/bin/pub', '--version']) - UninstallDart() - TestInstallation(assume_installed=False) + UninstallDart() + TestInstallation(assume_installed=False) if __name__ == '__main__': - # We pass in None for build_step to avoid building the sdk. - bot.RunBot(SrcConfig, SrcSteps, build_step=None) + # We pass in None for build_step to avoid building the sdk. + bot.RunBot(SrcConfig, SrcSteps, build_step=None) diff --git a/tools/bots/pub.py b/tools/bots/pub.py index deb9b6d2b54..7eda4b8edce 100755 --- a/tools/bots/pub.py +++ b/tools/bots/pub.py @@ -3,7 +3,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Pub buildbot steps. @@ -17,32 +16,35 @@ import bot PUB_BUILDER = r'pub-(linux|mac|win)' + def PubConfig(name, is_buildbot): - """Returns info for the current buildbot based on the name of the builder. + """Returns info for the current buildbot based on the name of the builder. Currently, this is just: - mode: always release, we don't run pub in debug mode - system: "linux", "mac", or "win" - checked: always true """ - pub_pattern = re.match(PUB_BUILDER, name) - if not pub_pattern: - return None + pub_pattern = re.match(PUB_BUILDER, name) + if not pub_pattern: + return None - system = pub_pattern.group(1) - mode = 'release' - if system == 'win': system = 'windows' + system = pub_pattern.group(1) + mode = 'release' + if system == 'win': system = 'windows' + + return bot.BuildInfo('none', 'vm', mode, system, checked=True, arch='x64') - return bot.BuildInfo('none', 'vm', mode, system, checked=True, arch='x64') def PubSteps(build_info): - pub_location = os.path.join('third_party', 'pkg', 'pub') - with bot.BuildStep('Running pub tests'): - bot.RunTestRunner(build_info, pub_location) + pub_location = os.path.join('third_party', 'pkg', 'pub') + with bot.BuildStep('Running pub tests'): + bot.RunTestRunner(build_info, pub_location) + + dartdoc_location = os.path.join('third_party', 'pkg', 'dartdoc') + with bot.BuildStep('Running dartdoc tests'): + bot.RunTestRunner(build_info, dartdoc_location) - dartdoc_location = os.path.join('third_party', 'pkg', 'dartdoc') - with bot.BuildStep('Running dartdoc tests'): - bot.RunTestRunner(build_info, dartdoc_location) if __name__ == '__main__': - bot.RunBot(PubConfig, PubSteps) + bot.RunBot(PubConfig, PubSteps) diff --git a/tools/bots/pub_integration_test.py b/tools/bots/pub_integration_test.py index 16d084ae851..6d71e1e1aea 100755 --- a/tools/bots/pub_integration_test.py +++ b/tools/bots/pub_integration_test.py @@ -16,39 +16,41 @@ dependencies: test: """ + def Main(): - parser = optparse.OptionParser() - parser.add_option('--mode', action='store', dest='mode', type='string', - default='release') + parser = optparse.OptionParser() + parser.add_option( + '--mode', action='store', dest='mode', type='string', default='release') - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - out_dir_subfolder = 'DebugX64' if options.mode == 'debug' else 'ReleaseX64' + out_dir_subfolder = 'DebugX64' if options.mode == 'debug' else 'ReleaseX64' - out_dir = 'xcodebuild' if sys.platform == 'darwin' else 'out' - extension = '' if not sys.platform == 'win32' else '.bat' - pub = os.path.abspath( - '%s/%s/dart-sdk/bin/pub%s' % (out_dir, out_dir_subfolder, extension)) - print(pub) + out_dir = 'xcodebuild' if sys.platform == 'darwin' else 'out' + extension = '' if not sys.platform == 'win32' else '.bat' + pub = os.path.abspath( + '%s/%s/dart-sdk/bin/pub%s' % (out_dir, out_dir_subfolder, extension)) + print(pub) - working_dir = tempfile.mkdtemp() - try: - pub_cache_dir = working_dir + '/pub_cache' - env = os.environ.copy() - env['PUB_CACHE'] = pub_cache_dir + working_dir = tempfile.mkdtemp() + try: + pub_cache_dir = working_dir + '/pub_cache' + env = os.environ.copy() + env['PUB_CACHE'] = pub_cache_dir - with open(working_dir + '/pubspec.yaml', 'w') as pubspec_yaml: - pubspec_yaml.write(PUBSPEC) + with open(working_dir + '/pubspec.yaml', 'w') as pubspec_yaml: + pubspec_yaml.write(PUBSPEC) - exit_code = subprocess.call([pub, 'get'], cwd=working_dir, env=env) - if exit_code is not 0: - return exit_code + exit_code = subprocess.call([pub, 'get'], cwd=working_dir, env=env) + if exit_code is not 0: + return exit_code + + exit_code = subprocess.call([pub, 'upgrade'], cwd=working_dir, env=env) + if exit_code is not 0: + return exit_code + finally: + shutil.rmtree(working_dir) - exit_code = subprocess.call([pub, 'upgrade'], cwd=working_dir, env=env) - if exit_code is not 0: - return exit_code - finally: - shutil.rmtree(working_dir); if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/tools/bots/upload_debian_packages.py b/tools/bots/upload_debian_packages.py index 04750c042da..83a3056ec31 100755 --- a/tools/bots/upload_debian_packages.py +++ b/tools/bots/upload_debian_packages.py @@ -4,50 +4,45 @@ # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - import os - import bot import bot_utils - utils = bot_utils.GetUtils() - HOST_OS = utils.GuessOS() def ArchiveArtifacts(tarfile, builddir, channel): - namer = bot_utils.GCSNamer(channel=channel) - gsutil = bot_utils.GSUtil() - revision = utils.GetArchiveVersion() - # Archive the src tar to the src dir - remote_tarfile = '/'.join([namer.src_directory(revision), - os.path.basename(tarfile)]) - gsutil.upload(tarfile, remote_tarfile, public=True) - # Archive all files except the tar file to the linux packages dir - for entry in os.listdir(builddir): - full_path = os.path.join(builddir, entry) - # We expect a flat structure, not subdirectories - assert(os.path.isfile(full_path)) - if full_path != tarfile: - package_dir = namer.linux_packages_directory(revision) - remote_file = '/'.join([package_dir, - os.path.basename(entry)]) - gsutil.upload(full_path, remote_file, public=True) + namer = bot_utils.GCSNamer(channel=channel) + gsutil = bot_utils.GSUtil() + revision = utils.GetArchiveVersion() + # Archive the src tar to the src dir + remote_tarfile = '/'.join( + [namer.src_directory(revision), + os.path.basename(tarfile)]) + gsutil.upload(tarfile, remote_tarfile, public=True) + # Archive all files except the tar file to the linux packages dir + for entry in os.listdir(builddir): + full_path = os.path.join(builddir, entry) + # We expect a flat structure, not subdirectories + assert (os.path.isfile(full_path)) + if full_path != tarfile: + package_dir = namer.linux_packages_directory(revision) + remote_file = '/'.join([package_dir, os.path.basename(entry)]) + gsutil.upload(full_path, remote_file, public=True) if __name__ == '__main__': - bot_name, _ = bot.GetBotName() - channel = bot_utils.GetChannelFromName(bot_name) - if channel != bot_utils.Channel.BLEEDING_EDGE: - builddir = os.path.join(bot_utils.DART_DIR, - utils.GetBuildDir(HOST_OS), - 'src_and_installation') - version = utils.GetVersion() - tarfilename = 'dart-%s.tar.gz' % version - tarfile = os.path.join(builddir, tarfilename) - ArchiveArtifacts(tarfile, builddir, channel) - else: - print 'Not uploading artifacts on bleeding edge' + bot_name, _ = bot.GetBotName() + channel = bot_utils.GetChannelFromName(bot_name) + if channel != bot_utils.Channel.BLEEDING_EDGE: + builddir = os.path.join(bot_utils.DART_DIR, utils.GetBuildDir(HOST_OS), + 'src_and_installation') + version = utils.GetVersion() + tarfilename = 'dart-%s.tar.gz' % version + tarfile = os.path.join(builddir, tarfilename) + ArchiveArtifacts(tarfile, builddir, channel) + else: + print 'Not uploading artifacts on bleeding edge' diff --git a/tools/bots/version_checker.py b/tools/bots/version_checker.py index d61668e08aa..6c72aaf26cc 100755 --- a/tools/bots/version_checker.py +++ b/tools/bots/version_checker.py @@ -15,71 +15,76 @@ utils = bot_utils.GetUtils() VERSION_BUILDER = r'versionchecker' + def VersionConfig(name, is_buildbot): - version_pattern = re.match(VERSION_BUILDER, name) - if not version_pattern: - return None - # We don't really use this, but we create it anyway to use the standard - # bot execution model. - return bot.BuildInfo('none', 'none', 'release', 'linux') + version_pattern = re.match(VERSION_BUILDER, name) + if not version_pattern: + return None + # We don't really use this, but we create it anyway to use the standard + # bot execution model. + return bot.BuildInfo('none', 'none', 'release', 'linux') + def GetLatestVersionFromGCS(channel): - namer = bot_utils.GCSNamer(channel=channel) - gsutil = bot_utils.GSUtil() - gcs_version_path = namer.version_filepath('latest') - print 'Getting latest version from: %s' % gcs_version_path - version_json = gsutil.cat(gcs_version_path) - version_map = json.loads(version_json) - return version_map['version'] + namer = bot_utils.GCSNamer(channel=channel) + gsutil = bot_utils.GSUtil() + gcs_version_path = namer.version_filepath('latest') + print 'Getting latest version from: %s' % gcs_version_path + version_json = gsutil.cat(gcs_version_path) + version_map = json.loads(version_json) + return version_map['version'] + def ValidateChannelVersion(latest_version, channel): - repo_version = utils.ReadVersionFile() - assert repo_version.channel == channel - if channel == bot_utils.Channel.STABLE: - assert int(repo_version.prerelease) == 0 - assert int(repo_version.prerelease_patch) == 0 + repo_version = utils.ReadVersionFile() + assert repo_version.channel == channel + if channel == bot_utils.Channel.STABLE: + assert int(repo_version.prerelease) == 0 + assert int(repo_version.prerelease_patch) == 0 - version_re = r'(\d+)\.(\d+)\.(\d+)(-dev\.(\d+)\.(\d+))?' + version_re = r'(\d+)\.(\d+)\.(\d+)(-dev\.(\d+)\.(\d+))?' - latest_match = re.match(version_re, latest_version) - latest_major = int(latest_match.group(1)) - latest_minor = int(latest_match.group(2)) - latest_patch = int(latest_match.group(3)) - # We don't use these on stable. - latest_prerelease = int(latest_match.group(5) or 0) - latest_prerelease_patch = int(latest_match.group(6) or 0) + latest_match = re.match(version_re, latest_version) + latest_major = int(latest_match.group(1)) + latest_minor = int(latest_match.group(2)) + latest_patch = int(latest_match.group(3)) + # We don't use these on stable. + latest_prerelease = int(latest_match.group(5) or 0) + latest_prerelease_patch = int(latest_match.group(6) or 0) + + if latest_major < int(repo_version.major): + return True + if latest_minor < int(repo_version.minor): + return True + if latest_patch < int(repo_version.patch): + return True + if latest_prerelease < int(repo_version.prerelease): + return True + if latest_prerelease_patch < int(repo_version.prerelease_patch): + return True + return False - if latest_major < int(repo_version.major): - return True - if latest_minor < int(repo_version.minor): - return True - if latest_patch < int(repo_version.patch): - return True - if latest_prerelease < int(repo_version.prerelease): - return True - if latest_prerelease_patch < int(repo_version.prerelease_patch): - return True - return False def VersionSteps(build_info): - with bot.BuildStep('Version file sanity checking'): - bot_name, _ = bot.GetBotName() - channel = bot_utils.GetChannelFromName(bot_name) - if channel == bot_utils.Channel.BLEEDING_EDGE: - print 'No sanity checking on bleeding edge' - else: - assert (channel == bot_utils.Channel.STABLE or - channel == bot_utils.Channel.DEV) - latest_version = GetLatestVersionFromGCS(channel) - version = utils.GetVersion() - print 'Latests version on GCS: %s' % latest_version - print 'Version currently building: %s' % version - if not ValidateChannelVersion(latest_version, channel): - print "Validation failed" - sys.exit(1) - else: - print 'Version file changed, sanity checks passed' + with bot.BuildStep('Version file sanity checking'): + bot_name, _ = bot.GetBotName() + channel = bot_utils.GetChannelFromName(bot_name) + if channel == bot_utils.Channel.BLEEDING_EDGE: + print 'No sanity checking on bleeding edge' + else: + assert (channel == bot_utils.Channel.STABLE or + channel == bot_utils.Channel.DEV) + latest_version = GetLatestVersionFromGCS(channel) + version = utils.GetVersion() + print 'Latests version on GCS: %s' % latest_version + print 'Version currently building: %s' % version + if not ValidateChannelVersion(latest_version, channel): + print "Validation failed" + sys.exit(1) + else: + print 'Version file changed, sanity checks passed' + if __name__ == '__main__': - # We pass in None for build_step to avoid building. - bot.RunBot(VersionConfig, VersionSteps, build_step=None) + # We pass in None for build_step to avoid building. + bot.RunBot(VersionConfig, VersionSteps, build_step=None) diff --git a/tools/build.py b/tools/build.py index 1782f2b2ece..904c65608b3 100755 --- a/tools/build.py +++ b/tools/build.py @@ -16,10 +16,11 @@ HOST_OS = utils.GuessOS() HOST_CPUS = utils.GuessCpus() SCRIPT_DIR = os.path.dirname(sys.argv[0]) DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) -AVAILABLE_ARCHS = ['ia32', 'x64', 'simarm', 'arm', 'simarmv6', 'armv6', - 'simarmv5te', 'armv5te', 'simarm64', 'arm64', - 'simdbc', 'simdbc64', 'armsimdbc', 'armsimdbc64', 'simarm_x64'] - +AVAILABLE_ARCHS = [ + 'ia32', 'x64', 'simarm', 'arm', 'simarmv6', 'armv6', 'simarmv5te', + 'armv5te', 'simarm64', 'arm64', 'simdbc', 'simdbc64', 'armsimdbc', + 'armsimdbc64', 'simarm_x64' +] usage = """\ usage: %%prog [options] [targets] @@ -29,315 +30,337 @@ This script invokes ninja to build Dart. def BuildOptions(): - result = optparse.OptionParser(usage=usage) - result.add_option("-a", "--arch", - help='Target architectures (comma-separated).', - metavar='[all,' + ','.join(AVAILABLE_ARCHS) + ']', - default=utils.GuessArchitecture()) - result.add_option("-b", "--bytecode", - help='Build with the kernel bytecode interpreter. DEPRECATED.', - default=False, - action='store_true') - result.add_option("-j", - type=int, - help='Ninja -j option for Goma builds.', - default=1000) - result.add_option("-l", - type=int, - help='Ninja -l option for Goma builds.', - default=64) - result.add_option("-m", "--mode", - help='Build variants (comma-separated).', - metavar='[all,debug,release,product]', - default='debug') - result.add_option("--no-start-goma", - help="Don't try to start goma", - default=False, - action='store_true') - result.add_option("--os", - help='Target OSs (comma-separated).', - metavar='[all,host,android]', - default='host') - result.add_option("-v", "--verbose", - help='Verbose output.', - default=False, action="store_true") - return result + result = optparse.OptionParser(usage=usage) + result.add_option( + "-a", + "--arch", + help='Target architectures (comma-separated).', + metavar='[all,' + ','.join(AVAILABLE_ARCHS) + ']', + default=utils.GuessArchitecture()) + result.add_option( + "-b", + "--bytecode", + help='Build with the kernel bytecode interpreter. DEPRECATED.', + default=False, + action='store_true') + result.add_option( + "-j", type=int, help='Ninja -j option for Goma builds.', default=1000) + result.add_option( + "-l", type=int, help='Ninja -l option for Goma builds.', default=64) + result.add_option( + "-m", + "--mode", + help='Build variants (comma-separated).', + metavar='[all,debug,release,product]', + default='debug') + result.add_option( + "--no-start-goma", + help="Don't try to start goma", + default=False, + action='store_true') + result.add_option( + "--os", + help='Target OSs (comma-separated).', + metavar='[all,host,android]', + default='host') + result.add_option( + "-v", + "--verbose", + help='Verbose output.', + default=False, + action="store_true") + return result def ProcessOsOption(os_name): - if os_name == 'host': - return HOST_OS - return os_name + if os_name == 'host': + return HOST_OS + return os_name def ProcessOptions(options, args): - if options.arch == 'all': - options.arch = 'ia32,x64,simarm,simarm64,simdbc64' - if options.mode == 'all': - options.mode = 'debug,release,product' - if options.os == 'all': - options.os = 'host,android' - options.mode = options.mode.split(',') - options.arch = options.arch.split(',') - options.os = options.os.split(',') - for mode in options.mode: - if not mode in ['debug', 'release', 'product']: - print ("Unknown mode %s" % mode) - return False - for arch in options.arch: - if not arch in AVAILABLE_ARCHS: - print ("Unknown arch %s" % arch) - return False - options.os = [ProcessOsOption(os_name) for os_name in options.os] - for os_name in options.os: - if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']: - print ("Unknown os %s" % os_name) - return False - if os_name != HOST_OS: - if os_name != 'android': - print ("Unsupported target os %s" % os_name) - return False - if not HOST_OS in ['linux', 'macos']: - print ("Cross-compilation to %s is not supported on host os %s." - % (os_name, HOST_OS)) - return False - if not arch in ['ia32', 'x64', 'arm', 'armv6', 'armv5te', 'arm64', - 'simdbc', 'simdbc64']: - print ("Cross-compilation to %s is not supported for architecture %s." - % (os_name, arch)) - return False - # We have not yet tweaked the v8 dart build to work with the Android - # NDK/SDK, so don't try to build it. - if not args: - print ("For android builds you must specify a target, such as 'runtime'.") - return False - return True + if options.arch == 'all': + options.arch = 'ia32,x64,simarm,simarm64,simdbc64' + if options.mode == 'all': + options.mode = 'debug,release,product' + if options.os == 'all': + options.os = 'host,android' + options.mode = options.mode.split(',') + options.arch = options.arch.split(',') + options.os = options.os.split(',') + for mode in options.mode: + if not mode in ['debug', 'release', 'product']: + print("Unknown mode %s" % mode) + return False + for arch in options.arch: + if not arch in AVAILABLE_ARCHS: + print("Unknown arch %s" % arch) + return False + options.os = [ProcessOsOption(os_name) for os_name in options.os] + for os_name in options.os: + if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']: + print("Unknown os %s" % os_name) + return False + if os_name != HOST_OS: + if os_name != 'android': + print("Unsupported target os %s" % os_name) + return False + if not HOST_OS in ['linux', 'macos']: + print("Cross-compilation to %s is not supported on host os %s." + % (os_name, HOST_OS)) + return False + if not arch in [ + 'ia32', 'x64', 'arm', 'armv6', 'armv5te', 'arm64', 'simdbc', + 'simdbc64' + ]: + print( + "Cross-compilation to %s is not supported for architecture %s." + % (os_name, arch)) + return False + # We have not yet tweaked the v8 dart build to work with the Android + # NDK/SDK, so don't try to build it. + if not args: + print( + "For android builds you must specify a target, such as 'runtime'." + ) + return False + return True def NotifyBuildDone(build_config, success, start): - if not success: - print ("BUILD FAILED") + if not success: + print("BUILD FAILED") - sys.stdout.flush() + sys.stdout.flush() - # Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY. - notification_delay = float( - os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max)) - if (time.time() - start) < notification_delay: - return + # Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY. + notification_delay = float( + os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max)) + if (time.time() - start) < notification_delay: + return - if success: - message = 'Build succeeded.' - else: - message = 'Build failed.' - title = build_config - - command = None - if HOST_OS == 'macos': - # Use AppleScript to display a UI non-modal notification. - script = 'display notification "%s" with title "%s" sound name "Glass"' % ( - message, title) - command = "osascript -e '%s' &" % script - elif HOST_OS == 'linux': if success: - icon = 'dialog-information' + message = 'Build succeeded.' else: - icon = 'dialog-error' - command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title) - elif HOST_OS == 'win32': - if success: - icon = 'info' - else: - icon = 'error' - command = ("powershell -command \"" - "[reflection.assembly]::loadwithpartialname('System.Windows.Forms')" - "| Out-Null;" - "[reflection.assembly]::loadwithpartialname('System.Drawing')" - "| Out-Null;" - "$n = new-object system.windows.forms.notifyicon;" - "$n.icon = [system.drawing.systemicons]::information;" - "$n.visible = $true;" - "$n.showballoontip(%d, '%s', '%s', " - "[system.windows.forms.tooltipicon]::%s);\"") % ( - 5000, # Notification stays on for this many milliseconds - message, title, icon) + message = 'Build failed.' + title = build_config - if command: - # Ignore return code, if this command fails, it doesn't matter. - os.system(command) + command = None + if HOST_OS == 'macos': + # Use AppleScript to display a UI non-modal notification. + script = 'display notification "%s" with title "%s" sound name "Glass"' % ( + message, title) + command = "osascript -e '%s' &" % script + elif HOST_OS == 'linux': + if success: + icon = 'dialog-information' + else: + icon = 'dialog-error' + command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title) + elif HOST_OS == 'win32': + if success: + icon = 'info' + else: + icon = 'error' + command = ( + "powershell -command \"" + "[reflection.assembly]::loadwithpartialname('System.Windows.Forms')" + "| Out-Null;" + "[reflection.assembly]::loadwithpartialname('System.Drawing')" + "| Out-Null;" + "$n = new-object system.windows.forms.notifyicon;" + "$n.icon = [system.drawing.systemicons]::information;" + "$n.visible = $true;" + "$n.showballoontip(%d, '%s', '%s', " + "[system.windows.forms.tooltipicon]::%s);\"") % ( + 5000, # Notification stays on for this many milliseconds + message, + title, + icon) + + if command: + # Ignore return code, if this command fails, it doesn't matter. + os.system(command) def GenerateBuildfilesIfNeeded(): - if os.path.exists(utils.GetBuildDir(HOST_OS)): + if os.path.exists(utils.GetBuildDir(HOST_OS)): + return True + command = [ + 'python', + os.path.join(DART_ROOT, 'tools', 'generate_buildfiles.py') + ] + print("Running " + ' '.join(command)) + process = subprocess.Popen(command) + process.wait() + if process.returncode != 0: + print("Tried to generate missing buildfiles, but failed. " + "Try running manually:\n\t$ " + ' '.join(command)) + return False return True - command = [ - 'python', - os.path.join(DART_ROOT, 'tools', 'generate_buildfiles.py') - ] - print ("Running " + ' '.join(command)) - process = subprocess.Popen(command) - process.wait() - if process.returncode != 0: - print ("Tried to generate missing buildfiles, but failed. " - "Try running manually:\n\t$ " + ' '.join(command)) - return False - return True def RunGNIfNeeded(out_dir, target_os, mode, arch): - if os.path.isfile(os.path.join(out_dir, 'args.gn')): - return - gn_os = 'host' if target_os == HOST_OS else target_os - gn_command = [ - 'python', - os.path.join(DART_ROOT, 'tools', 'gn.py'), - '-m', mode, - '-a', arch, - '--os', gn_os, - '-v', - ] - process = subprocess.Popen(gn_command) - process.wait() - if process.returncode != 0: - print ("Tried to run GN, but it failed. Try running it manually: \n\t$ " + - ' '.join(gn_command)) + if os.path.isfile(os.path.join(out_dir, 'args.gn')): + return + gn_os = 'host' if target_os == HOST_OS else target_os + gn_command = [ + 'python', + os.path.join(DART_ROOT, 'tools', 'gn.py'), + '-m', + mode, + '-a', + arch, + '--os', + gn_os, + '-v', + ] + process = subprocess.Popen(gn_command) + process.wait() + if process.returncode != 0: + print("Tried to run GN, but it failed. Try running it manually: \n\t$ " + + ' '.join(gn_command)) def UseGoma(out_dir): - args_gn = os.path.join(out_dir, 'args.gn') - return 'use_goma = true' in open(args_gn, 'r').read() + args_gn = os.path.join(out_dir, 'args.gn') + return 'use_goma = true' in open(args_gn, 'r').read() # Try to start goma, but don't bail out if we can't. Instead print an error # message, and let the build fail with its own error messages as well. goma_started = False + + def EnsureGomaStarted(out_dir): - global goma_started - if goma_started: + global goma_started + if goma_started: + return True + args_gn_path = os.path.join(out_dir, 'args.gn') + goma_dir = None + with open(args_gn_path, 'r') as fp: + for line in fp: + if 'goma_dir' in line: + words = line.split() + goma_dir = words[2][1:-1] # goma_dir = "/path/to/goma" + if not goma_dir: + print('Could not find goma for ' + out_dir) + return False + if not os.path.exists(goma_dir) or not os.path.isdir(goma_dir): + print('Could not find goma at ' + goma_dir) + return False + goma_ctl = os.path.join(goma_dir, 'goma_ctl.py') + goma_ctl_command = [ + 'python', + goma_ctl, + 'ensure_start', + ] + process = subprocess.Popen(goma_ctl_command) + process.wait() + if process.returncode != 0: + print( + "Tried to run goma_ctl.py, but it failed. Try running it manually: " + + "\n\t" + ' '.join(goma_ctl_command)) + return False + goma_started = True return True - args_gn_path = os.path.join(out_dir, 'args.gn') - goma_dir = None - with open(args_gn_path, 'r') as fp: - for line in fp: - if 'goma_dir' in line: - words = line.split() - goma_dir = words[2][1:-1] # goma_dir = "/path/to/goma" - if not goma_dir: - print ('Could not find goma for ' + out_dir) - return False - if not os.path.exists(goma_dir) or not os.path.isdir(goma_dir): - print ('Could not find goma at ' + goma_dir) - return False - goma_ctl = os.path.join(goma_dir, 'goma_ctl.py') - goma_ctl_command = [ - 'python', - goma_ctl, - 'ensure_start', - ] - process = subprocess.Popen(goma_ctl_command) - process.wait() - if process.returncode != 0: - print ("Tried to run goma_ctl.py, but it failed. Try running it manually: " - + "\n\t" + ' '.join(goma_ctl_command)) - return False - goma_started = True - return True # Returns a tuple (build_config, command to run, whether goma is used) def BuildOneConfig(options, targets, target_os, mode, arch): - build_config = utils.GetBuildConf(mode, arch, target_os) - out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os) - using_goma = False - # TODO(zra): Remove auto-run of gn, replace with prompt for user to run - # gn.py manually. - RunGNIfNeeded(out_dir, target_os, mode, arch) - command = ['ninja', '-C', out_dir] - if options.verbose: - command += ['-v'] - if UseGoma(out_dir): - if options.no_start_goma or EnsureGomaStarted(out_dir): - using_goma = True - command += [('-j%s' % str(options.j))] - command += [('-l%s' % str(options.l))] - else: - # If we couldn't ensure that goma is started, let the build start, but - # slowly so we can see any helpful error messages that pop out. - command += ['-j1'] - command += targets - return (build_config, command, using_goma) + build_config = utils.GetBuildConf(mode, arch, target_os) + out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os) + using_goma = False + # TODO(zra): Remove auto-run of gn, replace with prompt for user to run + # gn.py manually. + RunGNIfNeeded(out_dir, target_os, mode, arch) + command = ['ninja', '-C', out_dir] + if options.verbose: + command += ['-v'] + if UseGoma(out_dir): + if options.no_start_goma or EnsureGomaStarted(out_dir): + using_goma = True + command += [('-j%s' % str(options.j))] + command += [('-l%s' % str(options.l))] + else: + # If we couldn't ensure that goma is started, let the build start, but + # slowly so we can see any helpful error messages that pop out. + command += ['-j1'] + command += targets + return (build_config, command, using_goma) def RunOneBuildCommand(build_config, args): - start_time = time.time() - print (' '.join(args)) - process = subprocess.Popen(args, stdin=None) - process.wait() - if process.returncode != 0: - NotifyBuildDone(build_config, success=False, start=start_time) - return 1 - else: - NotifyBuildDone(build_config, success=True, start=start_time) + start_time = time.time() + print(' '.join(args)) + process = subprocess.Popen(args, stdin=None) + process.wait() + if process.returncode != 0: + NotifyBuildDone(build_config, success=False, start=start_time) + return 1 + else: + NotifyBuildDone(build_config, success=True, start=start_time) - return 0 + return 0 def RunOneGomaBuildCommand(args): - try: - print (' '.join(args)) - process = subprocess.Popen(args, stdin=None) - process.wait() - print (' '.join(args) + " done.") - return process.returncode - except KeyboardInterrupt: - return 1 + try: + print(' '.join(args)) + process = subprocess.Popen(args, stdin=None) + process.wait() + print(' '.join(args) + " done.") + return process.returncode + except KeyboardInterrupt: + return 1 def Main(): - starttime = time.time() - # Parse the options. - parser = BuildOptions() - (options, args) = parser.parse_args() - if not ProcessOptions(options, args): - parser.print_help() - return 1 - # Determine which targets to build. By default we build the "all" target. - if len(args) == 0: - targets = ['all'] - else: - targets = args + starttime = time.time() + # Parse the options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options, args): + parser.print_help() + return 1 + # Determine which targets to build. By default we build the "all" target. + if len(args) == 0: + targets = ['all'] + else: + targets = args - if not GenerateBuildfilesIfNeeded(): - return 1 + if not GenerateBuildfilesIfNeeded(): + return 1 - # Build all targets for each requested configuration. - configs = [] - for target_os in options.os: - for mode in options.mode: - for arch in options.arch: - configs.append(BuildOneConfig(options, targets, target_os, mode, arch)) + # Build all targets for each requested configuration. + configs = [] + for target_os in options.os: + for mode in options.mode: + for arch in options.arch: + configs.append( + BuildOneConfig(options, targets, target_os, mode, arch)) - # Build regular configs. - goma_builds = [] - for (build_config, args, goma) in configs: - if args is None: - return 1 - if goma: - goma_builds.append(args) - elif RunOneBuildCommand(build_config, args) != 0: - return 1 + # Build regular configs. + goma_builds = [] + for (build_config, args, goma) in configs: + if args is None: + return 1 + if goma: + goma_builds.append(args) + elif RunOneBuildCommand(build_config, args) != 0: + return 1 - # Run goma builds in parallel. - pool = multiprocessing.Pool(multiprocessing.cpu_count()) - results = pool.map(RunOneGomaBuildCommand, goma_builds, chunksize=1) - for r in results: - if r != 0: - return 1 + # Run goma builds in parallel. + pool = multiprocessing.Pool(multiprocessing.cpu_count()) + results = pool.map(RunOneGomaBuildCommand, goma_builds, chunksize=1) + for r in results: + if r != 0: + return 1 - endtime = time.time() - print ("The build took %.3f seconds" % (endtime - starttime)) - return 0 + endtime = time.time() + print("The build took %.3f seconds" % (endtime - starttime)) + return 0 if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/tools/buildtools/update.py b/tools/buildtools/update.py index 936e729b1a6..a39822507e7 100755 --- a/tools/buildtools/update.py +++ b/tools/buildtools/update.py @@ -2,7 +2,6 @@ # Copyright 2017 The Dart project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Pulls down tools required to build Dart.""" import errno @@ -23,76 +22,67 @@ DEPOT_PATH = find_depot_tools.add_depot_tools_to_path() def UpdateClangFormatOnWindows(): - sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1') - output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe') - downloader_script = os.path.join( - DEPOT_PATH, 'download_from_google_storage.py') - download_cmd = [ - 'python', - downloader_script, - '--no_auth', - '--no_resume', - '--quiet', - '--platform=win', - '--bucket', - 'chromium-clang-format', - '-s', - sha1_file, - '-o', - output_dir - ] - return subprocess.call(download_cmd) + sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1') + output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe') + downloader_script = os.path.join(DEPOT_PATH, + 'download_from_google_storage.py') + download_cmd = [ + 'python', downloader_script, '--no_auth', '--no_resume', '--quiet', + '--platform=win', '--bucket', 'chromium-clang-format', '-s', sha1_file, + '-o', output_dir + ] + return subprocess.call(download_cmd) def CreateSymlink(symlink, link_name): - try: - os.symlink(symlink, link_name) - except OSError, e: - if e.errno == errno.EEXIST: - os.remove(link_name) - os.symlink(symlink, link_name) - else: - raise e + try: + os.symlink(symlink, link_name) + except OSError, e: + if e.errno == errno.EEXIST: + os.remove(link_name) + os.symlink(symlink, link_name) + else: + raise e # On Mac and Linux we symlink clang-format and gn to the place where # 'git cl format' expects them to be. def LinksForGitCLFormat(): - if sys.platform == 'darwin': - platform = 'darwin' - tools = 'mac' - toolchain = 'mac-x64' - elif sys.platform.startswith('linux'): - platform = 'linux' - tools = 'linux64' - toolchain = 'linux-x64' - else: - print 'Unknown platform: ' + sys.platform - return 1 + if sys.platform == 'darwin': + platform = 'darwin' + tools = 'mac' + toolchain = 'mac-x64' + elif sys.platform.startswith('linux'): + platform = 'linux' + tools = 'linux64' + toolchain = 'linux-x64' + else: + print 'Unknown platform: ' + sys.platform + return 1 - clang_format = os.path.join( - BUILDTOOLS, toolchain, 'clang', 'bin', 'clang-format') - gn = os.path.join(BUILDTOOLS, 'gn') - dest_dir = os.path.join(BUILDTOOLS, tools) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - clang_format_dest = os.path.join(dest_dir, 'clang-format') - gn_dest = os.path.join(dest_dir, 'gn') - CreateSymlink(clang_format, clang_format_dest) - CreateSymlink(gn, gn_dest) - return 0 + clang_format = os.path.join(BUILDTOOLS, toolchain, 'clang', 'bin', + 'clang-format') + gn = os.path.join(BUILDTOOLS, 'gn') + dest_dir = os.path.join(BUILDTOOLS, tools) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + clang_format_dest = os.path.join(dest_dir, 'clang-format') + gn_dest = os.path.join(dest_dir, 'gn') + CreateSymlink(clang_format, clang_format_dest) + CreateSymlink(gn, gn_dest) + return 0 def main(argv): - arch_id = platform.machine() - # Don't try to download binaries if we're on an arm machine. - if arch_id.startswith('arm') or arch_id.startswith('aarch64'): - print('Not downloading buildtools binaries for ' + arch_id) - return 0 - if sys.platform.startswith('win'): - return UpdateClangFormatOnWindows() - return LinksForGitCLFormat() + arch_id = platform.machine() + # Don't try to download binaries if we're on an arm machine. + if arch_id.startswith('arm') or arch_id.startswith('aarch64'): + print('Not downloading buildtools binaries for ' + arch_id) + return 0 + if sys.platform.startswith('win'): + return UpdateClangFormatOnWindows() + return LinksForGitCLFormat() if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/tools/clean_output_directory.py b/tools/clean_output_directory.py index f0f0426e60b..e7233b061cc 100755 --- a/tools/clean_output_directory.py +++ b/tools/clean_output_directory.py @@ -11,17 +11,19 @@ import sys import subprocess import utils + def Main(): - build_root = utils.GetBuildRoot(utils.GuessOS()) - print 'Deleting %s' % build_root - if sys.platform != 'win32': - shutil.rmtree(build_root, ignore_errors=True) - else: - # Intentionally ignore return value since a directory might be in use. - subprocess.call(['rmdir', '/Q', '/S', build_root], - env=os.environ.copy(), - shell=True) - return 0 + build_root = utils.GetBuildRoot(utils.GuessOS()) + print 'Deleting %s' % build_root + if sys.platform != 'win32': + shutil.rmtree(build_root, ignore_errors=True) + else: + # Intentionally ignore return value since a directory might be in use. + subprocess.call(['rmdir', '/Q', '/S', build_root], + env=os.environ.copy(), + shell=True) + return 0 + if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/tools/copy_dart.py b/tools/copy_dart.py index 5bd464c5590..2bfb582c8c1 100755 --- a/tools/copy_dart.py +++ b/tools/copy_dart.py @@ -11,129 +11,135 @@ import re from os.path import basename, dirname, exists, isabs, join from glob import glob -re_directive = re.compile( - r'^(library|import|part|native|resource)\s+(.*);$') -re_comment = re.compile( - r'^(///|/\*| \*).*$') +re_directive = re.compile(r'^(library|import|part|native|resource)\s+(.*);$') +re_comment = re.compile(r'^(///|/\*| \*).*$') + class Library(object): - def __init__(self, name, imports, sources, natives, code, comment): - self.name = name - self.imports = imports - self.sources = sources - self.natives = natives - self.code = code - self.comment = comment + + def __init__(self, name, imports, sources, natives, code, comment): + self.name = name + self.imports = imports + self.sources = sources + self.natives = natives + self.code = code + self.comment = comment + def parseLibrary(library): - """ Parses a .dart source file that is the root of a library, and returns + """ Parses a .dart source file that is the root of a library, and returns information about it: the name, the imports, included sources, and any code in the file. """ - libraryname = None - imports = [] - sources = [] - natives = [] - inlinecode = [] - librarycomment = [] - if exists(library): - # TODO(sigmund): stop parsing when import/source - for line in fileinput.input(library): - match = re_directive.match(line) - if match: - directive = match.group(1) - if directive == 'library': - assert libraryname is None - libraryname = match.group(2) - elif directive == 'part': - suffix = match.group(2) - if not suffix.startswith('of '): - sources.append(match.group(2).strip('"\'')) - elif directive == 'import': - imports.append(match.group(2)) - else: - raise Exception('unknown directive %s in %s' % (directive, line)) - else: - # Check for library comment. - if not libraryname and re_comment.match(line): - librarycomment.append(line) - else: - inlinecode.append(line) - fileinput.close() - return Library(libraryname, imports, sources, natives, inlinecode, - librarycomment) + libraryname = None + imports = [] + sources = [] + natives = [] + inlinecode = [] + librarycomment = [] + if exists(library): + # TODO(sigmund): stop parsing when import/source + for line in fileinput.input(library): + match = re_directive.match(line) + if match: + directive = match.group(1) + if directive == 'library': + assert libraryname is None + libraryname = match.group(2) + elif directive == 'part': + suffix = match.group(2) + if not suffix.startswith('of '): + sources.append(match.group(2).strip('"\'')) + elif directive == 'import': + imports.append(match.group(2)) + else: + raise Exception( + 'unknown directive %s in %s' % (directive, line)) + else: + # Check for library comment. + if not libraryname and re_comment.match(line): + librarycomment.append(line) + else: + inlinecode.append(line) + fileinput.close() + return Library(libraryname, imports, sources, natives, inlinecode, + librarycomment) + def normjoin(*args): - return os.path.normpath(os.path.join(*args)) + return os.path.normpath(os.path.join(*args)) + def mergefiles(srcs, dstfile): - for src in srcs: - with open(src, 'r') as s: - for line in s: - if not line.startswith('part of '): - dstfile.write(line) - -def main(outdir = None, *inputs): - if not outdir or not inputs: - print "Usage: %s OUTDIR INPUTS" % sys.argv[0] - print " OUTDIR is the war directory to copy to" - print " INPUTS is a list of files or patterns used to specify the input" - print " .dart files" - print "This script should be run from the client root directory." - print "Files will be merged and copied to: OUTDIR/relative-path-of-file," - print "except for dart files with absolute paths, which will be copied to" - print " OUTDIR/absolute-path-as-directories" - return 1 - - entry_libraries = [] - for i in inputs: - entry_libraries.extend(glob(i)) - - for entrypoint in entry_libraries: - # Get the transitive set of dart files this entrypoint depends on, merging - # each library along the way. - worklist = [os.path.normpath(entrypoint)] - seen = set() - while len(worklist) > 0: - lib = worklist.pop() - if lib in seen: - continue - - seen.add(lib) - - if (dirname(dirname(lib)).endswith('dom/generated/src') - or dirname(lib).endswith('dom/src')): - continue - - library = parseLibrary(lib) - - # Ensure output directory exists - outpath = join(outdir, lib[1:] if isabs(lib) else lib) - dstpath = dirname(outpath) - if not exists(dstpath): - os.makedirs(dstpath) + for src in srcs: + with open(src, 'r') as s: + for line in s: + if not line.startswith('part of '): + dstfile.write(line) - # Create file containing all imports, and inlining all sources - with open(outpath, 'w') as f: - if library.name: - if library.comment: - f.write('%s' % (''.join(library.comment))) - f.write("library %s;\n\n" % library.name) - else: - f.write("library %s;\n\n" % basename(lib)) - for importfile in library.imports: - f.write("import %s;\n" % importfile) - f.write('%s' % (''.join(library.code))) - mergefiles([normjoin(dirname(lib), s) for s in library.sources], f) +def main(outdir=None, *inputs): + if not outdir or not inputs: + print "Usage: %s OUTDIR INPUTS" % sys.argv[0] + print " OUTDIR is the war directory to copy to" + print " INPUTS is a list of files or patterns used to specify the input" + print " .dart files" + print "This script should be run from the client root directory." + print "Files will be merged and copied to: OUTDIR/relative-path-of-file," + print "except for dart files with absolute paths, which will be copied to" + print " OUTDIR/absolute-path-as-directories" + return 1 - for suffix in library.imports: - m = re.match(r'[\'"]([^\'"]+)[\'"](\s+as\s+\w+)?.*$', suffix) - uri = m.group(1) - if not uri.startswith('dart:'): - worklist.append(normjoin(dirname(lib), uri)) + entry_libraries = [] + for i in inputs: + entry_libraries.extend(glob(i)) + + for entrypoint in entry_libraries: + # Get the transitive set of dart files this entrypoint depends on, merging + # each library along the way. + worklist = [os.path.normpath(entrypoint)] + seen = set() + while len(worklist) > 0: + lib = worklist.pop() + if lib in seen: + continue + + seen.add(lib) + + if (dirname(dirname(lib)).endswith('dom/generated/src') or + dirname(lib).endswith('dom/src')): + continue + + library = parseLibrary(lib) + + # Ensure output directory exists + outpath = join(outdir, lib[1:] if isabs(lib) else lib) + dstpath = dirname(outpath) + if not exists(dstpath): + os.makedirs(dstpath) + + # Create file containing all imports, and inlining all sources + with open(outpath, 'w') as f: + if library.name: + if library.comment: + f.write('%s' % (''.join(library.comment))) + f.write("library %s;\n\n" % library.name) + else: + f.write("library %s;\n\n" % basename(lib)) + for importfile in library.imports: + f.write("import %s;\n" % importfile) + f.write('%s' % (''.join(library.code))) + mergefiles([normjoin(dirname(lib), s) for s in library.sources], + f) + + for suffix in library.imports: + m = re.match(r'[\'"]([^\'"]+)[\'"](\s+as\s+\w+)?.*$', suffix) + uri = m.group(1) + if not uri.startswith('dart:'): + worklist.append(normjoin(dirname(lib), uri)) + + return 0 - return 0 if __name__ == '__main__': - sys.exit(main(*sys.argv[1:])) + sys.exit(main(*sys.argv[1:])) diff --git a/tools/copy_tree.py b/tools/copy_tree.py index bd6fd289b97..f5c5c569eee 100755 --- a/tools/copy_tree.py +++ b/tools/copy_tree.py @@ -10,156 +10,160 @@ import re import shutil import sys + def ParseArgs(args): - args = args[1:] - parser = argparse.ArgumentParser( - description='A script to copy a file tree somewhere') + args = args[1:] + parser = argparse.ArgumentParser( + description='A script to copy a file tree somewhere') - parser.add_argument('--exclude_patterns', '-e', - type=str, - help='Patterns to exclude [passed to shutil.copytree]') - parser.add_argument('--from', '-f', - dest="copy_from", - type=str, - help='Source directory') - parser.add_argument('--gn', '-g', - dest='gn', - default=False, - action='store_true', - help='Output for GN for multiple sources, but do not copy anything.') - parser.add_argument('gn_paths', - metavar='name path ignore_pattern', - type=str, - nargs='*', - default=None, - help='When --gn is given, the specification of source paths to list.') - parser.add_argument('--to', '-t', - type=str, - help='Destination directory') + parser.add_argument( + '--exclude_patterns', + '-e', + type=str, + help='Patterns to exclude [passed to shutil.copytree]') + parser.add_argument( + '--from', '-f', dest="copy_from", type=str, help='Source directory') + parser.add_argument( + '--gn', + '-g', + dest='gn', + default=False, + action='store_true', + help='Output for GN for multiple sources, but do not copy anything.') + parser.add_argument( + 'gn_paths', + metavar='name path ignore_pattern', + type=str, + nargs='*', + default=None, + help='When --gn is given, the specification of source paths to list.') + parser.add_argument('--to', '-t', type=str, help='Destination directory') - return parser.parse_args(args) + return parser.parse_args(args) def ValidateArgs(args): - if args.gn: - if args.exclude_patterns or args.copy_from or args.to: - print ("--gn mode does not accept other switches") - return False - if not args.gn_paths: - print ("--gn mode requires a list of source specifications") - return False + if args.gn: + if args.exclude_patterns or args.copy_from or args.to: + print("--gn mode does not accept other switches") + return False + if not args.gn_paths: + print("--gn mode requires a list of source specifications") + return False + return True + if not args.copy_from or not os.path.isdir(args.copy_from): + print("--from argument must refer to a directory") + return False + if not args.to: + print("--to is required") + return False return True - if not args.copy_from or not os.path.isdir(args.copy_from): - print ("--from argument must refer to a directory") - return False - if not args.to: - print ("--to is required") - return False - return True def CopyTree(src, dst, ignore=None): - # Recusive helper method to collect errors but keep processing. - def copy_tree(src, dst, ignore, errors): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - if not os.path.exists(dst): - os.makedirs(dst) - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if os.path.isdir(srcname): - copy_tree(srcname, dstname, ignore, errors) + # Recusive helper method to collect errors but keep processing. + def copy_tree(src, dst, ignore, errors): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) else: - shutil.copy(srcname, dstname) - except (IOError, os.error) as why: - errors.append((srcname, dstname, str(why))) - try: - shutil.copystat(src, dst) - except WindowsError: - # Can't copy file access times on Windows. - pass - except OSError as why: - errors.append((src, dst, str(why))) + ignored_names = set() - # Format errors from file copies. - def format_error(error): - if len(error) == 1: - return "Error: {msg}".format(msg=str(error[0])) - return "From: {src}\nTo: {dst}\n{msg}" \ - .format(src=error[0], dst=error[1], msg=error[2]) + if not os.path.exists(dst): + os.makedirs(dst) + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if os.path.isdir(srcname): + copy_tree(srcname, dstname, ignore, errors) + else: + shutil.copy(srcname, dstname) + except (IOError, os.error) as why: + errors.append((srcname, dstname, str(why))) + try: + shutil.copystat(src, dst) + except WindowsError: + # Can't copy file access times on Windows. + pass + except OSError as why: + errors.append((src, dst, str(why))) + + # Format errors from file copies. + def format_error(error): + if len(error) == 1: + return "Error: {msg}".format(msg=str(error[0])) + return "From: {src}\nTo: {dst}\n{msg}" \ + .format(src=error[0], dst=error[1], msg=error[2]) + + errors = [] + copy_tree(src, dst, ignore, errors) + if errors: + failures = "\n\n".join(format_error(error) for error in errors) + parts = ("Some file copies failed:", "=" * 78, failures) + msg = '\n'.join(parts) + raise RuntimeError(msg) - errors = [] - copy_tree(src,dst,ignore, errors) - if errors: - failures = "\n\n".join(format_error(error) for error in errors) - parts = ("Some file copies failed:", "="*78, failures) - msg = '\n'.join(parts) - raise RuntimeError(msg) def ListTree(src, ignore=None): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - srcnames = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - if os.path.isdir(srcname): - srcnames.extend(ListTree(srcname, ignore)) + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) else: - srcnames.append(srcname) - return srcnames + ignored_names = set() + + srcnames = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + if os.path.isdir(srcname): + srcnames.extend(ListTree(srcname, ignore)) + else: + srcnames.append(srcname) + return srcnames # source_dirs is organized such that sources_dirs[n] is the path for the source # directory, and source_dirs[n+1] is a list of ignore patterns. def SourcesToGN(source_dirs): - if len(source_dirs) % 2 != 0: - print ("--gn list length should be a multiple of 2.") - return False - data = [] - for i in range(0, len(source_dirs), 2): - path = source_dirs[i] - ignores = source_dirs[i + 1] - if ignores in ["{}"]: - sources = ListTree(path) - else: - patterns = ignores.split(',') - sources = ListTree(path, ignore=shutil.ignore_patterns(*patterns)) - data.append(sources) - scope_data = {"sources": data} - print (gn_helpers.ToGNString(scope_data)) - return True + if len(source_dirs) % 2 != 0: + print("--gn list length should be a multiple of 2.") + return False + data = [] + for i in range(0, len(source_dirs), 2): + path = source_dirs[i] + ignores = source_dirs[i + 1] + if ignores in ["{}"]: + sources = ListTree(path) + else: + patterns = ignores.split(',') + sources = ListTree(path, ignore=shutil.ignore_patterns(*patterns)) + data.append(sources) + scope_data = {"sources": data} + print(gn_helpers.ToGNString(scope_data)) + return True def Main(argv): - args = ParseArgs(argv) - if not ValidateArgs(args): - return -1 + args = ParseArgs(argv) + if not ValidateArgs(args): + return -1 - if args.gn: - SourcesToGN(args.gn_paths) + if args.gn: + SourcesToGN(args.gn_paths) + return 0 + + if args.exclude_patterns == None: + CopyTree(args.copy_from, args.to) + else: + patterns = args.exclude_patterns.split(',') + CopyTree( + args.copy_from, args.to, ignore=shutil.ignore_patterns(*patterns)) return 0 - if args.exclude_patterns == None: - CopyTree(args.copy_from, args.to) - else: - patterns = args.exclude_patterns.split(',') - CopyTree(args.copy_from, args.to, ignore=shutil.ignore_patterns(*patterns)) - return 0 - if __name__ == '__main__': - sys.exit(Main(sys.argv)) + sys.exit(Main(sys.argv)) diff --git a/tools/create_debian_packages.py b/tools/create_debian_packages.py index 5340393a170..e0e81bb1e46 100755 --- a/tools/create_debian_packages.py +++ b/tools/create_debian_packages.py @@ -22,131 +22,130 @@ HOST_OS = utils.GuessOS() HOST_CPUS = utils.GuessCpus() DART_DIR = abspath(join(__file__, '..', '..')) -def BuildOptions(): - result = optparse.OptionParser() - result.add_option("--tar_filename", - default=None, - help="The tar file to build from.") - result.add_option("--out_dir", - default=None, - help="Where to put the packages.") - result.add_option("-a", "--arch", - help='Target architectures (comma-separated).', - metavar='[all,ia32,x64,armel,armhf]', - default='x64') - result.add_option("-t", "--toolchain", - help='Cross-compilation toolchain prefix', - default=None) - return result +def BuildOptions(): + result = optparse.OptionParser() + result.add_option( + "--tar_filename", default=None, help="The tar file to build from.") + result.add_option( + "--out_dir", default=None, help="Where to put the packages.") + result.add_option( + "-a", + "--arch", + help='Target architectures (comma-separated).', + metavar='[all,ia32,x64,armel,armhf]', + default='x64') + result.add_option( + "-t", + "--toolchain", + help='Cross-compilation toolchain prefix', + default=None) + + return result + def RunBuildPackage(opt, cwd, toolchain=None): - env = os.environ.copy() - if toolchain != None: - env["TOOLCHAIN"] = '--toolchain=' + toolchain - cmd = ['dpkg-buildpackage', '-j%d' % HOST_CPUS] - cmd.extend(opt) - process = subprocess.check_call(cmd, cwd=cwd, env=env) + env = os.environ.copy() + if toolchain != None: + env["TOOLCHAIN"] = '--toolchain=' + toolchain + cmd = ['dpkg-buildpackage', '-j%d' % HOST_CPUS] + cmd.extend(opt) + process = subprocess.check_call(cmd, cwd=cwd, env=env) + def BuildDebianPackage(tarball, out_dir, arch, toolchain): - version = utils.GetVersion() - tarroot = 'dart-%s' % version - origtarname = 'dart_%s.orig.tar.gz' % version + version = utils.GetVersion() + tarroot = 'dart-%s' % version + origtarname = 'dart_%s.orig.tar.gz' % version - if not exists(tarball): - print 'Source tarball not found' - return -1 + if not exists(tarball): + print 'Source tarball not found' + return -1 - with utils.TempDir() as temp_dir: - origtarball = join(temp_dir, origtarname) - copyfile(tarball, origtarball) + with utils.TempDir() as temp_dir: + origtarball = join(temp_dir, origtarname) + copyfile(tarball, origtarball) - with tarfile.open(origtarball) as tar: - tar.extractall(path=temp_dir) + with tarfile.open(origtarball) as tar: + tar.extractall(path=temp_dir) - # Build source package. - print "Building source package" - RunBuildPackage(['-S', '-us', '-uc'], join(temp_dir, tarroot)) + # Build source package. + print "Building source package" + RunBuildPackage(['-S', '-us', '-uc'], join(temp_dir, tarroot)) - # Build 32-bit binary package. - if 'ia32' in arch: - print "Building i386 package" - RunBuildPackage(['-B', '-ai386', '-us', '-uc'], join(temp_dir, tarroot)) + # Build 32-bit binary package. + if 'ia32' in arch: + print "Building i386 package" + RunBuildPackage(['-B', '-ai386', '-us', '-uc'], + join(temp_dir, tarroot)) - # Build 64-bit binary package. - if 'x64' in arch: - print "Building amd64 package" - RunBuildPackage(['-B', '-aamd64', '-us', '-uc'], join(temp_dir, tarroot)) + # Build 64-bit binary package. + if 'x64' in arch: + print "Building amd64 package" + RunBuildPackage(['-B', '-aamd64', '-us', '-uc'], + join(temp_dir, tarroot)) - # Build armhf binary package. - if 'armhf' in arch: - print "Building armhf package" - RunBuildPackage( - ['-B', '-aarmhf', '-us', '-uc'], join(temp_dir, tarroot), toolchain) + # Build armhf binary package. + if 'armhf' in arch: + print "Building armhf package" + RunBuildPackage(['-B', '-aarmhf', '-us', '-uc'], + join(temp_dir, tarroot), toolchain) - # Build armel binary package. - if 'armel' in arch: - print "Building armel package" - RunBuildPackage( - ['-B', '-aarmel', '-us', '-uc'], join(temp_dir, tarroot), toolchain) + # Build armel binary package. + if 'armel' in arch: + print "Building armel package" + RunBuildPackage(['-B', '-aarmel', '-us', '-uc'], + join(temp_dir, tarroot), toolchain) - # Copy the Debian package files to the build directory. - debbase = 'dart_%s' % version - source_package = [ - '%s-1.dsc' % debbase, - '%s.orig.tar.gz' % debbase, - '%s-1.debian.tar.xz' % debbase - ] - i386_package = [ - '%s-1_i386.deb' % debbase - ] - amd64_package = [ - '%s-1_amd64.deb' % debbase - ] - armhf_package = [ - '%s-1_armhf.deb' % debbase - ] - armel_package = [ - '%s-1_armel.deb' % debbase - ] + # Copy the Debian package files to the build directory. + debbase = 'dart_%s' % version + source_package = [ + '%s-1.dsc' % debbase, + '%s.orig.tar.gz' % debbase, + '%s-1.debian.tar.xz' % debbase + ] + i386_package = ['%s-1_i386.deb' % debbase] + amd64_package = ['%s-1_amd64.deb' % debbase] + armhf_package = ['%s-1_armhf.deb' % debbase] + armel_package = ['%s-1_armel.deb' % debbase] - for name in source_package: - copyfile(join(temp_dir, name), join(out_dir, name)) - if 'ia32' in arch: - for name in i386_package: - copyfile(join(temp_dir, name), join(out_dir, name)) - if 'x64' in arch: - for name in amd64_package: - copyfile(join(temp_dir, name), join(out_dir, name)) - if ('armhf' in arch): - for name in armhf_package: - copyfile(join(temp_dir, name), join(out_dir, name)) - if ('armel' in arch): - for name in armel_package: - copyfile(join(temp_dir, name), join(out_dir, name)) + for name in source_package: + copyfile(join(temp_dir, name), join(out_dir, name)) + if 'ia32' in arch: + for name in i386_package: + copyfile(join(temp_dir, name), join(out_dir, name)) + if 'x64' in arch: + for name in amd64_package: + copyfile(join(temp_dir, name), join(out_dir, name)) + if ('armhf' in arch): + for name in armhf_package: + copyfile(join(temp_dir, name), join(out_dir, name)) + if ('armel' in arch): + for name in armel_package: + copyfile(join(temp_dir, name), join(out_dir, name)) def Main(): - if HOST_OS != 'linux': - print 'Debian build only supported on linux' - return -1 + if HOST_OS != 'linux': + print 'Debian build only supported on linux' + return -1 - options, args = BuildOptions().parse_args() - out_dir = options.out_dir - tar_filename = options.tar_filename - if options.arch == 'all': - options.arch = 'ia32,x64,armhf' - arch = options.arch.split(',') + options, args = BuildOptions().parse_args() + out_dir = options.out_dir + tar_filename = options.tar_filename + if options.arch == 'all': + options.arch = 'ia32,x64,armhf' + arch = options.arch.split(',') - if not options.out_dir: - out_dir = join(DART_DIR, utils.GetBuildDir(HOST_OS)) + if not options.out_dir: + out_dir = join(DART_DIR, utils.GetBuildDir(HOST_OS)) - if not tar_filename: - tar_filename = join(DART_DIR, - utils.GetBuildDir(HOST_OS), - 'dart-%s.tar.gz' % utils.GetVersion()) + if not tar_filename: + tar_filename = join(DART_DIR, utils.GetBuildDir(HOST_OS), + 'dart-%s.tar.gz' % utils.GetVersion()) + + BuildDebianPackage(tar_filename, out_dir, arch, options.toolchain) - BuildDebianPackage(tar_filename, out_dir, arch, options.toolchain) if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/tools/create_pkg_manifest.py b/tools/create_pkg_manifest.py index c49eac89660..0448ca9a85e 100755 --- a/tools/create_pkg_manifest.py +++ b/tools/create_pkg_manifest.py @@ -19,98 +19,104 @@ import utils SCRIPT_DIR = os.path.dirname(sys.argv[0]) DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) + # Used in parsing the DEPS file. class VarImpl(object): - _env_vars = { - "host_cpu": "x64", - "host_os": "linux", - } + _env_vars = { + "host_cpu": "x64", + "host_os": "linux", + } - def __init__(self, local_scope): - self._local_scope = local_scope + def __init__(self, local_scope): + self._local_scope = local_scope - def Lookup(self, var_name): - """Implements the Var syntax.""" - if var_name in self._local_scope.get("vars", {}): - return self._local_scope["vars"][var_name] - # Inject default values for env variables - if var_name in self._env_vars: - return self._env_vars[var_name] - raise Exception("Var is not defined: %s" % var_name) + def Lookup(self, var_name): + """Implements the Var syntax.""" + if var_name in self._local_scope.get("vars", {}): + return self._local_scope["vars"][var_name] + # Inject default values for env variables + if var_name in self._env_vars: + return self._env_vars[var_name] + raise Exception("Var is not defined: %s" % var_name) def ParseDepsFile(deps_file): - local_scope = {} - var = VarImpl(local_scope) - global_scope = { - 'Var': var.Lookup, - 'deps_os': {}, - } - # Read the content. - with open(deps_file, 'r') as fp: - deps_content = fp.read() + local_scope = {} + var = VarImpl(local_scope) + global_scope = { + 'Var': var.Lookup, + 'deps_os': {}, + } + # Read the content. + with open(deps_file, 'r') as fp: + deps_content = fp.read() - # Eval the content. - exec(deps_content, global_scope, local_scope) + # Eval the content. + exec (deps_content, global_scope, local_scope) - # Extract the deps and filter. - deps = local_scope.get('deps', {}) - filtered_deps = {} - for k, v in deps.iteritems(): - if 'sdk/third_party/pkg' in k: - new_key = k.replace('sdk', 'third_party/dart', 1) - filtered_deps[new_key] = v + # Extract the deps and filter. + deps = local_scope.get('deps', {}) + filtered_deps = {} + for k, v in deps.iteritems(): + if 'sdk/third_party/pkg' in k: + new_key = k.replace('sdk', 'third_party/dart', 1) + filtered_deps[new_key] = v - return filtered_deps + return filtered_deps def WriteManifest(deps, manifest_file): - project_template = """ + project_template = """ """ - warning = ('\n') - with open(manifest_file, 'w') as manifest: - manifest.write('\n') - manifest.write(warning) - manifest.write('\n') - manifest.write(' \n') - for path, remote in sorted(deps.iteritems()): - remote_components = remote.split('@') - remote_url = remote_components[0] - remote_version = remote_components[1] - manifest.write( - project_template % (path, path, remote_url, remote_version)) - manifest.write(' \n') - manifest.write('\n') + warning = ( + '\n') + with open(manifest_file, 'w') as manifest: + manifest.write('\n') + manifest.write(warning) + manifest.write('\n') + manifest.write(' \n') + for path, remote in sorted(deps.iteritems()): + remote_components = remote.split('@') + remote_url = remote_components[0] + remote_version = remote_components[1] + manifest.write( + project_template % (path, path, remote_url, remote_version)) + manifest.write(' \n') + manifest.write('\n') def ParseArgs(args): - args = args[1:] - parser = argparse.ArgumentParser( - description='A script to generate a jiri manifest for third_party/pkg.') + args = args[1:] + parser = argparse.ArgumentParser( + description='A script to generate a jiri manifest for third_party/pkg.') - parser.add_argument('--deps', '-d', - type=str, - help='Input DEPS file.', - default=os.path.join(DART_ROOT, 'DEPS')) - parser.add_argument('--output', '-o', - type=str, - help='Output jiri manifest.', - default=os.path.join(DART_ROOT, 'dart_third_party_pkg.manifest')) + parser.add_argument( + '--deps', + '-d', + type=str, + help='Input DEPS file.', + default=os.path.join(DART_ROOT, 'DEPS')) + parser.add_argument( + '--output', + '-o', + type=str, + help='Output jiri manifest.', + default=os.path.join(DART_ROOT, 'dart_third_party_pkg.manifest')) - return parser.parse_args(args) + return parser.parse_args(args) def Main(argv): - args = ParseArgs(argv) - deps = ParseDepsFile(args.deps) - WriteManifest(deps, args.output) - return 0 + args = ParseArgs(argv) + deps = ParseDepsFile(args.deps) + WriteManifest(deps, args.output) + return 0 if __name__ == '__main__': - sys.exit(Main(sys.argv)) + sys.exit(Main(sys.argv)) diff --git a/tools/create_tarball.py b/tools/create_tarball.py index 918f532c0ee..ee5630a88df 100755 --- a/tools/create_tarball.py +++ b/tools/create_tarball.py @@ -30,7 +30,6 @@ from os.path import join, split, abspath import utils - HOST_OS = utils.GuessOS() DART_DIR = abspath(join(__file__, '..', '..')) # Flags. @@ -40,137 +39,149 @@ verbose = False versiondir = '' # Ignore Git/SVN files, checked-in binaries, backup files, etc.. -ignoredPaths = ['third_party/7zip', 'third_party/android_tools', - 'third_party/clang', 'third_party/d8', - 'third_party/firefox_jsshell'] +ignoredPaths = [ + 'third_party/7zip', 'third_party/android_tools', 'third_party/clang', + 'third_party/d8', 'third_party/firefox_jsshell' +] ignoredDirs = ['.svn', '.git'] ignoredEndings = ['.mk', '.pyc', 'Makefile', '~'] -def BuildOptions(): - result = optparse.OptionParser() - result.add_option("-v", "--verbose", - help='Verbose output.', - default=False, action="store_true") - result.add_option("--tar_filename", - default=None, - help="The output file.") - return result +def BuildOptions(): + result = optparse.OptionParser() + result.add_option( + "-v", + "--verbose", + help='Verbose output.', + default=False, + action="store_true") + result.add_option("--tar_filename", default=None, help="The output file.") + + return result + def Filter(tar_info): - # Get the name of the file relative to the dart directory. Note the - # name from the TarInfo does not include a leading slash. - assert tar_info.name.startswith(DART_DIR[1:]) - original_name = tar_info.name[len(DART_DIR):] - _, tail = split(original_name) - if tail in ignoredDirs: - return None - for path in ignoredPaths: - if original_name.startswith(path): - return None - for ending in ignoredEndings: - if original_name.endswith(ending): - return None - # Add the dart directory name with version. Place the debian - # directory one level over the rest which are placed in the - # directory 'dart'. This enables building the Debian packages - # out-of-the-box. - tar_info.name = join(versiondir, 'dart', original_name) - if verbose: - print 'Adding %s as %s' % (original_name, tar_info.name) - return tar_info + # Get the name of the file relative to the dart directory. Note the + # name from the TarInfo does not include a leading slash. + assert tar_info.name.startswith(DART_DIR[1:]) + original_name = tar_info.name[len(DART_DIR):] + _, tail = split(original_name) + if tail in ignoredDirs: + return None + for path in ignoredPaths: + if original_name.startswith(path): + return None + for ending in ignoredEndings: + if original_name.endswith(ending): + return None + # Add the dart directory name with version. Place the debian + # directory one level over the rest which are placed in the + # directory 'dart'. This enables building the Debian packages + # out-of-the-box. + tar_info.name = join(versiondir, 'dart', original_name) + if verbose: + print 'Adding %s as %s' % (original_name, tar_info.name) + return tar_info + def GenerateCopyright(filename): - with open(join(DART_DIR, 'LICENSE')) as lf: - license_lines = lf.readlines() + with open(join(DART_DIR, 'LICENSE')) as lf: + license_lines = lf.readlines() + + with open(filename, 'w') as f: + f.write('Name: dart\n') + f.write('Maintainer: Dart Team \n') + f.write('Source: https://code.google.com/p/dart/\n') + f.write('License:\n') + for line in license_lines: + f.write(' %s' % line) # Line already contains trailing \n. - with open(filename, 'w') as f: - f.write('Name: dart\n') - f.write('Maintainer: Dart Team \n') - f.write('Source: https://code.google.com/p/dart/\n') - f.write('License:\n') - for line in license_lines: - f.write(' %s' % line) # Line already contains trailing \n. def GenerateChangeLog(filename, version): - with open(filename, 'w') as f: - f.write('dart (%s-1) UNRELEASED; urgency=low\n' % version) - f.write('\n') - f.write(' * Generated file.\n') - f.write('\n') - f.write(' -- Dart Team %s\n' % - datetime.datetime.utcnow().strftime('%a, %d %b %Y %X +0000')) + with open(filename, 'w') as f: + f.write('dart (%s-1) UNRELEASED; urgency=low\n' % version) + f.write('\n') + f.write(' * Generated file.\n') + f.write('\n') + f.write(' -- Dart Team %s\n' % + datetime.datetime.utcnow().strftime('%a, %d %b %Y %X +0000')) + def GenerateEmpty(filename): - f = open(filename, 'w') - f.close() + f = open(filename, 'w') + f.close() + def GenerateGitRevision(filename, git_revision): - with open(filename, 'w') as f: - f.write(str(git_revision)) + with open(filename, 'w') as f: + f.write(str(git_revision)) def CreateTarball(tarfilename): - global ignoredPaths # Used for adding the output directory. - # Generate the name of the tarfile - version = utils.GetVersion() - global versiondir - versiondir = 'dart-%s' % version - debian_dir = 'tools/linux_dist_support/debian' - # Don't include the build directory in the tarball (ignored paths - # are relative to DART_DIR). - builddir = utils.GetBuildDir(HOST_OS) - ignoredPaths.append(builddir) + global ignoredPaths # Used for adding the output directory. + # Generate the name of the tarfile + version = utils.GetVersion() + global versiondir + versiondir = 'dart-%s' % version + debian_dir = 'tools/linux_dist_support/debian' + # Don't include the build directory in the tarball (ignored paths + # are relative to DART_DIR). + builddir = utils.GetBuildDir(HOST_OS) + ignoredPaths.append(builddir) - print 'Creating tarball: %s' % tarfilename - with tarfile.open(tarfilename, mode='w:gz') as tar: - for f in listdir(DART_DIR): - tar.add(join(DART_DIR, f), filter=Filter) - for f in listdir(join(DART_DIR, debian_dir)): - tar.add(join(DART_DIR, debian_dir, f), - arcname='%s/debian/%s' % (versiondir, f)) + print 'Creating tarball: %s' % tarfilename + with tarfile.open(tarfilename, mode='w:gz') as tar: + for f in listdir(DART_DIR): + tar.add(join(DART_DIR, f), filter=Filter) + for f in listdir(join(DART_DIR, debian_dir)): + tar.add( + join(DART_DIR, debian_dir, f), + arcname='%s/debian/%s' % (versiondir, f)) - with utils.TempDir() as temp_dir: - # Generate and add debian/copyright - copyright_file = join(temp_dir, 'copyright') - GenerateCopyright(copyright_file) - tar.add(copyright_file, arcname='%s/debian/copyright' % versiondir) + with utils.TempDir() as temp_dir: + # Generate and add debian/copyright + copyright_file = join(temp_dir, 'copyright') + GenerateCopyright(copyright_file) + tar.add(copyright_file, arcname='%s/debian/copyright' % versiondir) - # Generate and add debian/changelog - change_log = join(temp_dir, 'changelog') - GenerateChangeLog(change_log, version) - tar.add(change_log, arcname='%s/debian/changelog' % versiondir) + # Generate and add debian/changelog + change_log = join(temp_dir, 'changelog') + GenerateChangeLog(change_log, version) + tar.add(change_log, arcname='%s/debian/changelog' % versiondir) - # For generated version file build dependency, add fake git reflog. - empty = join(temp_dir, 'empty') - GenerateEmpty(empty) - tar.add(empty, arcname='%s/dart/.git/logs/HEAD' % versiondir) + # For generated version file build dependency, add fake git reflog. + empty = join(temp_dir, 'empty') + GenerateEmpty(empty) + tar.add(empty, arcname='%s/dart/.git/logs/HEAD' % versiondir) + + # For bleeding_edge add the GIT_REVISION file. + if utils.GetChannel() == 'be': + git_revision = join(temp_dir, 'GIT_REVISION') + GenerateGitRevision(git_revision, utils.GetGitRevision()) + tar.add( + git_revision, + arcname='%s/dart/tools/GIT_REVISION' % versiondir) - # For bleeding_edge add the GIT_REVISION file. - if utils.GetChannel() == 'be': - git_revision = join(temp_dir, 'GIT_REVISION') - GenerateGitRevision(git_revision, utils.GetGitRevision()) - tar.add(git_revision, arcname='%s/dart/tools/GIT_REVISION' % versiondir) def Main(): - if HOST_OS != 'linux': - print 'Tarball can only be created on linux' - return -1 + if HOST_OS != 'linux': + print 'Tarball can only be created on linux' + return -1 - # Parse the options. - parser = BuildOptions() - (options, args) = parser.parse_args() - if options.verbose: - global verbose - verbose = True + # Parse the options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if options.verbose: + global verbose + verbose = True - tar_filename = options.tar_filename - if not tar_filename: - tar_filename = join(DART_DIR, - utils.GetBuildDir(HOST_OS), - 'dart-%s.tar.gz' % utils.GetVersion()) + tar_filename = options.tar_filename + if not tar_filename: + tar_filename = join(DART_DIR, utils.GetBuildDir(HOST_OS), + 'dart-%s.tar.gz' % utils.GetVersion()) + + CreateTarball(tar_filename) - CreateTarball(tar_filename) if __name__ == '__main__': - sys.exit(Main()) + sys.exit(Main()) diff --git a/tools/create_timestamp_file.py b/tools/create_timestamp_file.py index 0a97edcc24c..8f449a6f918 100755 --- a/tools/create_timestamp_file.py +++ b/tools/create_timestamp_file.py @@ -6,12 +6,14 @@ import sys import os + def main(args): - for file_name in args[1:]: - dir_name = os.path.dirname(file_name) - if not os.path.exists(dir_name): - os.mkdir(dir_name) - open(file_name, 'w').close() + for file_name in args[1:]: + dir_name = os.path.dirname(file_name) + if not os.path.exists(dir_name): + os.mkdir(dir_name) + open(file_name, 'w').close() + if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/tools/disguised_test.py b/tools/disguised_test.py index 40e8a0ca822..e3d8f0a88af 100755 --- a/tools/disguised_test.py +++ b/tools/disguised_test.py @@ -16,7 +16,8 @@ import os import subprocess import sys -exit(subprocess.call([sys.executable, - os.path.join(os.path.dirname(os.path.abspath(__file__)), - "test.py")] + - sys.argv[1:])) +exit( + subprocess.call([ + sys.executable, + os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.py") + ] + sys.argv[1:])) diff --git a/tools/dom/PRESUBMIT.py b/tools/dom/PRESUBMIT.py index 84ce19739a2..f6a64395838 100644 --- a/tools/dom/PRESUBMIT.py +++ b/tools/dom/PRESUBMIT.py @@ -1,7 +1,6 @@ # Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """ Presubmit tests for dom tools. @@ -11,50 +10,50 @@ any files at this level or lower are in the change list. See: http://www.chromium.org/developers/how-tos/depottools/presubmit-scripts """ - import os def _AnySdkFiles(input_api): - """ Returns true if any of the changed files are in the sdk, meaning we should + """ Returns true if any of the changed files are in the sdk, meaning we should check that docs.dart was run. """ - for f in input_api.change.AffectedFiles(): - if f.LocalPath().find('sdk') > -1: - return True - return False + for f in input_api.change.AffectedFiles(): + if f.LocalPath().find('sdk') > -1: + return True + return False def CheckChangeOnUpload(input_api, output_api): - results = [] - # TODO(amouravski): uncomment this check once docs.dart is faster. - # if _AnySdkFiles(input_api): - # results.extend(CheckDocs(input_api, output_api)) - return results + results = [] + # TODO(amouravski): uncomment this check once docs.dart is faster. + # if _AnySdkFiles(input_api): + # results.extend(CheckDocs(input_api, output_api)) + return results def CheckChangeOnCommit(input_api, output_api): - results = [] - if _AnySdkFiles(input_api): - results.extend(CheckDocs(input_api, output_api)) - return results + results = [] + if _AnySdkFiles(input_api): + results.extend(CheckDocs(input_api, output_api)) + return results def CheckDocs(input_api, output_api): - """Ensure that documentation has been generated if it needs to be generated. + """Ensure that documentation has been generated if it needs to be generated. Prompts with a warning if documentation needs to be generated. """ - results = [] + results = [] - cmd = [os.path.join(input_api.PresubmitLocalPath(), 'dom.py'), 'test_docs'] + cmd = [os.path.join(input_api.PresubmitLocalPath(), 'dom.py'), 'test_docs'] - try: - input_api.subprocess.check_output(cmd, - stderr=input_api.subprocess.STDOUT) - except (OSError, input_api.subprocess.CalledProcessError), e: - results.append(output_api.PresubmitPromptWarning( - ('Docs test failed!%s\nYou should run `dom.py docs`' % ( - e if input_api.verbose else '')))) + try: + input_api.subprocess.check_output( + cmd, stderr=input_api.subprocess.STDOUT) + except (OSError, input_api.subprocess.CalledProcessError), e: + results.append( + output_api.PresubmitPromptWarning( + ('Docs test failed!%s\nYou should run `dom.py docs`' % + (e if input_api.verbose else '')))) - return results + return results diff --git a/tools/dom/dom.py b/tools/dom/dom.py index 1316ab1e3ff..8fa8254a760 100755 --- a/tools/dom/dom.py +++ b/tools/dom/dom.py @@ -16,199 +16,227 @@ import utils dart_out_dir = utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32') if utils.IsWindows(): - dart_bin = os.path.join(dart_out_dir, 'dart.exe') + dart_bin = os.path.join(dart_out_dir, 'dart.exe') else: - dart_bin = os.path.join(dart_out_dir, 'dart') + dart_bin = os.path.join(dart_out_dir, 'dart') + +dart_dir = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.path.pardir, + os.path.pardir)) -dart_dir = os.path.abspath(os.path.join( - os.path.dirname(os.path.realpath(__file__)), - os.path.pardir, os.path.pardir)) def help(): - print('Helper script to make it easy to perform common tasks encountered ' - 'during the life of a Dart DOM developer.\n' - '\n' - 'For example, to re-generate DOM classes then run a specific test:\n' - ' dom.py gen test_drt html/element_test\n' - '\n' - 'Or re-generate DOM classes and run the Dart analyzer:\n' - ' dom.py gen analyze\n') - print('Commands: ') - for cmd in sorted(commands.keys()): - print('\t%s - %s' % (cmd, commands[cmd][1])) + print('Helper script to make it easy to perform common tasks encountered ' + 'during the life of a Dart DOM developer.\n' + '\n' + 'For example, to re-generate DOM classes then run a specific test:\n' + ' dom.py gen test_drt html/element_test\n' + '\n' + 'Or re-generate DOM classes and run the Dart analyzer:\n' + ' dom.py gen analyze\n') + print('Commands: ') + for cmd in sorted(commands.keys()): + print('\t%s - %s' % (cmd, commands[cmd][1])) + def analyze(): - ''' Runs the dart analyzer. ''' - return call([ - os.path.join(dart_out_dir, 'dart-sdk', 'bin', 'dartanalyzer'), - os.path.join('tests', 'html', 'element_test.dart'), - '--dart-sdk=sdk', - '--show-package-warnings', - ]) + ''' Runs the dart analyzer. ''' + return call([ + os.path.join(dart_out_dir, 'dart-sdk', 'bin', 'dartanalyzer'), + os.path.join('tests', 'html', 'element_test.dart'), + '--dart-sdk=sdk', + '--show-package-warnings', + ]) + def build(): - ''' Builds the Dart binary ''' - return call([ - os.path.join('tools', 'build.py'), - '--mode=release', - '--arch=ia32', - 'runtime', - ]) + ''' Builds the Dart binary ''' + return call([ + os.path.join('tools', 'build.py'), + '--mode=release', + '--arch=ia32', + 'runtime', + ]) + def dart2js(): - compile_dart2js(argv.pop(0), True) + compile_dart2js(argv.pop(0), True) + def docs(): - return call([ - os.path.join(dart_out_dir, 'dart-sdk', 'bin', 'dart'), - '--package-root=%s' % os.path.join(dart_out_dir, 'packages/'), - os.path.join('tools', 'dom', 'docs', 'bin', 'docs.dart'), - ]) + return call([ + os.path.join(dart_out_dir, 'dart-sdk', 'bin', 'dart'), + '--package-root=%s' % os.path.join(dart_out_dir, 'packages/'), + os.path.join('tools', 'dom', 'docs', 'bin', 'docs.dart'), + ]) + def test_docs(): - return call([ - os.path.join('tools', 'test.py'), - '--mode=release', - '--checked', - 'docs' - ]) + return call([ + os.path.join('tools', 'test.py'), '--mode=release', '--checked', 'docs' + ]) + def compile_dart2js(dart_file, checked): - out_file = dart_file + '.js' - dart2js_path = os.path.join(dart_out_dir, 'dart-sdk', 'bin', 'dart2js') - args = [ - dart2js_path, - dart_file, - '--library-root=sdk/', - '-o%s' % out_file - ] - if checked: - args.append('--checked') + out_file = dart_file + '.js' + dart2js_path = os.path.join(dart_out_dir, 'dart-sdk', 'bin', 'dart2js') + args = [dart2js_path, dart_file, '--library-root=sdk/', '-o%s' % out_file] + if checked: + args.append('--checked') + + call(args) + return out_file - call(args) - return out_file def gen(): - os.chdir(os.path.join('tools', 'dom', 'scripts')) - result = call([os.path.join(os.getcwd(), 'dartdomgenerator.py'), - '--rebuild', '--parallel', '--systems=htmldart2js,htmldartium']) - os.chdir(os.path.join('..', '..', '..')) - return result + os.chdir(os.path.join('tools', 'dom', 'scripts')) + result = call([ + os.path.join(os.getcwd(), 'dartdomgenerator.py'), '--rebuild', + '--parallel', '--systems=htmldart2js,htmldartium' + ]) + os.chdir(os.path.join('..', '..', '..')) + return result + def size_check(): - ''' Displays the dart2js size of swarm. ''' - dart_file = os.path.join('samples', 'swarm', 'swarm.dart') - out_file = compile_dart2js(dart_file, False) + ''' Displays the dart2js size of swarm. ''' + dart_file = os.path.join('samples', 'swarm', 'swarm.dart') + out_file = compile_dart2js(dart_file, False) - return call([ - 'du', - '-kh', - '--apparent-size', - out_file, - ]) + return call([ + 'du', + '-kh', + '--apparent-size', + out_file, + ]) + + os.remove(out_file) + os.remove(out_file + '.deps') + os.remove(out_file + '.map') - os.remove(out_file) - os.remove(out_file + '.deps') - os.remove(out_file + '.map') def test_ff(): - test_dart2js('ff', argv) + test_dart2js('ff', argv) + def test_drt(): - test_dart2js('drt', argv) + test_dart2js('drt', argv) + def test_chrome(): - test_dart2js('chrome', argv) + test_dart2js('chrome', argv) + def test_dart2js(browser, argv): - cmd = [ - os.path.join('tools', 'test.py'), - '-c', 'dart2js', - '-r', browser, - '--mode=release', - '--checked', - '--arch=ia32', - '-v', - ] - if argv: - cmd.append(argv.pop(0)) - else: - print( - 'Test commands should be followed by tests to run. Defaulting to html') - cmd.append('html') - return call(cmd) + cmd = [ + os.path.join('tools', 'test.py'), + '-c', + 'dart2js', + '-r', + browser, + '--mode=release', + '--checked', + '--arch=ia32', + '-v', + ] + if argv: + cmd.append(argv.pop(0)) + else: + print( + 'Test commands should be followed by tests to run. Defaulting to html' + ) + cmd.append('html') + return call(cmd) + def test_server(): - start_test_server(5400, os.path.join('out', 'ReleaseX64')) + start_test_server(5400, os.path.join('out', 'ReleaseX64')) + def test_server_dartium(): - start_test_server(5500, os.path.join('..', 'out', 'Release')) + start_test_server(5500, os.path.join('..', 'out', 'Release')) + def start_test_server(port, build_directory): - print('Browse tests at ' - '\033[94mhttp://localhost:%d/root_build/generated_tests/\033[0m' % port) - return call([ - utils.CheckedInSdkExecutable(), - os.path.join('tools', 'testing', 'dart', 'http_server.dart'), - '--port=%d' % port, - '--crossOriginPort=%d' % (port + 1), - '--network=0.0.0.0', - '--build-directory=%s' % build_directory - ]) + print( + 'Browse tests at ' + '\033[94mhttp://localhost:%d/root_build/generated_tests/\033[0m' % port) + return call([ + utils.CheckedInSdkExecutable(), + os.path.join('tools', 'testing', 'dart', 'http_server.dart'), + '--port=%d' % port, + '--crossOriginPort=%d' % (port + 1), '--network=0.0.0.0', + '--build-directory=%s' % build_directory + ]) def call(args): - print ' '.join(args) - pipe = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = pipe.communicate() - if output: - print output - if error: - print error - return pipe.returncode + print ' '.join(args) + pipe = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = pipe.communicate() + if output: + print output + if error: + print error + return pipe.returncode + commands = { - 'analyze': [analyze, 'Run the dart analyzer'], - 'build': [build, 'Build dart in release mode'], - 'dart2js': [dart2js, 'Run dart2js on the .dart file specified'], - 'docs': [docs, 'Generates docs.json'], - 'gen': [gen, 'Re-generate DOM generated files (run go.sh)'], - 'size_check': [size_check, 'Check the size of dart2js compiled Swarm'], - 'test_docs': [test_docs, 'Tests docs.dart'], - 'test_chrome': [test_chrome, 'Run tests in checked mode in Chrome.\n' - '\t\tOptionally provide name of test to run.'], - # TODO(antonm): fix option name. - 'test_drt': [test_drt, 'Run tests in checked mode in content shell.\n' - '\t\tOptionally provide name of test to run.'], - 'test_ff': [test_ff, 'Run tests in checked mode in Firefox.\n' - '\t\tOptionally provide name of test to run.'], - 'test_server': [test_server, 'Starts the testing server for manually ' - 'running browser tests.'], - 'test_server_dartium': [test_server_dartium, 'Starts the testing server for ' - 'manually running browser tests from a dartium enlistment.'], + 'analyze': [analyze, 'Run the dart analyzer'], + 'build': [build, 'Build dart in release mode'], + 'dart2js': [dart2js, 'Run dart2js on the .dart file specified'], + 'docs': [docs, 'Generates docs.json'], + 'gen': [gen, 'Re-generate DOM generated files (run go.sh)'], + 'size_check': [size_check, 'Check the size of dart2js compiled Swarm'], + 'test_docs': [test_docs, 'Tests docs.dart'], + 'test_chrome': [ + test_chrome, 'Run tests in checked mode in Chrome.\n' + '\t\tOptionally provide name of test to run.' + ], + # TODO(antonm): fix option name. + 'test_drt': [ + test_drt, 'Run tests in checked mode in content shell.\n' + '\t\tOptionally provide name of test to run.' + ], + 'test_ff': [ + test_ff, 'Run tests in checked mode in Firefox.\n' + '\t\tOptionally provide name of test to run.' + ], + 'test_server': [ + test_server, 'Starts the testing server for manually ' + 'running browser tests.' + ], + 'test_server_dartium': [ + test_server_dartium, 'Starts the testing server for ' + 'manually running browser tests from a dartium enlistment.' + ], } + def main(): - success = True - argv.pop(0) + success = True + argv.pop(0) - if not argv: - help() - success = False + if not argv: + help() + success = False - while (argv): - # Make sure that we're always rooted in the dart root folder. - os.chdir(dart_dir) - command = argv.pop(0) + while (argv): + # Make sure that we're always rooted in the dart root folder. + os.chdir(dart_dir) + command = argv.pop(0) - if not command in commands: - help(); - success = False - break - returncode = commands[command][0]() - success = success and not bool(returncode) + if not command in commands: + help() + success = False + break + returncode = commands[command][0]() + success = success and not bool(returncode) + + sys.exit(not success) - sys.exit(not success) if __name__ == '__main__': - main() + main() diff --git a/tools/dom/new_scripts/code_generator_dart.py b/tools/dom/new_scripts/code_generator_dart.py index c5a5709f578..6fdd2a95f67 100644 --- a/tools/dom/new_scripts/code_generator_dart.py +++ b/tools/dom/new_scripts/code_generator_dart.py @@ -25,7 +25,6 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Generate Blink C++ bindings (.h and .cpp files) for use by Dart:HTML. If run itself, caches Jinja templates (and creates dummy file for build, @@ -48,7 +47,6 @@ import cPickle as pickle import re import sys - # Path handling for libraries and templates # Paths have to be normalized because Jinja uses the exact template path to # determine the hash used in the cache filename, and we need a pre-caching step @@ -58,8 +56,9 @@ import sys # is regenerated, which causes a race condition and breaks concurrent build, # since some compile processes will try to read the partially written cache. module_path, module_filename = os.path.split(os.path.realpath(__file__)) -third_party_dir = os.path.normpath(os.path.join( - module_path, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir)) +third_party_dir = os.path.normpath( + os.path.join(module_path, os.pardir, os.pardir, os.pardir, os.pardir, + os.pardir)) templates_dir = os.path.normpath(os.path.join(module_path, 'templates')) # Make sure extension is .py, not .pyc or .pyo, so doesn't depend on caching @@ -72,8 +71,8 @@ sys.path.insert(1, third_party_dir) # Add the base compiler scripts to the path here as in compiler.py dart_script_path = os.path.dirname(os.path.abspath(__file__)) -script_path = os.path.join(os.path.dirname(os.path.dirname(dart_script_path)), - 'scripts') +script_path = os.path.join( + os.path.dirname(os.path.dirname(dart_script_path)), 'scripts') sys.path.extend([script_path]) import jinja2 @@ -84,44 +83,41 @@ from utilities import write_pickle_file from v8_globals import includes from dart_utilities import DartUtilities - # TODO(jacobr): remove this hacked together list. INTERFACES_WITHOUT_RESOLVERS = frozenset([ - 'TypeConversions', - 'GCObservation', - 'InternalProfilers', - 'InternalRuntimeFlags', - 'InternalSettings', - 'InternalSettingsGenerated', - 'Internals', - 'LayerRect', - 'LayerRectList', - 'MallocStatistics', - 'TypeConversions']) + 'TypeConversions', 'GCObservation', 'InternalProfilers', + 'InternalRuntimeFlags', 'InternalSettings', 'InternalSettingsGenerated', + 'Internals', 'LayerRect', 'LayerRectList', 'MallocStatistics', + 'TypeConversions' +]) + class CodeGeneratorDart(object): + def __init__(self, interfaces_info, cache_dir): interfaces_info = interfaces_info or {} self.interfaces_info = interfaces_info self.jinja_env = initialize_jinja_env(cache_dir) # Set global type info - idl_types.set_ancestors(dict( - (interface_name, interface_info['ancestors']) - for interface_name, interface_info in interfaces_info.iteritems() - if interface_info['ancestors'])) - IdlType.set_callback_interfaces(set( - interface_name - for interface_name, interface_info in interfaces_info.iteritems() - if interface_info['is_callback_interface'])) - IdlType.set_implemented_as_interfaces(dict( - (interface_name, interface_info['implemented_as']) - for interface_name, interface_info in interfaces_info.iteritems() - if interface_info['implemented_as'])) - IdlType.set_garbage_collected_types(set( - interface_name - for interface_name, interface_info in interfaces_info.iteritems() - if 'GarbageCollected' in interface_info['inherited_extended_attributes'])) + idl_types.set_ancestors( + dict((interface_name, interface_info['ancestors']) + for interface_name, interface_info in interfaces_info. + iteritems() + if interface_info['ancestors'])) + IdlType.set_callback_interfaces( + set(interface_name for interface_name, interface_info in + interfaces_info.iteritems() + if interface_info['is_callback_interface'])) + IdlType.set_implemented_as_interfaces( + dict((interface_name, interface_info['implemented_as']) + for interface_name, interface_info in interfaces_info. + iteritems() + if interface_info['implemented_as'])) + IdlType.set_garbage_collected_types( + set(interface_name for interface_name, interface_info in + interfaces_info.iteritems() if 'GarbageCollected' in + interface_info['inherited_extended_attributes'])) def generate_code(self, definitions, interface_name, idl_pickle_filename, only_if_changed): @@ -158,7 +154,8 @@ class CodeGeneratorDart(object): # Add includes for interface itself and any dependencies interface_info = self.interfaces_info[interface_name] template_contents['header_includes'].add(interface_info['include_path']) - template_contents['header_includes'] = sorted(template_contents['header_includes']) + template_contents['header_includes'] = sorted( + template_contents['header_includes']) includes.update(interface_info.get('dependencies_include_paths', [])) # Remove includes that are not needed for Dart and trigger fatal @@ -181,20 +178,29 @@ class CodeGeneratorDart(object): idl_world['callback'] = idl_global_data['callback'] if 'interface_name' in template_contents: - interface_global = {'name': template_contents['interface_name'], - 'parent_interface': template_contents['parent_interface'], - 'is_active_dom_object': template_contents['is_active_dom_object'], - 'is_event_target': template_contents['is_event_target'], - 'has_resolver': template_contents['interface_name'] not in INTERFACES_WITHOUT_RESOLVERS, - 'is_node': template_contents['is_node'], - 'conditional_string': template_contents['conditional_string'], - } + interface_global = { + 'name': + template_contents['interface_name'], + 'parent_interface': + template_contents['parent_interface'], + 'is_active_dom_object': + template_contents['is_active_dom_object'], + 'is_event_target': + template_contents['is_event_target'], + 'has_resolver': + template_contents['interface_name'] not in + INTERFACES_WITHOUT_RESOLVERS, + 'is_node': + template_contents['is_node'], + 'conditional_string': + template_contents['conditional_string'], + } idl_world['interface'] = interface_global else: callback_global = {'name': template_contents['cpp_class']} idl_world['callback'] = callback_global - write_pickle_file(idl_pickle_filename, idl_world, only_if_changed) + write_pickle_file(idl_pickle_filename, idl_world, only_if_changed) # Render Jinja templates header_text = header_template.render(template_contents) @@ -232,8 +238,10 @@ class CodeGeneratorDart(object): world['callbacks'].append(idl_world['callback']) idl_pickle_file.close() - world['interfaces'] = sorted(world['interfaces'], key=lambda (x): x['name']) - world['callbacks'] = sorted(world['callbacks'], key=lambda (x): x['name']) + world['interfaces'] = sorted( + world['interfaces'], key=lambda (x): x['name']) + world['callbacks'] = sorted( + world['callbacks'], key=lambda (x): x['name']) template_contents = world template_contents['code_generator'] = module_pyname @@ -259,7 +267,7 @@ def initialize_jinja_env(cache_dir): 'blink_capitalize': DartUtilities.capitalize, 'conditional': conditional_if_endif, 'runtime_enabled': runtime_enabled_if, - }) + }) return jinja_env @@ -268,8 +276,7 @@ def conditional_if_endif(code, conditional_string): # Jinja2 filter to generate if/endif directive blocks if not conditional_string: return code - return ('#if %s\n' % conditional_string + - code + + return ('#if %s\n' % conditional_string + code + '#endif // %s\n' % conditional_string) @@ -285,6 +292,7 @@ def runtime_enabled_if(code, runtime_enabled_function_name): ################################################################################ + def main(argv): # If file itself executed, cache templates try: @@ -296,9 +304,11 @@ def main(argv): # Cache templates jinja_env = initialize_jinja_env(cache_dir) - template_filenames = [filename for filename in os.listdir(templates_dir) - # Skip .svn, directories, etc. - if filename.endswith(('.cpp', '.h', '.template'))] + template_filenames = [ + filename for filename in os.listdir(templates_dir) + # Skip .svn, directories, etc. + if filename.endswith(('.cpp', '.h', '.template')) + ] for template_filename in template_filenames: jinja_env.get_template(template_filename) diff --git a/tools/dom/new_scripts/compiler.py b/tools/dom/new_scripts/compiler.py index 2a6d09a9831..9a27581b386 100755 --- a/tools/dom/new_scripts/compiler.py +++ b/tools/dom/new_scripts/compiler.py @@ -26,7 +26,6 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Compile an .idl file to Dart bindings (.h and .cpp files). Design doc: ?????? @@ -37,8 +36,8 @@ import os import sys dart_script_path = os.path.dirname(os.path.abspath(__file__)) -script_path = os.path.join(os.path.dirname(os.path.dirname(dart_script_path)), - 'scripts') +script_path = os.path.join( + os.path.dirname(os.path.dirname(dart_script_path)), 'scripts') sys.path.extend([script_path]) from dart_compiler import IdlCompiler @@ -58,7 +57,8 @@ def parse_options(): options, args = parser.parse_args() if options.output_directory is None: parser.error('Must specify output directory using --output-directory.') - options.write_file_only_if_changed = bool(options.write_file_only_if_changed) + options.write_file_only_if_changed = bool( + options.write_file_only_if_changed) options.generate_global = bool(options.generate_global) if len(args) != 1: # parser.error('Must specify exactly 1 input file as argument, but %d given.' % len(args)) @@ -74,13 +74,15 @@ def idl_filename_to_interface_name(idl_filename): class IdlCompilerDart(IdlCompiler): + def __init__(self, *args, **kwargs): IdlCompiler.__init__(self, *args, **kwargs) interfaces_info = self.interfaces_info self.output_directory = self.output_directory - self.code_generator = CodeGeneratorDart(interfaces_info, self.output_directory) + self.code_generator = CodeGeneratorDart(interfaces_info, + self.output_directory) def compile_file(self, idl_filename): interface_name = idl_filename_to_interface_name(idl_filename) @@ -88,26 +90,32 @@ class IdlCompilerDart(IdlCompiler): 'Dart%s.h' % interface_name) cpp_filename = os.path.join(self.output_directory, 'Dart%s.cpp' % interface_name) - return self.compile_and_write(idl_filename, (header_filename, cpp_filename)) + return self.compile_and_write(idl_filename, + (header_filename, cpp_filename)) def generate_global(self): - global_header_filename = os.path.join(self.output_directory, 'DartWebkitClassIds.h') - global_cpp_filename = os.path.join(self.output_directory, 'DartWebkitClassIds.cpp') - self.generate_global_and_write((global_header_filename, global_cpp_filename)) + global_header_filename = os.path.join(self.output_directory, + 'DartWebkitClassIds.h') + global_cpp_filename = os.path.join(self.output_directory, + 'DartWebkitClassIds.cpp') + self.generate_global_and_write((global_header_filename, + global_cpp_filename)) def main(): options, idl_filename = parse_options() if options.generate_global: - idl_compiler = IdlCompilerDart(options.output_directory, - interfaces_info_filename=options.interfaces_info_file, - only_if_changed=options.write_file_only_if_changed) + idl_compiler = IdlCompilerDart( + options.output_directory, + interfaces_info_filename=options.interfaces_info_file, + only_if_changed=options.write_file_only_if_changed) idl_compiler.generate_global() else: - idl_compiler = IdlCompilerDart(options.output_directory, - interfaces_info_filename=options.interfaces_info_file, - only_if_changed=options.write_file_only_if_changed) + idl_compiler = IdlCompilerDart( + options.output_directory, + interfaces_info_filename=options.interfaces_info_file, + only_if_changed=options.write_file_only_if_changed) idl_compiler.compile_file(idl_filename) diff --git a/tools/dom/new_scripts/dart_compiler.py b/tools/dom/new_scripts/dart_compiler.py index b766fb39ac9..5dd2b348c60 100755 --- a/tools/dom/new_scripts/dart_compiler.py +++ b/tools/dom/new_scripts/dart_compiler.py @@ -26,7 +26,6 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Compile an .idl file to Blink C++ bindings (.h and .cpp files) for Dart:HTML. Design doc: http://www.chromium.org/developers/design-documents/idl-compiler @@ -40,7 +39,6 @@ import cPickle as pickle from idl_reader import IdlReader from utilities import write_file - # TODO(terry): Temporary whitelist of IDL files to skip code generating. e.g., # adding 'Animation.idl' to this list will skip that IDL file. SKIP_IDL_FILES = [''] @@ -48,8 +46,9 @@ SKIP_IDL_FILES = [''] def parse_options(): parser = OptionParser() - parser.add_option('--idl-attributes-file', - help="location of bindings/IDLExtendedAttributes.txt") + parser.add_option( + '--idl-attributes-file', + help="location of bindings/IDLExtendedAttributes.txt") parser.add_option('--output-directory') parser.add_option('--interfaces-info-file') parser.add_option('--write-file-only-if-changed', type='int') @@ -59,9 +58,12 @@ def parse_options(): options, args = parser.parse_args() if options.output_directory is None: parser.error('Must specify output directory using --output-directory.') - options.write_file_only_if_changed = bool(options.write_file_only_if_changed) + options.write_file_only_if_changed = bool( + options.write_file_only_if_changed) if len(args) != 1: - parser.error('Must specify exactly 1 input file as argument, but %d given.' % len(args)) + parser.error( + 'Must specify exactly 1 input file as argument, but %d given.' % + len(args)) idl_filename = os.path.realpath(args[0]) return options, idl_filename @@ -82,8 +84,11 @@ class IdlCompiler(object): """ __metaclass__ = abc.ABCMeta - def __init__(self, output_directory, code_generator=None, - interfaces_info=None, interfaces_info_filename='', + def __init__(self, + output_directory, + code_generator=None, + interfaces_info=None, + interfaces_info_filename='', only_if_changed=False): """ Args: diff --git a/tools/dom/new_scripts/dart_utilities.py b/tools/dom/new_scripts/dart_utilities.py index 7a57a4fdb24..73df127c815 100644 --- a/tools/dom/new_scripts/dart_utilities.py +++ b/tools/dom/new_scripts/dart_utilities.py @@ -25,7 +25,6 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Functions shared by various parts of the code generator. Extends IdlType and IdlUnion type with |enum_validation_expression| property. @@ -33,13 +32,11 @@ Extends IdlType and IdlUnion type with |enum_validation_expression| property. Design doc: http://www.chromium.org/developers/design-documents/idl-compiler """ - ################################################################################ # Utility function exposed for Dart CodeGenerator. Only 6 methods are special # to Dart the rest delegate to the v8_utilities functions. ################################################################################ - import v8_types # Required import v8_utilities @@ -47,11 +44,12 @@ import v8_utilities def _scoped_name(interface, definition, base_name): # partial interfaces are implemented as separate classes, with their members # implemented as static member functions - partial_interface_implemented_as = definition.extended_attributes.get('PartialInterfaceImplementedAs') + partial_interface_implemented_as = definition.extended_attributes.get( + 'PartialInterfaceImplementedAs') if partial_interface_implemented_as: return '%s::%s' % (partial_interface_implemented_as, base_name) if (definition.is_static or - definition.name in ('Constructor', 'NamedConstructor')): + definition.name in ('Constructor', 'NamedConstructor')): return '%s::%s' % (v8_utilities.cpp_name(interface), base_name) return 'receiver->%s' % base_name @@ -104,12 +102,16 @@ _CALL_WITH_VALUES = [ def _call_with_arguments(member, call_with_values=None): # Optional parameter so setter can override with [SetterCallWith] - call_with_values = call_with_values or member.extended_attributes.get('CallWith') + call_with_values = call_with_values or member.extended_attributes.get( + 'CallWith') if not call_with_values: return [] - return [_CALL_WITH_ARGUMENTS[value] - for value in _CALL_WITH_VALUES - if v8_utilities.extended_attribute_value_contains(call_with_values, value)] + return [ + _CALL_WITH_ARGUMENTS[value] + for value in _CALL_WITH_VALUES + if v8_utilities.extended_attribute_value_contains( + call_with_values, value) + ] # [DeprecateAs] @@ -139,9 +141,11 @@ def _measure_as(definition_or_member): class dart_utilities_monkey(): + def __init__(self): self.base_class_name = 'dart_utilities' + DartUtilities = dart_utilities_monkey() DartUtilities.activity_logging_world_list = _activity_logging_world_list diff --git a/tools/dom/new_scripts/dependency.py b/tools/dom/new_scripts/dependency.py index 060277ed02d..4f54d448010 100644 --- a/tools/dom/new_scripts/dependency.py +++ b/tools/dom/new_scripts/dependency.py @@ -1,9 +1,11 @@ builder = None + def set_builder(created_builder): - global builder; - builder = created_builder + global builder + builder = created_builder + def get_interfaces_info(): - global builder; - return builder._info_collector.interfaces_info + global builder + return builder._info_collector.interfaces_info diff --git a/tools/dom/scripts/all_tests.py b/tools/dom/scripts/all_tests.py index f7d84fd226c..40a5a49f896 100755 --- a/tools/dom/scripts/all_tests.py +++ b/tools/dom/scripts/all_tests.py @@ -2,23 +2,17 @@ # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """This entry point runs all script tests.""" import logging.config import unittest if __name__ == '__main__': - logging.config.fileConfig('logging.conf') - suite = unittest.TestLoader().loadTestsFromNames([ - 'templateloader_test', - 'pegparser_test', - 'idlparser_test', - 'idlnode_test', - 'idlrenderer_test', - 'database_test', - 'databasebuilder_test', - 'emitter_test', - 'dartgenerator_test', - 'multiemitter_test']) - unittest.TextTestRunner().run(suite) + logging.config.fileConfig('logging.conf') + suite = unittest.TestLoader().loadTestsFromNames([ + 'templateloader_test', 'pegparser_test', 'idlparser_test', + 'idlnode_test', 'idlrenderer_test', 'database_test', + 'databasebuilder_test', 'emitter_test', 'dartgenerator_test', + 'multiemitter_test' + ]) + unittest.TextTestRunner().run(suite) diff --git a/tools/dom/scripts/css_code_generator.py b/tools/dom/scripts/css_code_generator.py index 1d921efb128..4bf4870f464 100644 --- a/tools/dom/scripts/css_code_generator.py +++ b/tools/dom/scripts/css_code_generator.py @@ -3,7 +3,6 @@ # Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """Generates CSSStyleDeclaration template file from css property definitions defined in WebKit.""" @@ -28,65 +27,74 @@ BROWSER_PATHS = [ 'cssProperties.safari-7.1.3.txt', 'cssProperties.mobileSafari-8.2.txt', 'cssProperties.iPad4Air.onGoogleSites.txt', - ] +] # Supported annotations for any specific CSS properties. annotated = { - 'transition': '''@SupportedBrowser(SupportedBrowser.CHROME) + 'transition': + '''@SupportedBrowser(SupportedBrowser.CHROME) @SupportedBrowser(SupportedBrowser.FIREFOX) @SupportedBrowser(SupportedBrowser.IE, '10') @SupportedBrowser(SupportedBrowser.SAFARI)''' } + class Error: - def __init__(self, message): - self.message = message - def __repr__(self): - return self.message + + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + def camelCaseName(name): - """Convert a CSS property name to a lowerCamelCase name.""" - name = name.replace('-webkit-', '') - words = [] - for word in name.split('-'): - if words: - words.append(word.title()) - else: - words.append(word) - return ''.join(words) + """Convert a CSS property name to a lowerCamelCase name.""" + name = name.replace('-webkit-', '') + words = [] + for word in name.split('-'): + if words: + words.append(word.title()) + else: + words.append(word) + return ''.join(words) + def dashifyName(camelName): - def fix(match): - return '-' + match.group(0).lower() - return re.sub(r'[A-Z]', fix, camelName) + + def fix(match): + return '-' + match.group(0).lower() + + return re.sub(r'[A-Z]', fix, camelName) + def isCommentLine(line): - return line.strip() == '' or line.startswith('#') or line.startswith('//') + return line.strip() == '' or line.startswith('#') or line.startswith('//') + def readCssProperties(filename): - data = open(filename).readlines() - data = sorted([d.strip() for d in set(data) if not isCommentLine(d)]) - return data + data = open(filename).readlines() + data = sorted([d.strip() for d in set(data) if not isCommentLine(d)]) + return data + def GenerateCssTemplateFile(): - data = open(SOURCE_PATH).readlines() + data = open(SOURCE_PATH).readlines() - # filter CSSPropertyNames.in to only the properties - # TODO(efortuna): do we also want CSSPropertyNames.in? - data = [d.strip() for d in data - if not isCommentLine(d) - and not '=' in d] + # filter CSSPropertyNames.in to only the properties + # TODO(efortuna): do we also want CSSPropertyNames.in? + data = [d.strip() for d in data if not isCommentLine(d) and not '=' in d] - browser_props = [readCssProperties(file) for file in BROWSER_PATHS] - universal_properties = reduce( - lambda a, b: set(a).intersection(b), browser_props) - universal_properties = universal_properties.difference(['cssText']) - universal_properties = universal_properties.intersection( + browser_props = [readCssProperties(file) for file in BROWSER_PATHS] + universal_properties = reduce(lambda a, b: set(a).intersection(b), + browser_props) + universal_properties = universal_properties.difference(['cssText']) + universal_properties = universal_properties.intersection( map(camelCaseName, data)) - class_file = open(TEMPLATE_FILE, 'w') + class_file = open(TEMPLATE_FILE, 'w') - class_file.write(""" + class_file.write(""" // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. @@ -100,8 +108,7 @@ def GenerateCssTemplateFile(): part of $LIBRARYNAME; """ % SOURCE_PATH) - - class_file.write(""" + class_file.write(""" $(ANNOTATIONS)$(NATIVESPEC)$(CLASS_MODIFIERS)class $CLASSNAME $EXTENDS with $(CLASSNAME)Base $IMPLEMENTS { factory $CLASSNAME() => new CssStyleDeclaration.css(''); @@ -200,9 +207,9 @@ $(ANNOTATIONS)$(NATIVESPEC)$(CLASS_MODIFIERS)class $CLASSNAME $EXTENDS with $!MEMBERS """) - for camelName in sorted(universal_properties): - property = dashifyName(camelName) - class_file.write(""" + for camelName in sorted(universal_properties): + property = dashifyName(camelName) + class_file.write(""" /** Gets the value of "%s" */ String get %s => this._%s; @@ -213,11 +220,10 @@ $!MEMBERS @Returns('String') @JSName('%s') String _%s; - """ % (property, camelName, camelName, - property, camelName, camelName, + """ % (property, camelName, camelName, property, camelName, camelName, camelName, camelName)) - class_file.write(""" + class_file.write(""" } class _CssStyleDeclarationSet extends Object with CssStyleDeclarationBase { @@ -240,7 +246,7 @@ class _CssStyleDeclarationSet extends Object with CssStyleDeclarationBase { """) - class_file.write(""" + class_file.write(""" void _setAll(String propertyName, String value) { value = value == null ? '' : value; for (Element element in _elementIterable) { @@ -249,17 +255,16 @@ class _CssStyleDeclarationSet extends Object with CssStyleDeclarationBase { } """) - - for camelName in sorted(universal_properties): - property = dashifyName(camelName) - class_file.write(""" + for camelName in sorted(universal_properties): + property = dashifyName(camelName) + class_file.write(""" /** Sets the value of "%s" */ set %s(String value) { _setAll('%s', value); } """ % (property, camelName, camelName)) - class_file.write(""" + class_file.write(""" // Important note: CssStyleDeclarationSet does NOT implement every method // available in CssStyleDeclaration. Some of the methods don't make so much @@ -273,39 +278,39 @@ abstract class CssStyleDeclarationBase { void setProperty(String propertyName, String value, [String priority]); """) - class_lines = []; + class_lines = [] - seen = set() - for prop in sorted(data, key=camelCaseName): - camel_case_name = camelCaseName(prop) - upper_camel_case_name = camel_case_name[0].upper() + camel_case_name[1:]; - css_name = prop.replace('-webkit-', '') - base_css_name = prop.replace('-webkit-', '') + seen = set() + for prop in sorted(data, key=camelCaseName): + camel_case_name = camelCaseName(prop) + upper_camel_case_name = camel_case_name[0].upper() + camel_case_name[1:] + css_name = prop.replace('-webkit-', '') + base_css_name = prop.replace('-webkit-', '') - if base_css_name in seen or base_css_name.startswith('-internal'): - continue - seen.add(base_css_name) + if base_css_name in seen or base_css_name.startswith('-internal'): + continue + seen.add(base_css_name) - comment = ' /** %s the value of "' + base_css_name + '" */' - class_lines.append('\n'); - class_lines.append(comment % 'Gets') - if base_css_name in annotated: - class_lines.append(annotated[base_css_name]) - class_lines.append(""" + comment = ' /** %s the value of "' + base_css_name + '" */' + class_lines.append('\n') + class_lines.append(comment % 'Gets') + if base_css_name in annotated: + class_lines.append(annotated[base_css_name]) + class_lines.append(""" String get %s => getPropertyValue('%s'); """ % (camel_case_name, css_name)) - class_lines.append(comment % 'Sets') - if base_css_name in annotated: - class_lines.append(annotated[base_css_name]) - class_lines.append(""" + class_lines.append(comment % 'Sets') + if base_css_name in annotated: + class_lines.append(annotated[base_css_name]) + class_lines.append(""" set %s(String value) { setProperty('%s', value, ''); } """ % (camel_case_name, css_name)) - class_file.write(''.join(class_lines)); - class_file.write('}\n') - class_file.close() + class_file.write(''.join(class_lines)) + class_file.write('}\n') + class_file.close() diff --git a/tools/dom/scripts/dartdomgenerator.py b/tools/dom/scripts/dartdomgenerator.py index 91487677faa..3e1368e43b9 100755 --- a/tools/dom/scripts/dartdomgenerator.py +++ b/tools/dom/scripts/dartdomgenerator.py @@ -2,7 +2,6 @@ # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """This is the entry point to create Dart APIs from the IDL database.""" import css_code_generator @@ -13,9 +12,11 @@ import sys # dart_dir is the location of dart's enlistment dartium (dartium-git/src/dart) # and Dart (dart-git/dart). -dart_dir = os.path.abspath(os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))) +dart_dir = os.path.abspath( + os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))) sys.path.insert(1, os.path.join(dart_dir, 'tools/dom/new_scripts')) -sys.path.insert(1, os.path.join(dart_dir, 'third_party/WebCore/bindings/scripts')) +sys.path.insert(1, os.path.join(dart_dir, + 'third_party/WebCore/bindings/scripts')) # Dartium's third_party directory location is dartium-git/src/third_party # and Dart's third_party directory location is dart-git/dart/third_party. @@ -25,17 +26,17 @@ ply_dir = os.path.join(third_party_dir, 'ply') # If ply directory found then we're a Dart enlistment; third_party location # is dart-git/dart/third_party if not os.path.exists(ply_dir): - # For Dartium (ply directory is dartium-git/src/third_party/ply) third_party - # location is dartium-git/src/third_party - third_party_dir = os.path.join(dart_dir, '..', 'third_party') - assert(os.path.exists(third_party_dir)) + # For Dartium (ply directory is dartium-git/src/third_party/ply) third_party + # location is dartium-git/src/third_party + third_party_dir = os.path.join(dart_dir, '..', 'third_party') + assert (os.path.exists(third_party_dir)) else: - # It's Dart we need to make sure that tools in injected in our search path - # because this is where idl_parser is located for a Dart enlistment. Dartium - # can figure out the tools directory because of the location of where the - # scripts blink scripts are located. - tools_dir = os.path.join(dart_dir, 'tools') - sys.path.insert(1, tools_dir) + # It's Dart we need to make sure that tools in injected in our search path + # because this is where idl_parser is located for a Dart enlistment. Dartium + # can figure out the tools directory because of the location of where the + # scripts blink scripts are located. + tools_dir = os.path.join(dart_dir, 'tools') + sys.path.insert(1, tools_dir) sys.path.insert(1, third_party_dir) @@ -66,141 +67,153 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) import utils - _logger = logging.getLogger('dartdomgenerator') + class GeneratorOptions(object): - def __init__(self, templates, database, type_registry, renamer, - metadata, dart_js_interop): - self.templates = templates - self.database = database - self.type_registry = type_registry - self.renamer = renamer - self.metadata = metadata; - self.dart_js_interop = dart_js_interop + + def __init__(self, templates, database, type_registry, renamer, metadata, + dart_js_interop): + self.templates = templates + self.database = database + self.type_registry = type_registry + self.renamer = renamer + self.metadata = metadata + self.dart_js_interop = dart_js_interop + def LoadDatabase(database_dir, use_database_cache): - common_database = database.Database(database_dir) - if use_database_cache: - common_database.LoadFromCache() - else: - common_database.Load() - return common_database + common_database = database.Database(database_dir) + if use_database_cache: + common_database.LoadFromCache() + else: + common_database.Load() + return common_database + def GenerateFromDatabase(common_database, dart2js_output_dir, update_dom_metadata=False, - logging_level=logging.WARNING, dart_js_interop=False): - print '\n ----- Accessing DOM using %s -----\n' % ('dart:js' if dart_js_interop else 'C++') + logging_level=logging.WARNING, + dart_js_interop=False): + print '\n ----- Accessing DOM using %s -----\n' % ( + 'dart:js' if dart_js_interop else 'C++') - start_time = time.time() - - current_dir = os.path.dirname(__file__) - auxiliary_dir = os.path.join(current_dir, '..', 'src') - template_dir = os.path.join(current_dir, '..', 'templates') - - _logger.setLevel(logging_level) - - generator = dartgenerator.DartGenerator(logging_level) - generator.LoadAuxiliary(auxiliary_dir) - - generator.FilterMembersWithUnidentifiedTypes(common_database) - webkit_database = common_database.Clone() - - # Generate Dart interfaces for the WebKit DOM. - generator.FilterInterfaces(database = webkit_database, - or_annotations = ['WebKit', 'Dart'], - exclude_displaced = ['WebKit'], - exclude_suppressed = ['WebKit', 'Dart']) - generator.FixEventTargets(webkit_database) - generator.AddMissingArguments(webkit_database) - generator.CleanupOperationArguments(webkit_database) - - emitters = multiemitter.MultiEmitter(logging_level) - metadata = DartMetadata( - os.path.join(current_dir, '..', 'dom.json'), - os.path.join(current_dir, '..', 'docs', 'docs.json'), - logging_level) - renamer = HtmlRenamer(webkit_database, metadata) - type_registry = TypeRegistry(webkit_database, renamer) - - print 'GenerateFromDatabase %s seconds' % round((time.time() - start_time), 2) - - def RunGenerator(dart_libraries, dart_output_dir, - template_loader, backend_factory, dart_js_interop): - options = GeneratorOptions( - template_loader, webkit_database, type_registry, renamer, - metadata, dart_js_interop) - dart_library_emitter = DartLibraryEmitter( - emitters, dart_output_dir, dart_libraries) - event_generator = HtmlEventGenerator(webkit_database, renamer, metadata, - template_loader) - - def generate_interface(interface, gl_constants=None): - backend = backend_factory(interface) - interface_generator = HtmlDartInterfaceGenerator( - options, dart_library_emitter, event_generator, interface, backend) - interface_generator.Generate() - if len(backend._gl_constants) > 0 and not(gl_constants is None): - gl_constants.extend(backend._gl_constants) - - generator.Generate(webkit_database, common_database, generate_interface) - - dart_library_emitter.EmitLibraries(auxiliary_dir, dart_js_interop) - - if dart2js_output_dir: - template_paths = ['html/dart2js', 'html/impl', 'html/interface', ''] - template_loader = TemplateLoader(template_dir, - template_paths, - {'DARTIUM': False, - 'DART2JS': True, - 'JSINTEROP': False}) - backend_options = GeneratorOptions( - template_loader, webkit_database, type_registry, renamer, - metadata, dart_js_interop) - backend_factory = lambda interface:\ - Dart2JSBackend(interface, backend_options, logging_level) - - dart_output_dir = os.path.join(dart2js_output_dir, 'dart') - dart_libraries = DartLibraries( - HTML_LIBRARY_NAMES, template_loader, 'dart2js', dart2js_output_dir, dart_js_interop) - - print '\nGenerating dart2js:\n' start_time = time.time() - RunGenerator(dart_libraries, dart_output_dir, template_loader, - backend_factory, dart_js_interop) + current_dir = os.path.dirname(__file__) + auxiliary_dir = os.path.join(current_dir, '..', 'src') + template_dir = os.path.join(current_dir, '..', 'templates') - print 'Generated dart2js in %s seconds' % round(time.time() - start_time, 2) + _logger.setLevel(logging_level) - emitters.Flush() + generator = dartgenerator.DartGenerator(logging_level) + generator.LoadAuxiliary(auxiliary_dir) - if update_dom_metadata: - metadata.Flush() + generator.FilterMembersWithUnidentifiedTypes(common_database) + webkit_database = common_database.Clone() + + # Generate Dart interfaces for the WebKit DOM. + generator.FilterInterfaces( + database=webkit_database, + or_annotations=['WebKit', 'Dart'], + exclude_displaced=['WebKit'], + exclude_suppressed=['WebKit', 'Dart']) + generator.FixEventTargets(webkit_database) + generator.AddMissingArguments(webkit_database) + generator.CleanupOperationArguments(webkit_database) + + emitters = multiemitter.MultiEmitter(logging_level) + metadata = DartMetadata( + os.path.join(current_dir, '..', 'dom.json'), + os.path.join(current_dir, '..', 'docs', 'docs.json'), logging_level) + renamer = HtmlRenamer(webkit_database, metadata) + type_registry = TypeRegistry(webkit_database, renamer) + + print 'GenerateFromDatabase %s seconds' % round( + (time.time() - start_time), 2) + + def RunGenerator(dart_libraries, dart_output_dir, template_loader, + backend_factory, dart_js_interop): + options = GeneratorOptions(template_loader, webkit_database, + type_registry, renamer, metadata, + dart_js_interop) + dart_library_emitter = DartLibraryEmitter(emitters, dart_output_dir, + dart_libraries) + event_generator = HtmlEventGenerator(webkit_database, renamer, metadata, + template_loader) + + def generate_interface(interface, gl_constants=None): + backend = backend_factory(interface) + interface_generator = HtmlDartInterfaceGenerator( + options, dart_library_emitter, event_generator, interface, + backend) + interface_generator.Generate() + if len(backend._gl_constants) > 0 and not (gl_constants is None): + gl_constants.extend(backend._gl_constants) + + generator.Generate(webkit_database, common_database, generate_interface) + + dart_library_emitter.EmitLibraries(auxiliary_dir, dart_js_interop) + + if dart2js_output_dir: + template_paths = ['html/dart2js', 'html/impl', 'html/interface', ''] + template_loader = TemplateLoader(template_dir, template_paths, { + 'DARTIUM': False, + 'DART2JS': True, + 'JSINTEROP': False + }) + backend_options = GeneratorOptions(template_loader, webkit_database, + type_registry, renamer, metadata, + dart_js_interop) + backend_factory = lambda interface:\ + Dart2JSBackend(interface, backend_options, logging_level) + + dart_output_dir = os.path.join(dart2js_output_dir, 'dart') + dart_libraries = DartLibraries(HTML_LIBRARY_NAMES, template_loader, + 'dart2js', dart2js_output_dir, + dart_js_interop) + + print '\nGenerating dart2js:\n' + start_time = time.time() + + RunGenerator(dart_libraries, dart_output_dir, template_loader, + backend_factory, dart_js_interop) + + print 'Generated dart2js in %s seconds' % round( + time.time() - start_time, 2) + + emitters.Flush() + + if update_dom_metadata: + metadata.Flush() + + monitored.FinishMonitoring(dart2js_output_dir, _logger) - monitored.FinishMonitoring(dart2js_output_dir, _logger) def GenerateSingleFile(library_path, output_dir, generated_output_dir=None): - library_dir = os.path.dirname(library_path) - library_filename = os.path.basename(library_path) - copy_dart_script = os.path.relpath('../../copy_dart.py', - library_dir) - output_dir = os.path.relpath(output_dir, library_dir) - if not os.path.exists(library_dir): - os.makedirs(library_dir) - command = ' '.join(['cd', library_dir, ';', - copy_dart_script, output_dir, library_filename]) - subprocess.call([command], shell=True) - prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt') - sdk_file = os.path.join(library_dir, output_dir, library_filename) - formatCommand = ' '.join([prebuilt_dartfmt, '-w', sdk_file]) - subprocess.call([formatCommand], shell=True) + library_dir = os.path.dirname(library_path) + library_filename = os.path.basename(library_path) + copy_dart_script = os.path.relpath('../../copy_dart.py', library_dir) + output_dir = os.path.relpath(output_dir, library_dir) + if not os.path.exists(library_dir): + os.makedirs(library_dir) + command = ' '.join([ + 'cd', library_dir, ';', copy_dart_script, output_dir, library_filename + ]) + subprocess.call([command], shell=True) + prebuilt_dartfmt = os.path.join(utils.CheckedInSdkPath(), 'bin', 'dartfmt') + sdk_file = os.path.join(library_dir, output_dir, library_filename) + formatCommand = ' '.join([prebuilt_dartfmt, '-w', sdk_file]) + subprocess.call([formatCommand], shell=True) + def UpdateCssProperties(): - """Regenerate the CssStyleDeclaration template file with the current CSS + """Regenerate the CssStyleDeclaration template file with the current CSS properties.""" - _logger.info('Updating Css Properties.') - css_code_generator.GenerateCssTemplateFile() + _logger.info('Updating Css Properties.') + css_code_generator.GenerateCssTemplateFile() + CACHED_PATCHES = """ // START_OF_CACHED_PATCHES @@ -221,90 +234,126 @@ var cached_patches = { }; """ + def main(): - parser = optparse.OptionParser() - parser.add_option('--parallel', dest='parallel', - action='store_true', default=False, - help='Use fremontcut in parallel mode.') - parser.add_option('--systems', dest='systems', - action='store', type='string', - default='htmldart2js,htmldartium,_blink', - help='Systems to generate (htmldart2js, htmldartium, _blink)') - parser.add_option('--output-dir', dest='output_dir', - action='store', type='string', - default=None, - help='Directory to put the generated files') - parser.add_option('--use-database-cache', dest='use_database_cache', - action='store_true', - default=False, - help='''Use the cached database from the previous run to + parser = optparse.OptionParser() + parser.add_option( + '--parallel', + dest='parallel', + action='store_true', + default=False, + help='Use fremontcut in parallel mode.') + parser.add_option( + '--systems', + dest='systems', + action='store', + type='string', + default='htmldart2js,htmldartium,_blink', + help='Systems to generate (htmldart2js, htmldartium, _blink)') + parser.add_option( + '--output-dir', + dest='output_dir', + action='store', + type='string', + default=None, + help='Directory to put the generated files') + parser.add_option( + '--use-database-cache', + dest='use_database_cache', + action='store_true', + default=False, + help='''Use the cached database from the previous run to improve startup performance''') - parser.add_option('--update-dom-metadata', dest='update_dom_metadata', - action='store_true', - default=False, - help='''Update the metadata list of DOM APIs''') - parser.add_option('--verbose', dest='logging_level', - action='store_false', default=logging.WARNING, - help='Output all informational messages') - parser.add_option('--examine', dest='examine_idls', - action='store_true', default=None, - help='Analyze IDL files') - parser.add_option('--logging', dest='logging', type='int', - action='store', default=logging.NOTSET, - help='Level of logging 20 is Info, 30 is Warnings, 40 is Errors') - parser.add_option('--gen-interop', dest='dart_js_interop', - action='store_true', default=False, - help='Use Javascript objects (dart:js) accessing the DOM in _blink') - parser.add_option('--no-cached-patches', dest='no_cached_patches', - action='store_true', default=False, - help='Do not generate the sdk/lib/js/cached_patches.dart file') + parser.add_option( + '--update-dom-metadata', + dest='update_dom_metadata', + action='store_true', + default=False, + help='''Update the metadata list of DOM APIs''') + parser.add_option( + '--verbose', + dest='logging_level', + action='store_false', + default=logging.WARNING, + help='Output all informational messages') + parser.add_option( + '--examine', + dest='examine_idls', + action='store_true', + default=None, + help='Analyze IDL files') + parser.add_option( + '--logging', + dest='logging', + type='int', + action='store', + default=logging.NOTSET, + help='Level of logging 20 is Info, 30 is Warnings, 40 is Errors') + parser.add_option( + '--gen-interop', + dest='dart_js_interop', + action='store_true', + default=False, + help='Use Javascript objects (dart:js) accessing the DOM in _blink') + parser.add_option( + '--no-cached-patches', + dest='no_cached_patches', + action='store_true', + default=False, + help='Do not generate the sdk/lib/js/cached_patches.dart file') - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() - current_dir = os.path.dirname(__file__) - database_dir = os.path.join(current_dir, '..', 'database') - logging.config.fileConfig(os.path.join(current_dir, 'logging.conf')) - systems = options.systems.split(',') + current_dir = os.path.dirname(__file__) + database_dir = os.path.join(current_dir, '..', 'database') + logging.config.fileConfig(os.path.join(current_dir, 'logging.conf')) + systems = options.systems.split(',') - output_dir = options.output_dir or os.path.join( - current_dir, '..', '..', '..', utils.GetBuildDir(utils.GuessOS()), - 'generated') + output_dir = options.output_dir or os.path.join( + current_dir, '..', '..', '..', utils.GetBuildDir(utils.GuessOS()), + 'generated') - dart2js_output_dir = None - if 'htmldart2js' in systems: - dart2js_output_dir = os.path.join(output_dir, 'dart2js') + dart2js_output_dir = None + if 'htmldart2js' in systems: + dart2js_output_dir = os.path.join(output_dir, 'dart2js') - logging_level = options.logging_level \ - if options.logging == logging.NOTSET else options.logging + logging_level = options.logging_level \ + if options.logging == logging.NOTSET else options.logging - start_time = time.time() + start_time = time.time() - UpdateCssProperties() + UpdateCssProperties() - # Parse the IDL and create the database. - database = fremontcutbuilder.main(options.parallel, logging_level=logging_level, examine_idls=options.examine_idls) + # Parse the IDL and create the database. + database = fremontcutbuilder.main( + options.parallel, + logging_level=logging_level, + examine_idls=options.examine_idls) - GenerateFromDatabase(database, - dart2js_output_dir, - options.update_dom_metadata, - logging_level, - options.dart_js_interop) + GenerateFromDatabase(database, dart2js_output_dir, + options.update_dom_metadata, logging_level, + options.dart_js_interop) - file_generation_start_time = time.time() + file_generation_start_time = time.time() - if 'htmldart2js' in systems: - _logger.info('Generating dart2js single files.') + if 'htmldart2js' in systems: + _logger.info('Generating dart2js single files.') - for library_name in HTML_LIBRARY_NAMES: - GenerateSingleFile( - os.path.join(dart2js_output_dir, '%s_dart2js.dart' % library_name), - os.path.join('..', '..', '..', 'sdk', 'lib', library_name, 'dart2js')) + for library_name in HTML_LIBRARY_NAMES: + GenerateSingleFile( + os.path.join(dart2js_output_dir, + '%s_dart2js.dart' % library_name), + os.path.join('..', '..', '..', 'sdk', 'lib', library_name, + 'dart2js')) - print '\nGenerating single file %s seconds' % round(time.time() - file_generation_start_time, 2) + print '\nGenerating single file %s seconds' % round( + time.time() - file_generation_start_time, 2) - end_time = time.time() + end_time = time.time() + + print '\nDone (dartdomgenerator) %s seconds' % round( + end_time - start_time, 2) - print '\nDone (dartdomgenerator) %s seconds' % round(end_time - start_time, 2) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/tools/dom/scripts/dartgenerator.py b/tools/dom/scripts/dartgenerator.py index 8faa94565ff..eb5674e7ec7 100755 --- a/tools/dom/scripts/dartgenerator.py +++ b/tools/dom/scripts/dartgenerator.py @@ -2,7 +2,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """This module generates Dart APIs from the IDL database.""" import emitter @@ -16,102 +15,109 @@ from idlnode import IDLType, IDLInterface, resolveTypedef _logger = logging.getLogger('dartgenerator') -def MergeNodes(node, other): - node.operations.extend(other.operations) - for attribute in other.attributes: - if not node.has_attribute(attribute): - node.attributes.append(attribute) - node.constants.extend(other.constants) +def MergeNodes(node, other): + node.operations.extend(other.operations) + for attribute in other.attributes: + if not node.has_attribute(attribute): + node.attributes.append(attribute) + + node.constants.extend(other.constants) + class DartGenerator(object): - """Utilities to generate Dart APIs and corresponding JavaScript.""" + """Utilities to generate Dart APIs and corresponding JavaScript.""" - def __init__(self, logging_level=logging.WARNING): - self._auxiliary_files = {} - self._dart_templates_re = re.compile(r'[\w.:]+<([\w \.<>:]+)>') - _logger.setLevel(logging_level) + def __init__(self, logging_level=logging.WARNING): + self._auxiliary_files = {} + self._dart_templates_re = re.compile(r'[\w.:]+<([\w \.<>:]+)>') + _logger.setLevel(logging_level) - def _StripModules(self, type_name): - return type_name.split('::')[-1] + def _StripModules(self, type_name): + return type_name.split('::')[-1] - def _IsCompoundType(self, database, type_name): - if IsRegisteredType(type_name): - return True + def _IsCompoundType(self, database, type_name): + if IsRegisteredType(type_name): + return True - # References a typedef - normally a union type. - if database.HasTypeDef(type_name): - return True + # References a typedef - normally a union type. + if database.HasTypeDef(type_name): + return True - if type_name.endswith('?'): - return self._IsCompoundType(database, type_name[:-len('?')]) + if type_name.endswith('?'): + return self._IsCompoundType(database, type_name[:-len('?')]) - if type_name.endswith('[]'): - return self._IsCompoundType(database, type_name[:-len('[]')]) + if type_name.endswith('[]'): + return self._IsCompoundType(database, type_name[:-len('[]')]) - stripped_type_name = self._StripModules(type_name) - if (database.HasInterface(stripped_type_name) or - database.HasDictionary(stripped_type_name)): - return True + stripped_type_name = self._StripModules(type_name) + if (database.HasInterface(stripped_type_name) or + database.HasDictionary(stripped_type_name)): + return True - if database.HasEnum(stripped_type_name): - return True + if database.HasEnum(stripped_type_name): + return True - dart_template_match = self._dart_templates_re.match(type_name) - if dart_template_match: - # Dart templates - parent_type_name = type_name[0 : dart_template_match.start(1) - 1] - sub_type_name = dart_template_match.group(1) - return (self._IsCompoundType(database, parent_type_name) and - self._IsCompoundType(database, sub_type_name)) - return False + dart_template_match = self._dart_templates_re.match(type_name) + if dart_template_match: + # Dart templates + parent_type_name = type_name[0:dart_template_match.start(1) - 1] + sub_type_name = dart_template_match.group(1) + return (self._IsCompoundType(database, parent_type_name) and + self._IsCompoundType(database, sub_type_name)) + return False - def _IsDartType(self, type_name): - return '.' in type_name + def _IsDartType(self, type_name): + return '.' in type_name - def LoadAuxiliary(self, auxiliary_dir): - def Visitor(_, dirname, names): - for name in names: - if name.endswith('.dart'): - name = name[0:-5] # strip off ".dart" - self._auxiliary_files[name] = os.path.join(dirname, name) - os.path.walk(auxiliary_dir, Visitor, None) + def LoadAuxiliary(self, auxiliary_dir): - def FilterMembersWithUnidentifiedTypes(self, database): - """Removes unidentified types. + def Visitor(_, dirname, names): + for name in names: + if name.endswith('.dart'): + name = name[0:-5] # strip off ".dart" + self._auxiliary_files[name] = os.path.join(dirname, name) + + os.path.walk(auxiliary_dir, Visitor, None) + + def FilterMembersWithUnidentifiedTypes(self, database): + """Removes unidentified types. Removes constants, attributes, operations and parents with unidentified types. """ - for interface in database.GetInterfaces(): - def IsIdentified(idl_node): - node_name = idl_node.id if idl_node.id else 'parent' - for idl_type in idl_node.all(idlnode.IDLType): - type_name = idl_type.id - if (type_name is not None and - self._IsCompoundType(database, type_name)): - continue - # Ignore constructor warnings. - if not (interface.id in ['Window', 'WorkerContext', - 'WorkerGlobalScope'] and - type_name.endswith('Constructor')): - _logger.warn('removing %s in %s which has unidentified type %s' % - (node_name, interface.id, type_name)) - return False - return True + for interface in database.GetInterfaces(): - interface.constants = filter(IsIdentified, interface.constants) - interface.attributes = filter(IsIdentified, interface.attributes) - interface.operations = filter(IsIdentified, interface.operations) - interface.parents = filter(IsIdentified, interface.parents) + def IsIdentified(idl_node): + node_name = idl_node.id if idl_node.id else 'parent' + for idl_type in idl_node.all(idlnode.IDLType): + type_name = idl_type.id + if (type_name is not None and + self._IsCompoundType(database, type_name)): + continue + # Ignore constructor warnings. + if not (interface.id in [ + 'Window', 'WorkerContext', 'WorkerGlobalScope' + ] and type_name.endswith('Constructor')): + _logger.warn( + 'removing %s in %s which has unidentified type %s' % + (node_name, interface.id, type_name)) + return False + return True - def FilterInterfaces(self, database, - and_annotations=[], - or_annotations=[], - exclude_displaced=[], - exclude_suppressed=[]): - """Filters a database to remove interfaces and members that are missing + interface.constants = filter(IsIdentified, interface.constants) + interface.attributes = filter(IsIdentified, interface.attributes) + interface.operations = filter(IsIdentified, interface.operations) + interface.parents = filter(IsIdentified, interface.parents) + + def FilterInterfaces(self, + database, + and_annotations=[], + or_annotations=[], + exclude_displaced=[], + exclude_suppressed=[]): + """Filters a database to remove interfaces and members that are missing annotations. The FremontCut IDLs use annotations to specify implementation @@ -131,142 +137,149 @@ class DartGenerator(object): is marked as suppressed it will always be filtered. """ - # Filter interfaces and members whose annotations don't match. - for interface in database.GetInterfaces(): - def HasAnnotations(idl_node): - """Utility for determining if an IDLNode has all + # Filter interfaces and members whose annotations don't match. + for interface in database.GetInterfaces(): + + def HasAnnotations(idl_node): + """Utility for determining if an IDLNode has all the required annotations""" - for a in exclude_displaced: - if (a in idl_node.annotations - and 'via' in idl_node.annotations[a]): - return False - for a in exclude_suppressed: - if (a in idl_node.annotations - and 'suppressed' in idl_node.annotations[a]): - return False - for a in or_annotations: - if a in idl_node.annotations: + for a in exclude_displaced: + if (a in idl_node.annotations and + 'via' in idl_node.annotations[a]): + return False + for a in exclude_suppressed: + if (a in idl_node.annotations and + 'suppressed' in idl_node.annotations[a]): + return False + for a in or_annotations: + if a in idl_node.annotations: + return True + if and_annotations == []: + return False + for a in and_annotations: + if a not in idl_node.annotations: + return False + return True + + if HasAnnotations(interface): + interface.constants = filter(HasAnnotations, + interface.constants) + interface.attributes = filter(HasAnnotations, + interface.attributes) + interface.operations = filter(HasAnnotations, + interface.operations) + interface.parents = filter(HasAnnotations, interface.parents) + else: + database.DeleteInterface(interface.id) + + self.FilterMembersWithUnidentifiedTypes(database) + + def Generate(self, database, super_database, generate_interface): + self._database = database + + # Collect interfaces + interfaces = [] + for interface in database.GetInterfaces(): + if not MatchSourceFilter(interface): + # Skip this interface since it's not present in the required source + _logger.info('Omitting interface - %s' % interface.id) + continue + interfaces.append(interface) + + # All web_gl constants from WebGLRenderingContextBase, WebGL2RenderingContextBase, WebGLDrawBuffers are generated + # in a synthesized class WebGL. Those IDLConstants are in web_gl_constants. + web_gl_constants = [] + + # Render all interfaces into Dart and save them in files. + for interface in self._PreOrderInterfaces(interfaces): + interface_name = interface.id + auxiliary_file = self._auxiliary_files.get(interface_name) + if auxiliary_file is not None: + _logger.info('Skipping %s because %s exists' % (interface_name, + auxiliary_file)) + continue + + _logger.info('Generating %s' % interface.id) + generate_interface(interface, gl_constants=web_gl_constants) + + # Generate the WEB_GL constants + web_gl_constants_interface = IDLInterface(None, "WebGL") + web_gl_constants_interface.constants = web_gl_constants + self._database._all_interfaces['WebGL'] = web_gl_constants_interface + generate_interface(web_gl_constants_interface) + + def _PreOrderInterfaces(self, interfaces): + """Returns the interfaces in pre-order, i.e. parents first.""" + seen = set() + ordered = [] + + def visit(interface): + if interface.id in seen: + return + seen.add(interface.id) + for parent in interface.parents: + if IsDartCollectionType(parent.type.id): + continue + if self._database.HasInterface(parent.type.id): + parent_interface = self._database.GetInterface( + parent.type.id) + visit(parent_interface) + ordered.append(interface) + + for interface in interfaces: + visit(interface) + return ordered + + def IsEventTarget(self, database, interface): + if interface.id == 'EventTarget': return True - if and_annotations == []: - return False - for a in and_annotations: - if a not in idl_node.annotations: - return False - return True + for parent in interface.parents: + parent_name = parent.type.id + if database.HasInterface(parent_name): + parent_interface = database.GetInterface(parent.type.id) + if self.IsEventTarget(database, parent_interface): + return True + return False - if HasAnnotations(interface): - interface.constants = filter(HasAnnotations, interface.constants) - interface.attributes = filter(HasAnnotations, interface.attributes) - interface.operations = filter(HasAnnotations, interface.operations) - interface.parents = filter(HasAnnotations, interface.parents) - else: - database.DeleteInterface(interface.id) + def FixEventTargets(self, database): + for interface in database.GetInterfaces(): + if self.IsEventTarget(database, interface): + # Add as an attribute for easy querying in generation code. + interface.ext_attrs['EventTarget'] = None + elif 'EventTarget' in interface.ext_attrs: + # Create fake EventTarget parent interface for interfaces that have + # 'EventTarget' extended attribute. + ast = [('Annotation', [('Id', 'WebKit')]), + ('InterfaceType', ('ScopedName', 'EventTarget'))] + interface.parents.append(idlnode.IDLParentInterface(ast)) - self.FilterMembersWithUnidentifiedTypes(database) + def AddMissingArguments(self, database): + ARG = idlnode.IDLArgument([('Type', ('ScopedName', 'object')), + ('Id', 'arg')]) + for interface in database.GetInterfaces(): + for operation in interface.operations: + call_with = operation.ext_attrs.get('CallWith', []) + if not (isinstance(call_with, list)): + call_with = [call_with] + constructor_with = operation.ext_attrs.get( + 'ConstructorCallWith', []) + if not (isinstance(constructor_with, list)): + constructor_with = [constructor_with] + call_with = call_with + constructor_with - def Generate(self, database, super_database, generate_interface): - self._database = database + if 'ScriptArguments' in call_with: + operation.arguments.append(ARG) - # Collect interfaces - interfaces = [] - for interface in database.GetInterfaces(): - if not MatchSourceFilter(interface): - # Skip this interface since it's not present in the required source - _logger.info('Omitting interface - %s' % interface.id) - continue - interfaces.append(interface) - - # All web_gl constants from WebGLRenderingContextBase, WebGL2RenderingContextBase, WebGLDrawBuffers are generated - # in a synthesized class WebGL. Those IDLConstants are in web_gl_constants. - web_gl_constants = [] - - # Render all interfaces into Dart and save them in files. - for interface in self._PreOrderInterfaces(interfaces): - interface_name = interface.id - auxiliary_file = self._auxiliary_files.get(interface_name) - if auxiliary_file is not None: - _logger.info('Skipping %s because %s exists' % ( - interface_name, auxiliary_file)) - continue - - _logger.info('Generating %s' % interface.id) - generate_interface(interface, gl_constants = web_gl_constants) - - # Generate the WEB_GL constants - web_gl_constants_interface = IDLInterface(None, "WebGL") - web_gl_constants_interface.constants = web_gl_constants - self._database._all_interfaces['WebGL'] = web_gl_constants_interface - generate_interface(web_gl_constants_interface) - - def _PreOrderInterfaces(self, interfaces): - """Returns the interfaces in pre-order, i.e. parents first.""" - seen = set() - ordered = [] - def visit(interface): - if interface.id in seen: - return - seen.add(interface.id) - for parent in interface.parents: - if IsDartCollectionType(parent.type.id): - continue - if self._database.HasInterface(parent.type.id): - parent_interface = self._database.GetInterface(parent.type.id) - visit(parent_interface) - ordered.append(interface) - - for interface in interfaces: - visit(interface) - return ordered - - def IsEventTarget(self, database, interface): - if interface.id == 'EventTarget': - return True - for parent in interface.parents: - parent_name = parent.type.id - if database.HasInterface(parent_name): - parent_interface = database.GetInterface(parent.type.id) - if self.IsEventTarget(database, parent_interface): - return True - return False - - def FixEventTargets(self, database): - for interface in database.GetInterfaces(): - if self.IsEventTarget(database, interface): - # Add as an attribute for easy querying in generation code. - interface.ext_attrs['EventTarget'] = None - elif 'EventTarget' in interface.ext_attrs: - # Create fake EventTarget parent interface for interfaces that have - # 'EventTarget' extended attribute. - ast = [('Annotation', [('Id', 'WebKit')]), - ('InterfaceType', ('ScopedName', 'EventTarget'))] - interface.parents.append(idlnode.IDLParentInterface(ast)) - - def AddMissingArguments(self, database): - ARG = idlnode.IDLArgument([('Type', ('ScopedName', 'object')), ('Id', 'arg')]) - for interface in database.GetInterfaces(): - for operation in interface.operations: - call_with = operation.ext_attrs.get('CallWith', []) - if not(isinstance(call_with, list)): - call_with = [call_with] - constructor_with = operation.ext_attrs.get('ConstructorCallWith', []) - if not(isinstance(constructor_with, list)): - constructor_with = [constructor_with] - call_with = call_with + constructor_with - - if 'ScriptArguments' in call_with: - operation.arguments.append(ARG) - - def CleanupOperationArguments(self, database): - for interface in database.GetInterfaces(): - for operation in interface.operations: - # TODO(terry): Hack to remove 3rd arguments in setInterval/setTimeout. - if ((operation.id == 'setInterval' or operation.id == 'setTimeout') and \ - operation.arguments[0].type.id == 'any'): - operation.arguments.pop(2) - - # Massage any operation argument type that is IDLEnum to String. - for index, argument in enumerate(operation.arguments): - type_name = argument.type.id - if database.HasEnum(type_name): - operation.arguments[index].type = IDLType('DOMString') + def CleanupOperationArguments(self, database): + for interface in database.GetInterfaces(): + for operation in interface.operations: + # TODO(terry): Hack to remove 3rd arguments in setInterval/setTimeout. + if ((operation.id == 'setInterval' or operation.id == 'setTimeout') and \ + operation.arguments[0].type.id == 'any'): + operation.arguments.pop(2) + # Massage any operation argument type that is IDLEnum to String. + for index, argument in enumerate(operation.arguments): + type_name = argument.type.id + if database.HasEnum(type_name): + operation.arguments[index].type = IDLType('DOMString') diff --git a/tools/dom/scripts/dartgenerator_test.py b/tools/dom/scripts/dartgenerator_test.py index f065eb20aca..cfd7b2e1781 100755 --- a/tools/dom/scripts/dartgenerator_test.py +++ b/tools/dom/scripts/dartgenerator_test.py @@ -2,7 +2,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """Tests for dartgenerator.""" import logging.config @@ -19,76 +18,75 @@ import idlparser class DartGeneratorTestCase(unittest.TestCase): - def _InDatabase(self, interface_name): - return os.path.exists(os.path.join(self._database_dir, - '%s.idl' % interface_name)) + def _InDatabase(self, interface_name): + return os.path.exists( + os.path.join(self._database_dir, '%s.idl' % interface_name)) - def _FilePathForDartInterface(self, interface_name): - return os.path.join(self._generator._output_dir, 'src', 'interface', - '%s.dart' % interface_name) + def _FilePathForDartInterface(self, interface_name): + return os.path.join(self._generator._output_dir, 'src', 'interface', + '%s.dart' % interface_name) - def _InOutput(self, interface_name): - return os.path.exists( - self._FilePathForDartInterface(interface_name)) + def _InOutput(self, interface_name): + return os.path.exists(self._FilePathForDartInterface(interface_name)) - def _ReadOutputFile(self, interface_name): - self.assertTrue(self._InOutput(interface_name)) - file_path = self._FilePathForDartInterface(interface_name) - f = open(file_path, 'r') - content = f.read() - f.close() - return content, file_path + def _ReadOutputFile(self, interface_name): + self.assertTrue(self._InOutput(interface_name)) + file_path = self._FilePathForDartInterface(interface_name) + f = open(file_path, 'r') + content = f.read() + f.close() + return content, file_path - def _AssertOutputSansHeaderEquals(self, interface_name, expected_content): - full_actual_content, file_path = self._ReadOutputFile(interface_name) - # Remove file header comments in // or multiline /* ... */ syntax. - header_re = re.compile(r'^(\s*(//.*|/\*([^*]|\*[^/])*\*/)\s*)*') - actual_content = header_re.sub('', full_actual_content) - if expected_content != actual_content: - msg = """ + def _AssertOutputSansHeaderEquals(self, interface_name, expected_content): + full_actual_content, file_path = self._ReadOutputFile(interface_name) + # Remove file header comments in // or multiline /* ... */ syntax. + header_re = re.compile(r'^(\s*(//.*|/\*([^*]|\*[^/])*\*/)\s*)*') + actual_content = header_re.sub('', full_actual_content) + if expected_content != actual_content: + msg = """ FILE: %s EXPECTED: %s ACTUAL: %s """ % (file_path, expected_content, actual_content) - self.fail(msg) + self.fail(msg) - def _AssertOutputContains(self, interface_name, expected_content): - actual_content, file_path = self._ReadOutputFile(interface_name) - if expected_content not in actual_content: - msg = """ + def _AssertOutputContains(self, interface_name, expected_content): + actual_content, file_path = self._ReadOutputFile(interface_name) + if expected_content not in actual_content: + msg = """ STRING: %s Was found not in output file: %s FILE CONTENT: %s """ % (expected_content, file_path, actual_content) - self.fail(msg) + self.fail(msg) - def _AssertOutputDoesNotContain(self, interface_name, expected_content): - actual_content, file_path = self._ReadOutputFile(interface_name) - if expected_content in actual_content: - msg = """ + def _AssertOutputDoesNotContain(self, interface_name, expected_content): + actual_content, file_path = self._ReadOutputFile(interface_name) + if expected_content in actual_content: + msg = """ STRING: %s Was found in output file: %s FILE CONTENT: %s """ % (expected_content, file_path, actual_content) - self.fail(msg) + self.fail(msg) - def setUp(self): - self._working_dir = tempfile.mkdtemp() - self._output_dir = os.path.join(self._working_dir, 'output') - self._database_dir = os.path.join(self._working_dir, 'database') - self._auxiliary_dir = os.path.join(self._working_dir, 'auxiliary') - self.assertFalse(os.path.exists(self._database_dir)) + def setUp(self): + self._working_dir = tempfile.mkdtemp() + self._output_dir = os.path.join(self._working_dir, 'output') + self._database_dir = os.path.join(self._working_dir, 'database') + self._auxiliary_dir = os.path.join(self._working_dir, 'auxiliary') + self.assertFalse(os.path.exists(self._database_dir)) - # Create database and add one interface. - db = database.Database(self._database_dir) - os.mkdir(self._auxiliary_dir) - self.assertTrue(os.path.exists(self._database_dir)) + # Create database and add one interface. + db = database.Database(self._database_dir) + os.mkdir(self._auxiliary_dir) + self.assertTrue(os.path.exists(self._database_dir)) - content = """ + content = """ module shapes { @A1 @A2 interface Shape { @@ -120,50 +118,50 @@ FILE CONTENT: }; """ - parser = idlparser.IDLParser(idlparser.FREMONTCUT_SYNTAX) - ast = parser.parse(content) - idl_file = idlnode.IDLFile(ast) - for interface in idl_file.interfaces: - db.AddInterface(interface) - db.Save() + parser = idlparser.IDLParser(idlparser.FREMONTCUT_SYNTAX) + ast = parser.parse(content) + idl_file = idlnode.IDLFile(ast) + for interface in idl_file.interfaces: + db.AddInterface(interface) + db.Save() - self.assertTrue(self._InDatabase('Shape')) - self.assertTrue(self._InDatabase('Rectangle')) - self.assertTrue(self._InDatabase('Line')) + self.assertTrue(self._InDatabase('Shape')) + self.assertTrue(self._InDatabase('Rectangle')) + self.assertTrue(self._InDatabase('Line')) - self._database = database.Database(self._database_dir) - self._generator = dartgenerator.DartGenerator(self._auxiliary_dir, - '../templates', - 'test') + self._database = database.Database(self._database_dir) + self._generator = dartgenerator.DartGenerator(self._auxiliary_dir, + '../templates', 'test') - def tearDown(self): - shutil.rmtree(self._database_dir) - shutil.rmtree(self._auxiliary_dir) + def tearDown(self): + shutil.rmtree(self._database_dir) + shutil.rmtree(self._auxiliary_dir) - def testBasicGeneration(self): - # Generate all interfaces: - self._database.Load() - self._generator.Generate(self._database, self._output_dir) - self._generator.Flush() + def testBasicGeneration(self): + # Generate all interfaces: + self._database.Load() + self._generator.Generate(self._database, self._output_dir) + self._generator.Flush() - self.assertTrue(self._InOutput('Shape')) - self.assertTrue(self._InOutput('Rectangle')) - self.assertTrue(self._InOutput('Line')) + self.assertTrue(self._InOutput('Shape')) + self.assertTrue(self._InOutput('Rectangle')) + self.assertTrue(self._InOutput('Line')) - def testFilterByAnnotations(self): - self._database.Load() - self._generator.FilterInterfaces(self._database, ['A1', 'A2'], ['A3']) - self._generator.Generate(self._database, self._output_dir) - self._generator.Flush() + def testFilterByAnnotations(self): + self._database.Load() + self._generator.FilterInterfaces(self._database, ['A1', 'A2'], ['A3']) + self._generator.Generate(self._database, self._output_dir) + self._generator.Flush() - # Only interfaces with (@A1 and @A2) or @A3 should be generated: - self.assertTrue(self._InOutput('Shape')) - self.assertTrue(self._InOutput('Rectangle')) - self.assertFalse(self._InOutput('Line')) + # Only interfaces with (@A1 and @A2) or @A3 should be generated: + self.assertTrue(self._InOutput('Shape')) + self.assertTrue(self._InOutput('Rectangle')) + self.assertFalse(self._InOutput('Line')) - # Only members with (@A1 and @A2) or @A3 should be generated: - # TODO(sra): make th - self._AssertOutputSansHeaderEquals('Shape', """interface Shape { + # Only members with (@A1 and @A2) or @A3 should be generated: + # TODO(sra): make th + self._AssertOutputSansHeaderEquals( + 'Shape', """interface Shape { final int attr; @@ -171,66 +169,58 @@ FILE CONTENT: } """) - self._AssertOutputContains('Rectangle', - 'interface Rectangle extends shapes::Shape') + self._AssertOutputContains('Rectangle', + 'interface Rectangle extends shapes::Shape') - def testTypeRenames(self): - self._database.Load() - # Translate 'Shape' to spanish: - self._generator.RenameTypes(self._database, {'Shape': 'Forma'}, False) - self._generator.Generate(self._database, self._output_dir) - self._generator.Flush() + def testTypeRenames(self): + self._database.Load() + # Translate 'Shape' to spanish: + self._generator.RenameTypes(self._database, {'Shape': 'Forma'}, False) + self._generator.Generate(self._database, self._output_dir) + self._generator.Flush() - # Validate that all references to Shape have been converted: - self._AssertOutputContains('Forma', - 'interface Forma') - self._AssertOutputContains('Forma', 'Forma create();') - self._AssertOutputContains('Forma', - 'bool compare(Forma s);') - self._AssertOutputContains('Rectangle', - 'interface Rectangle extends Forma') + # Validate that all references to Shape have been converted: + self._AssertOutputContains('Forma', 'interface Forma') + self._AssertOutputContains('Forma', 'Forma create();') + self._AssertOutputContains('Forma', 'bool compare(Forma s);') + self._AssertOutputContains('Rectangle', + 'interface Rectangle extends Forma') - def testQualifiedDartTypes(self): - self._database.Load() - self._generator.FilterMembersWithUnidentifiedTypes(self._database) - self._generator.Generate(self._database, self._output_dir) - self._generator.Flush() + def testQualifiedDartTypes(self): + self._database.Load() + self._generator.FilterMembersWithUnidentifiedTypes(self._database) + self._generator.Generate(self._database, self._output_dir) + self._generator.Flush() - # Verify primitive conversions are working: - self._AssertOutputContains('Shape', - 'static const int CONSTANT = 1') - self._AssertOutputContains('Shape', - 'final String strAttr;') + # Verify primitive conversions are working: + self._AssertOutputContains('Shape', 'static const int CONSTANT = 1') + self._AssertOutputContains('Shape', 'final String strAttr;') - # Verify interface names are converted: - self._AssertOutputContains('Shape', - 'interface Shape {') - self._AssertOutputContains('Shape', - ' Shape create();') - # TODO(sra): Why is this broken? Output contains qualified type. - #self._AssertOutputContains('Shape', - # 'void addLine(Line line);') - self._AssertOutputContains('Shape', - 'Rectangle createRectangle();') - # TODO(sra): Why is this broken? Output contains qualified type. - #self._AssertOutputContains('Rectangle', - # 'interface Rectangle extends Shape') - # Verify dart names are preserved: - # TODO(vsm): Re-enable when package / namespaces are enabled. - # self._AssertOutputContains('shapes', 'Shape', - # 'void someDartType(File file);') + # Verify interface names are converted: + self._AssertOutputContains('Shape', 'interface Shape {') + self._AssertOutputContains('Shape', ' Shape create();') + # TODO(sra): Why is this broken? Output contains qualified type. + #self._AssertOutputContains('Shape', + # 'void addLine(Line line);') + self._AssertOutputContains('Shape', 'Rectangle createRectangle();') + # TODO(sra): Why is this broken? Output contains qualified type. + #self._AssertOutputContains('Rectangle', + # 'interface Rectangle extends Shape') + # Verify dart names are preserved: + # TODO(vsm): Re-enable when package / namespaces are enabled. + # self._AssertOutputContains('shapes', 'Shape', + # 'void someDartType(File file);') - # Verify that unidentified types are not removed: - self._AssertOutputDoesNotContain('Shape', - 'someUnidentifiedType') + # Verify that unidentified types are not removed: + self._AssertOutputDoesNotContain('Shape', 'someUnidentifiedType') - # Verify template conversion: - # TODO(vsm): Re-enable when core collections are supported. - # self._AssertOutputContains('rectangles', 'Rectangle', - # 'void someTemplatedType(List list)') + # Verify template conversion: + # TODO(vsm): Re-enable when core collections are supported. + # self._AssertOutputContains('rectangles', 'Rectangle', + # 'void someTemplatedType(List list)') if __name__ == '__main__': - logging.config.fileConfig('logging.conf') - if __name__ == '__main__': - unittest.main() + logging.config.fileConfig('logging.conf') + if __name__ == '__main__': + unittest.main() diff --git a/tools/dom/scripts/dartmetadata.py b/tools/dom/scripts/dartmetadata.py index a5a92f180c0..f7a6bcb393e 100644 --- a/tools/dom/scripts/dartmetadata.py +++ b/tools/dom/scripts/dartmetadata.py @@ -2,7 +2,6 @@ # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. - """This module provides shared functionality to provide Dart metadata for DOM APIs. """ @@ -27,882 +26,928 @@ _logger = logging.getLogger('dartmetadata') # -TYPE: add annotations only if there are no member annotations. # TYPE: add regardless of member annotations. -_dart2js_annotations = monitored.Dict('dartmetadata._dart2js_annotations', { - - 'AnimationEffectTiming.duration': [ - "@Creates('Null')", - "@Returns('num|String')", - ], - - 'ArrayBufferView': [ - "@Creates('TypedData')", - "@Returns('TypedData|Null')", - ], - - 'CanvasRenderingContext2D.createImageData': [ - "@Creates('ImageData|=Object')", - ], - - 'CanvasRenderingContext2D.getImageData': [ - "@Creates('ImageData|=Object')", - ], - - 'CanvasRenderingContext2D.webkitGetImageDataHD': [ - "@Creates('ImageData|=Object')", - ], - - 'CanvasRenderingContext2D.fillStyle': [ - "@Creates('String|CanvasGradient|CanvasPattern')", - "@Returns('String|CanvasGradient|CanvasPattern')", - ], - - 'CanvasRenderingContext2D.strokeStyle': [ - "@Creates('String|CanvasGradient|CanvasPattern')", - "@Returns('String|CanvasGradient|CanvasPattern')", - ], - - 'CryptoKey.algorithm': [ - "@Creates('Null')", - ], - - 'CustomEvent._detail': [ - "@Creates('Null')", - ], - - # Normally Window is never null, but starting from a