From 86fe7ca75cb324bb4128515a191a1c576925670f Mon Sep 17 00:00:00 2001 From: Joseph Richey Date: Tue, 23 Apr 2019 07:48:15 +0000 Subject: [PATCH] Fix build when python=python3 Right now most of the dart SDK's python is compatible with python2 or python3. This change fixes a few of the build scripts to make that completely true (at least when building the standard build on Linux). There are only four types of changes: - Bare `print` statements now use the `print ()` function - `commands.getoutput` becomes `subprocess.check_output` with `shell=True` - `xrange` becomes `range` - `print >> sys.stderr` becomes `sys.stderr.write` Starts work on addressing (but does not completely fix): https://github.com/dart-lang/sdk/issues/28793 See related issue: https://fuchsia-review.googlesource.com/c/fuchsia/+/272925 This change applys to both the `dev` and `master` branches. Change-Id: Ibd3eb9b1f57520d2d745f05c2ac430b1d20943da Closes #36662 https://github.com/dart-lang/sdk/pull/36662 GitOrigin-RevId: beab165294982a7e369daf6d61aea63efcab1b9b Change-Id: I6d240749a9ba0889b5a45a08f3c4c2c20291f484 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/99707 Reviewed-by: Alexander Thomas Commit-Queue: Alexander Thomas --- build/gn_run_binary.py | 4 +- tools/build.py | 20 +++---- tools/copy_tree.py | 14 ++--- tools/download_abi_dills.py | 2 +- tools/generate_buildfiles.py | 6 +- tools/gn.py | 16 +++--- tools/make_version.py | 4 +- tools/minidump.py | 2 +- tools/utils.py | 105 +++++++++++++++++------------------ 9 files changed, 86 insertions(+), 87 deletions(-) diff --git a/build/gn_run_binary.py b/build/gn_run_binary.py index f6f9bac0a69..718bf240974 100755 --- a/build/gn_run_binary.py +++ b/build/gn_run_binary.py @@ -48,7 +48,7 @@ def main(argv): path = './' + argv[2] if not os.path.isfile(path): - print "Binary not found: " + path + print ("Binary not found: " + path) return error_exit # The rest of the arguments are passed directly to the executable. @@ -56,7 +56,7 @@ def main(argv): result = run_command(args) if result != 0: - print result + print (result) return error_exit return 0 diff --git a/tools/build.py b/tools/build.py index 237c48677cd..fa05e239ef0 100755 --- a/tools/build.py +++ b/tools/build.py @@ -82,20 +82,20 @@ def ProcessOptions(options, args): options.os = options.os.split(',') for mode in options.mode: if not mode in ['debug', 'release', 'product']: - print "Unknown mode %s" % mode + print ("Unknown mode %s" % mode) return False for arch in options.arch: if not arch in AVAILABLE_ARCHS: - print "Unknown arch %s" % arch + print ("Unknown arch %s" % arch) return False options.os = [ProcessOsOption(os_name) for os_name in options.os] for os_name in options.os: if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']: - print "Unknown os %s" % os_name + print ("Unknown os %s" % os_name) return False if os_name != HOST_OS: if os_name != 'android': - print "Unsupported target os %s" % os_name + print ("Unsupported target os %s" % os_name) return False if not HOST_OS in ['linux', 'macos']: print ("Cross-compilation to %s is not supported on host os %s." @@ -109,14 +109,14 @@ def ProcessOptions(options, args): # We have not yet tweaked the v8 dart build to work with the Android # NDK/SDK, so don't try to build it. if not args: - print "For android builds you must specify a target, such as 'runtime'." + print ("For android builds you must specify a target, such as 'runtime'.") return False return True def NotifyBuildDone(build_config, success, start): if not success: - print "BUILD FAILED" + print ("BUILD FAILED") sys.stdout.flush() @@ -223,10 +223,10 @@ def EnsureGomaStarted(out_dir): words = line.split() goma_dir = words[2][1:-1] # goma_dir = "/path/to/goma" if not goma_dir: - print 'Could not find goma for ' + out_dir + print ('Could not find goma for ' + out_dir) return False if not os.path.exists(goma_dir) or not os.path.isdir(goma_dir): - print 'Could not find goma at ' + goma_dir + print ('Could not find goma at ' + goma_dir) return False goma_ctl = os.path.join(goma_dir, 'goma_ctl.py') goma_ctl_command = [ @@ -270,7 +270,7 @@ def BuildOneConfig(options, targets, target_os, mode, arch): def RunOneBuildCommand(build_config, args): start_time = time.time() - print ' '.join(args) + print (' '.join(args)) process = subprocess.Popen(args, stdin=None) process.wait() if process.returncode != 0: @@ -284,7 +284,7 @@ def RunOneBuildCommand(build_config, args): def RunOneGomaBuildCommand(args): try: - print ' '.join(args) + print (' '.join(args)) process = subprocess.Popen(args, stdin=None) process.wait() print (' '.join(args) + " done.") diff --git a/tools/copy_tree.py b/tools/copy_tree.py index 5d727504eba..d175b1491a0 100755 --- a/tools/copy_tree.py +++ b/tools/copy_tree.py @@ -43,17 +43,17 @@ def ParseArgs(args): def ValidateArgs(args): if args.gn: if args.exclude_patterns or args.copy_from or args.to: - print "--gn mode does not accept other switches" + print ("--gn mode does not accept other switches") return False if not args.gn_paths: - print "--gn mode requires a list of source specifications" + print ("--gn mode requires a list of source specifications") return False return True if not args.copy_from or not os.path.isdir(args.copy_from): - print "--from argument must refer to a directory" + print ("--from argument must refer to a directory") return False if not args.to: - print "--to is required" + print ("--to is required") return False return True @@ -118,10 +118,10 @@ def ListTree(src, ignore=None): # directory, and source_dirs[n+1] is a list of ignore patterns. def SourcesToGN(source_dirs): if len(source_dirs) % 2 != 0: - print "--gn list length should be a multiple of 2." + print ("--gn list length should be a multiple of 2.") return False data = [] - for i in xrange(0, len(source_dirs), 2): + for i in range(0, len(source_dirs), 2): path = source_dirs[i] ignores = source_dirs[i + 1] if ignores in ["{}"]: @@ -131,7 +131,7 @@ def SourcesToGN(source_dirs): sources = ListTree(path, ignore=shutil.ignore_patterns(*patterns)) data.append(sources) scope_data = {"sources": data} - print gn_helpers.ToGNString(scope_data) + print (gn_helpers.ToGNString(scope_data)) return True diff --git a/tools/download_abi_dills.py b/tools/download_abi_dills.py index 51dd8c1cbcd..5de464b06a1 100644 --- a/tools/download_abi_dills.py +++ b/tools/download_abi_dills.py @@ -28,7 +28,7 @@ def main(): oldest_abi_version = int(utils.GetOldestSupportedAbiVersion()) cmd = ['cipd', 'ensure', '-root', 'tools/abiversions', '-ensure-file', '-'] ensure_file = '' - for i in xrange(oldest_abi_version, abi_version + 1): + for i in range(oldest_abi_version, abi_version + 1): if findAbiVersion(i): ensure_file += '@Subdir %d\ndart/abiversions/%d latest\n\n' % (i, i) if not ensure_file: diff --git a/tools/generate_buildfiles.py b/tools/generate_buildfiles.py index 611895720a0..201c6669727 100755 --- a/tools/generate_buildfiles.py +++ b/tools/generate_buildfiles.py @@ -37,7 +37,7 @@ def RunAndroidGn(options): ] if options.verbose: gn_command.append('-v') - print ' '.join(gn_command) + print (' '.join(gn_command)) return Execute(gn_command) @@ -52,7 +52,7 @@ def RunCrossGn(options): ] if options.verbose: gn_command.append('-v') - print ' '.join(gn_command) + print (' '.join(gn_command)) return Execute(gn_command) @@ -65,7 +65,7 @@ def RunHostGn(options): ] if options.verbose: gn_command.append('-v') - print ' '.join(gn_command) + print (' '.join(gn_command)) return Execute(gn_command) diff --git a/tools/gn.py b/tools/gn.py index fbd1814402e..2bf358f9d6a 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -301,23 +301,23 @@ def ProcessOptions(args): args.os = args.os.split(',') for mode in args.mode: if not mode in ['debug', 'release', 'product']: - print "Unknown mode %s" % mode + print ("Unknown mode %s" % mode) return False for arch in args.arch: archs = ['ia32', 'x64', 'simarm', 'arm', 'simarmv6', 'armv6', 'simarmv5te', 'armv5te', 'simarm64', 'arm64', 'simdbc', 'simdbc64', 'armsimdbc', 'armsimdbc64'] if not arch in archs: - print "Unknown arch %s" % arch + print ("Unknown arch %s" % arch) return False oses = [ProcessOsOption(os_name) for os_name in args.os] for os_name in oses: if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']: - print "Unknown os %s" % os_name + print ("Unknown os %s" % os_name) return False if os_name != HOST_OS: if os_name != 'android': - print "Unsupported target os %s" % os_name + print ("Unsupported target os %s" % os_name) return False if not HOST_OS in ['linux', 'macos']: print ("Cross-compilation to %s is not supported on host os %s." @@ -329,7 +329,7 @@ def ProcessOptions(args): % (os_name, arch)) return False if HOST_OS != 'win' and args.use_crashpad: - print "Crashpad is only supported on Windows" + print ("Crashpad is only supported on Windows") return False return True @@ -501,7 +501,7 @@ def Main(argv): gn = os.path.join(DART_ROOT, 'buildtools', 'gn.exe' if utils.IsWindows() else 'gn') if not os.path.isfile(gn): - print "Couldn't find the gn binary at path: " + gn + print ("Couldn't find the gn binary at path: " + gn) return 1 commands = [] @@ -516,7 +516,7 @@ def Main(argv): gn_args = ToCommandLine(ToGnArgs(args, mode, arch, target_os)) gn_args += GetGNArgs(args) if args.verbose: - print "gn gen --check in %s" % out_dir + print ("gn gen --check in %s" % out_dir) if args.ide: command.append(ide_switch(HOST_OS)) command.append('--args=%s' % ' '.join(gn_args)) @@ -526,7 +526,7 @@ def Main(argv): results = pool.map(RunCommand, commands, chunksize=1) for r in results: if r != 0: - print r.strip() + print (r.strip()) return 1 endtime = time.time() diff --git a/tools/make_version.py b/tools/make_version.py index d63dfa816e0..dc763e91922 100755 --- a/tools/make_version.py +++ b/tools/make_version.py @@ -126,7 +126,7 @@ def main(args): if options.output: open(options.output, 'w').write(version_string) else: - print version_string + print (version_string) return 0 if not options.output: @@ -145,7 +145,7 @@ def main(args): return -1 return 0 - except Exception, inst: + except Exception as inst: sys.stderr.write('make_version.py exception\n') sys.stderr.write(str(inst)) sys.stderr.write('\n') diff --git a/tools/minidump.py b/tools/minidump.py index 2f382e418b5..e74be943d34 100644 --- a/tools/minidump.py +++ b/tools/minidump.py @@ -158,7 +158,7 @@ class MinidumpFile(object): raise Exception('Unsupported minidump header magic') self.directories = [] offset = self.header.stream_directories_rva - for _ in xrange(self.header.stream_count): + for _ in range(self.header.stream_count): self.directories.append(self.Read(MINIDUMP_DIRECTORY, offset)) offset += MINIDUMP_DIRECTORY.size diff --git a/tools/utils.py b/tools/utils.py index 32d843e2090..429033d17ed 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -5,7 +5,6 @@ # This file contains a set of utilities functions used by other Python-based # scripts. -import commands import contextlib import datetime import glob @@ -94,8 +93,8 @@ def GuessArchitecture(): return 'ia32' else: guess_os = GuessOS() - print "Warning: Guessing architecture %s based on os %s\n"\ - % (os_id, guess_os) + print ("Warning: Guessing architecture %s based on os %s\n"\ + % (os_id, guess_os)) if guess_os == 'win32': return 'ia32' return None @@ -106,11 +105,11 @@ def GuessCpus(): if os.getenv("DART_NUMBER_OF_CORES") is not None: return int(os.getenv("DART_NUMBER_OF_CORES")) if os.path.exists("/proc/cpuinfo"): - return int(commands.getoutput("grep -E '^processor' /proc/cpuinfo | wc -l")) + return int(subprocess.check_output("grep -E '^processor' /proc/cpuinfo | wc -l", shell=True)) if os.path.exists("/usr/bin/hostinfo"): - return int(commands.getoutput('/usr/bin/hostinfo |' + return int(subprocess.check_output('/usr/bin/hostinfo |' ' grep "processors are logically available." |' - ' awk "{ print \$1 }"')) + ' awk "{ print \$1 }"', shell=True)) win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS") if win_cpu_count: return int(win_cpu_count) @@ -409,7 +408,7 @@ def ReadVersionFile(): content = fd.read() fd.close() except: - print "Warning: Couldn't read VERSION file (%s)" % VERSION_FILE + print ("Warning: Couldn't read VERSION file (%s)" % VERSION_FILE) return None channel = match_against('^CHANNEL ([A-Za-z0-9]+)$', content) @@ -428,7 +427,7 @@ def ReadVersionFile(): channel, major, minor, patch, prerelease, prerelease_patch, abi_version, oldest_supported_abi_version) else: - print "Warning: VERSION file (%s) has wrong format" % VERSION_FILE + print ("Warning: VERSION file (%s) has wrong format" % VERSION_FILE) return None @@ -466,7 +465,7 @@ def GetGitRevision(): output = output.strip() # We expect a full git hash if len(output) != 40: - print "Warning: could not parse git commit, output was %s" % output + print ("Warning: could not parse git commit, output was %s" % output) return None return output @@ -498,7 +497,7 @@ def GetLatestDevTag(): cwd = DART_DIR) output, _ = p.communicate() if p.wait() != 0: - print "Warning: Could not get the most recent dev branch tag %s" % output + print ("Warning: Could not get the most recent dev branch tag %s" % output) return None return output.strip() @@ -528,7 +527,7 @@ def GetGitNumber(): number = int(output) return number + GIT_NUMBER_BASE except: - print "Warning: could not parse git count, output was %s" % output + print ("Warning: could not parse git count, output was %s" % output) return None @@ -610,20 +609,20 @@ def CheckedUnlink(name): """Unlink a file without throwing an exception.""" try: os.unlink(name) - except OSError, e: + except OSError as e: PrintError("os.unlink() " + str(e)) def Main(): - print "GuessOS() -> ", GuessOS() - print "GuessArchitecture() -> ", GuessArchitecture() - print "GuessCpus() -> ", GuessCpus() - print "IsWindows() -> ", IsWindows() - print "GuessVisualStudioPath() -> ", GuessVisualStudioPath() - print "GetGitRevision() -> ", GetGitRevision() - print "GetGitTimestamp() -> ", GetGitTimestamp() - print "GetVersionFileContent() -> ", GetVersionFileContent() - print "GetGitNumber() -> ", GetGitNumber() + print ("GuessOS() -> ", GuessOS()) + print ("GuessArchitecture() -> ", GuessArchitecture()) + print ("GuessCpus() -> ", GuessCpus()) + print ("IsWindows() -> ", IsWindows()) + print ("GuessVisualStudioPath() -> ", GuessVisualStudioPath()) + print ("GetGitRevision() -> ", GetGitRevision()) + print ("GetGitTimestamp() -> ", GetGitTimestamp()) + print ("GetVersionFileContent() -> ", GetVersionFileContent()) + print ("GetGitNumber() -> ", GetGitNumber()) class Error(Exception): @@ -660,7 +659,7 @@ def Touch(name): def ExecuteCommand(cmd): """Execute a command in a subprocess.""" - print 'Executing: ' + ' '.join(cmd) + print ('Executing: ' + ' '.join(cmd)) pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=IsWindows()) output = pipe.communicate() @@ -679,7 +678,7 @@ def CheckedInSdkPath(): try: osname = osdict[system] except KeyError: - print >>sys.stderr, ('WARNING: platform "%s" not supported') % (system) + sys.stderr.write('WARNING: platform "%s" not supported\n' % system) return None tools_dir = os.path.dirname(os.path.realpath(__file__)) return os.path.join(tools_dir, @@ -719,7 +718,7 @@ def CheckLinuxCoreDumpPattern(fatal=False): if fatal: raise Exception(message) else: - print message + print (message) return False return True @@ -743,11 +742,11 @@ class ChangedWorkingDirectory(object): def __enter__(self): self._old_cwd = os.getcwd() - print "Enter directory = ", self._working_directory + print ("Enter directory = ", self._working_directory) os.chdir(self._working_directory) def __exit__(self, *_): - print "Enter directory = ", self._old_cwd + print ("Enter directory = ", self._old_cwd) os.chdir(self._old_cwd) @@ -797,7 +796,7 @@ class WindowsCoreDumpEnabler(object): pass def __enter__(self): - print "INFO: Enabling coredump archiving into %s" % (WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER) + print ("INFO: Enabling coredump archiving into %s" % (WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER)) os.environ['DART_CRASHPAD_CRASHES_DIR'] = WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER def __exit__(self, *_): @@ -808,7 +807,7 @@ def TryUnlink(file): try: os.unlink(file) except Exception as error: - print "ERROR: Failed to remove %s: %s" % (file, error) + print ("ERROR: Failed to remove %s: %s" % (file, error)) class BaseCoreDumpArchiver(object): @@ -830,15 +829,15 @@ class BaseCoreDumpArchiver(object): try: return self._cleanup(); except Exception as error: - print "ERROR: Failure during cleanup: %s" % error + print ("ERROR: Failure during cleanup: %s" % error) return False def __enter__(self): - print "INFO: Core dump archiving is activated" + print ("INFO: Core dump archiving is activated") # Cleanup any stale files if self._safe_cleanup(): - print "WARNING: Found and removed stale coredumps" + print ("WARNING: Found and removed stale coredumps") def __exit__(self, *_): try: @@ -846,23 +845,23 @@ class BaseCoreDumpArchiver(object): if crashes: # If we get a ton of crashes, only archive 10 dumps. archive_crashes = crashes[:10] - print 'Archiving coredumps for crash (if possible):' + print ('Archiving coredumps for crash (if possible):') for crash in archive_crashes: - print '----> %s' % crash + print ('----> %s' % crash) sys.stdout.flush() self._archive(archive_crashes) else: - print "INFO: No unexpected crashes recorded" + print ("INFO: No unexpected crashes recorded") dumps = self._find_all_coredumps() if dumps: - print "INFO: However there are %d core dumps found" % len(dumps) + print ("INFO: However there are %d core dumps found" % len(dumps)) for dump in dumps: - print "INFO: -> %s" % dump - print + print ("INFO: -> %s" % dump) + print () except Exception as error: - print "ERROR: Failed to archive crashes: %s" % error + print ("ERROR: Failed to archive crashes: %s" % error) raise finally: @@ -902,10 +901,10 @@ class BaseCoreDumpArchiver(object): def _report_missing_crashes(self, missing, throw=True): missing_as_string = ', '.join([str(c) for c in missing]) other_files = list(glob.glob(os.path.join(self._search_dir, '*'))) - print >> sys.stderr, ( + sys.stderr.write( "Could not find crash dumps for '%s' in search directory '%s'.\n" "Existing files which *did not* match the pattern inside the search " - "directory are are:\n %s" + "directory are are:\n %s\n" % (missing_as_string, self._search_dir, '\n '.join(other_files))) # TODO: Figure out why windows coredump generation does not work. # See http://dartbug.com/36469 @@ -927,7 +926,7 @@ class BaseCoreDumpArchiver(object): def _move(self, files): for file in files: - print '+++ Moving %s to output_directory (%s)' % (file, self._output_directory) + print ('+++ Moving %s to output_directory (%s)' % (file, self._output_directory)) (name, is_binary) = self._get_file_name(file) destination = os.path.join(self._output_directory, name) shutil.move(file, destination) @@ -956,7 +955,7 @@ class BaseCoreDumpArchiver(object): gs_prefix = 'gs://%s' % storage_path http_prefix = 'https://storage.cloud.google.com/%s' % storage_path - print '\n--- Uploading into %s (%s) ---' % (gs_prefix, http_prefix) + print ('\n--- Uploading into %s (%s) ---' % (gs_prefix, http_prefix)) for file in files: tarname = self._tar(file) @@ -966,13 +965,13 @@ class BaseCoreDumpArchiver(object): try: gsutil.upload(tarname, gs_url) - print '+++ Uploaded %s (%s)' % (gs_url, http_url) + print ('+++ Uploaded %s (%s)' % (gs_url, http_url)) except Exception as error: - print '!!! Failed to upload %s, error: %s' % (tarname, error) + print ('!!! Failed to upload %s, error: %s' % (tarname, error)) TryUnlink(tarname) - print '--- Done ---\n' + print ('--- Done ---\n') def _find_all_coredumps(self): """Return coredumps that were recorded (if supported by the platform). @@ -1069,10 +1068,10 @@ class WindowsCoreDumpArchiver(BaseCoreDumpArchiver): if not dumps: return - print "### Collected %d crash dumps" % len(dumps) + print ("### Collected %d crash dumps" % len(dumps)) for dump in dumps: - print - print "### Dumping stacks from %s using CDB" % dump + print () + print ("### Dumping stacks from %s using CDB" % dump) cdb_output = subprocess.check_output( '"%s" -z "%s" -kqm -c "!uniqstack -b -v -p;qd"' % (cdb_path, dump), stderr=subprocess.STDOUT) @@ -1084,17 +1083,17 @@ class WindowsCoreDumpArchiver(BaseCoreDumpArchiver): elif line.startswith("quit:"): break elif output: - print line - print - print "#############################################" - print + print (line) + print () + print ("#############################################") + print () def __exit__(self, *args): try: self._dump_all_stacks() except Exception as error: - print "ERROR: Unable to dump stacks from dumps: %s" % error + print ("ERROR: Unable to dump stacks from dumps: %s" % error) super(WindowsCoreDumpArchiver, self).__exit__(*args)