diff --git a/build/README.dart b/build/README.dart.md similarity index 53% rename from build/README.dart rename to build/README.dart.md index ac57fbe6222..8fa8057f15c 100644 --- a/build/README.dart +++ b/build/README.dart.md @@ -3,5 +3,5 @@ This directory was taken from a snapshot of flutter/engine/src/build/. The snapshot was taken with a recursive copy `cp -R` of the directory from the flutter repository. -The contents is used to support the GN build system. - +The contents has now diverged significantly from Flutter, and is maintained +independantly by the Dart team. It is used to support the GN build system. diff --git a/build/vs_toolchain.py b/build/vs_toolchain.py index 357a7c54185..7c6a772edc2 100644 --- a/build/vs_toolchain.py +++ b/build/vs_toolchain.py @@ -3,11 +3,21 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# For Dart developers: +# This file keeps the MSVC toolchain up-to-date for Google developers. +# It is copied from Chromium: +# https://cs.chromium.org/chromium/src/build/vs_toolchain.py +# with modifications that update paths, and remove dependencies on gyp. +# To update to a new MSVC toolchain, copy the updated script from the Chromium +# tree, and edit to make it work in the Dart tree by updating paths and removing +# dependencies on any 'import gyp' in the original script. + import glob import json import os import pipes import platform +import re import shutil import stat import subprocess @@ -21,8 +31,8 @@ sys.path.insert(0, os.path.join(chrome_src, 'tools')) json_data_file = os.path.join(script_dir, 'win_toolchain.json') -# Use MSVS2015 as the default toolchain. -CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2015' +# Use MSVS2017 as the default toolchain. +CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2017' def SetEnvironmentAndGetRuntimeDllDirs(): @@ -51,12 +61,13 @@ def SetEnvironmentAndGetRuntimeDllDirs(): win_sdk = toolchain_data['win8sdk'] wdk = toolchain_data['wdk'] # TODO(scottmg): The order unfortunately matters in these. They should be - # split into separate keys for x86 and x64. (See CopyVsRuntimeDlls call - # below). http://crbug.com/345992 + # split into separate keys for x86 and x64. (See CopyDlls call below). + # http://crbug.com/345992 vs_runtime_dll_dirs = toolchain_data['runtime_dirs'] os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain os.environ['GYP_MSVS_VERSION'] = version + os.environ['WINDOWSSDKDIR'] = win_sdk os.environ['WDK_DIR'] = wdk # Include the VS runtime in the PATH in case it's not machine-installed. @@ -122,39 +133,39 @@ def DetectVisualStudioPath(): # build/toolchain/win/setup_toolchain.py as well. version_as_year = GetVisualStudioVersion() year_to_version = { - '2013': '12.0', '2015': '14.0', + '2017': '15.0', } if version_as_year not in year_to_version: raise Exception(('Visual Studio version %s (from GYP_MSVS_VERSION)' ' not supported. Supported versions are: %s') % ( version_as_year, ', '.join(year_to_version.keys()))) version = year_to_version[version_as_year] - keys = [r'HKLM\Software\Microsoft\VisualStudio\%s' % version, - r'HKLM\Software\Wow6432Node\Microsoft\VisualStudio\%s' % version] - for key in keys: - path = _RegistryGetValue(key, 'InstallDir') - if not path: - continue - path = os.path.normpath(os.path.join(path, '..', '..')) - return path + if version_as_year == '2017': + # The VC++ 2017 install location needs to be located using COM instead of + # the registry. For details see: + # https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/ + # For now we use a hardcoded default with an environment variable override. + for path in ( + os.environ.get('vs2017_install'), + r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional', + r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community'): + if path and os.path.exists(path): + return path + else: + keys = [r'HKLM\Software\Microsoft\VisualStudio\%s' % version, + r'HKLM\Software\Wow6432Node\Microsoft\VisualStudio\%s' % version] + for key in keys: + path = _RegistryGetValue(key, 'InstallDir') + if not path: + continue + path = os.path.normpath(os.path.join(path, '..', '..')) + return path raise Exception(('Visual Studio Version %s (from GYP_MSVS_VERSION)' ' not found.') % (version_as_year)) -def _VersionNumber(): - """Gets the standard version number ('120', '140', etc.) based on - GYP_MSVS_VERSION.""" - vs_version = GetVisualStudioVersion() - if vs_version == '2013': - return '120' - elif vs_version == '2015': - return '140' - else: - raise ValueError('Unexpected GYP_MSVS_VERSION') - - def _CopyRuntimeImpl(target, source, verbose=True): """Copy |source| to |target| if it doesn't already exist or if it needs to be updated (comparing last modified time as an approximate float match as for @@ -167,25 +178,17 @@ def _CopyRuntimeImpl(target, source, verbose=True): if verbose: print 'Copying %s to %s...' % (source, target) if os.path.exists(target): - # Make the file writable so that we can delete it now. - os.chmod(target, stat.S_IWRITE) + # Make the file writable so that we can delete it now, and keep it + # readable. + os.chmod(target, stat.S_IWRITE | stat.S_IREAD) os.unlink(target) shutil.copy2(source, target) - # Make the file writable so that we can overwrite or delete it later. - os.chmod(target, stat.S_IWRITE) + # Make the file writable so that we can overwrite or delete it later, + # keep it readable. + os.chmod(target, stat.S_IWRITE | stat.S_IREAD) -def _CopyRuntime2013(target_dir, source_dir, dll_pattern): - """Copy both the msvcr and msvcp runtime DLLs, only if the target doesn't - exist, but the target directory does exist.""" - for file_part in ('p', 'r'): - dll = dll_pattern % file_part - target = os.path.join(target_dir, dll) - source = os.path.join(source_dir, dll) - _CopyRuntimeImpl(target, source) - - -def _CopyRuntime2015(target_dir, source_dir, dll_pattern, suffix): +def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix): """Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't exist, but the target directory does exist.""" for file_part in ('msvcp', 'vccorlib', 'vcruntime'): @@ -193,14 +196,17 @@ def _CopyRuntime2015(target_dir, source_dir, dll_pattern, suffix): target = os.path.join(target_dir, dll) source = os.path.join(source_dir, dll) _CopyRuntimeImpl(target, source) - # OS installs of Visual Studio (and all installs of Windows 10) put the - # universal CRT files in c:\Windows\System32\downlevel - look for them there - # to support DEPOT_TOOLS_WIN_TOOLCHAIN=0. - if os.path.exists(os.path.join(source_dir, 'downlevel')): - ucrt_src_glob = os.path.join(source_dir, 'downlevel', 'api-ms-win-*.dll') - else: - ucrt_src_glob = os.path.join(source_dir, 'api-ms-win-*.dll') - ucrt_files = glob.glob(ucrt_src_glob) + # Copy the UCRT files needed by VS 2015 from the Windows SDK. This location + # includes the api-ms-win-crt-*.dll files that are not found in the Windows + # directory. These files are needed for component builds. + # If WINDOWSSDKDIR is not set use the default SDK path. This will be the case + # when DEPOT_TOOLS_WIN_TOOLCHAIN=0 and vcvarsall.bat has not been run. + win_sdk_dir = os.path.normpath( + os.environ.get('WINDOWSSDKDIR', + 'C:\\Program Files (x86)\\Windows Kits\\10')) + ucrt_dll_dirs = os.path.join(win_sdk_dir, 'Redist', 'ucrt', 'DLLs', + target_cpu) + ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll')) assert len(ucrt_files) > 0 for ucrt_src_file in ucrt_files: file_part = os.path.basename(ucrt_src_file) @@ -210,62 +216,72 @@ def _CopyRuntime2015(target_dir, source_dir, dll_pattern, suffix): os.path.join(source_dir, 'ucrtbase' + suffix)) +def FindVCToolsRoot(): + """In VS2017 the PGO runtime dependencies are located in + {toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/Host{target_cpu}/{target_cpu}/, the + {version_number} part is likely to change in case of a minor update of the + toolchain so we don't hardcode this value here (except for the major number). + + This returns the '{toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/' path. + + This function should only be called when using VS2017. + """ + assert GetVisualStudioVersion() == '2017' + SetEnvironmentAndGetRuntimeDllDirs() + assert ('GYP_MSVS_OVERRIDE_PATH' in os.environ) + vc_tools_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], + 'VC', 'Tools', 'MSVC') + for directory in os.listdir(vc_tools_msvc_root): + if not os.path.isdir(os.path.join(vc_tools_msvc_root, directory)): + continue + if re.match('14\.\d+\.\d+', directory): + return os.path.join(vc_tools_msvc_root, directory, 'bin') + raise Exception('Unable to find the VC tools directory.') + + +def _CopyPGORuntime(target_dir, target_cpu): + """Copy the runtime dependencies required during a PGO build. + """ + env_version = GetVisualStudioVersion() + # These dependencies will be in a different location depending on the version + # of the toolchain. + if env_version == '2015': + pgo_x86_runtime_dir = os.path.join(os.environ.get('GYP_MSVS_OVERRIDE_PATH'), + 'VC', 'bin') + pgo_x64_runtime_dir = os.path.join(pgo_x86_runtime_dir, 'amd64') + elif env_version == '2017': + pgo_runtime_root = FindVCToolsRoot() + assert pgo_runtime_root + # There's no version of pgosweep.exe in HostX64/x86, so we use the copy + # from HostX86/x86. + pgo_x86_runtime_dir = os.path.join(pgo_runtime_root, 'HostX86', 'x86') + pgo_x64_runtime_dir = os.path.join(pgo_runtime_root, 'HostX64', 'x64') + else: + raise Exception('Unexpected toolchain version: %s.' % env_version) + + # We need to copy 2 runtime dependencies used during the profiling step: + # - pgort140.dll: runtime library required to run the instrumented image. + # - pgosweep.exe: executable used to collect the profiling data + pgo_runtimes = ['pgort140.dll', 'pgosweep.exe'] + for runtime in pgo_runtimes: + if target_cpu == 'x86': + source = os.path.join(pgo_x86_runtime_dir, runtime) + elif target_cpu == 'x64': + source = os.path.join(pgo_x64_runtime_dir, runtime) + else: + raise NotImplementedError("Unexpected target_cpu value: " + target_cpu) + if not os.path.exists(source): + raise Exception('Unable to find %s.' % source) + _CopyRuntimeImpl(os.path.join(target_dir, runtime), source) + + def _CopyRuntime(target_dir, source_dir, target_cpu, debug): """Copy the VS runtime DLLs, only if the target doesn't exist, but the target - directory does exist. Handles VS 2013 and VS 2015.""" + directory does exist. Handles VS 2015 and VS 2017.""" suffix = "d.dll" if debug else ".dll" - if GetVisualStudioVersion() == '2015': - _CopyRuntime2015(target_dir, source_dir, '%s140' + suffix, suffix) - else: - _CopyRuntime2013(target_dir, source_dir, 'msvc%s120' + suffix) - - # Copy the PGO runtime library to the release directories. - if not debug and os.environ.get('GYP_MSVS_OVERRIDE_PATH'): - pgo_x86_runtime_dir = os.path.join(os.environ.get('GYP_MSVS_OVERRIDE_PATH'), - 'VC', 'bin') - pgo_x64_runtime_dir = os.path.join(pgo_x86_runtime_dir, 'amd64') - pgo_runtime_dll = 'pgort' + _VersionNumber() + '.dll' - if target_cpu == "x86": - source_x86 = os.path.join(pgo_x86_runtime_dir, pgo_runtime_dll) - if os.path.exists(source_x86): - _CopyRuntimeImpl(os.path.join(target_dir, pgo_runtime_dll), source_x86) - elif target_cpu == "x64": - source_x64 = os.path.join(pgo_x64_runtime_dir, pgo_runtime_dll) - if os.path.exists(source_x64): - _CopyRuntimeImpl(os.path.join(target_dir, pgo_runtime_dll), - source_x64) - else: - raise NotImplementedError("Unexpected target_cpu value:" + target_cpu) - - -def CopyVsRuntimeDlls(output_dir, runtime_dirs): - """Copies the VS runtime DLLs from the given |runtime_dirs| to the output - directory so that even if not system-installed, built binaries are likely to - be able to run. - - This needs to be run after gyp has been run so that the expected target - output directories are already created. - - This is used for the GYP build and gclient runhooks. - """ - x86, x64 = runtime_dirs - out_debug = os.path.join(output_dir, 'Debug') - out_debug_nacl64 = os.path.join(output_dir, 'Debug', 'x64') - out_release = os.path.join(output_dir, 'Release') - out_release_nacl64 = os.path.join(output_dir, 'Release', 'x64') - out_debug_x64 = os.path.join(output_dir, 'Debug_x64') - out_release_x64 = os.path.join(output_dir, 'Release_x64') - - if os.path.exists(out_debug) and not os.path.exists(out_debug_nacl64): - os.makedirs(out_debug_nacl64) - if os.path.exists(out_release) and not os.path.exists(out_release_nacl64): - os.makedirs(out_release_nacl64) - _CopyRuntime(out_debug, x86, "x86", debug=True) - _CopyRuntime(out_release, x86, "x86", debug=False) - _CopyRuntime(out_debug_x64, x64, "x64", debug=True) - _CopyRuntime(out_release_x64, x64, "x64", debug=False) - _CopyRuntime(out_debug_nacl64, x64, "x64", debug=True) - _CopyRuntime(out_release_nacl64, x64, "x64", debug=False) + # VS 2017 uses the same CRT DLLs as VS 2015. + _CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix, + suffix) def CopyDlls(target_dir, configuration, target_cpu): @@ -276,8 +292,6 @@ def CopyDlls(target_dir, configuration, target_cpu): The debug configuration gets both the debug and release DLLs; the release config only the latter. - - This is used for the GN build. """ vs_runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() if not vs_runtime_dll_dirs: @@ -288,16 +302,56 @@ def CopyDlls(target_dir, configuration, target_cpu): _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False) if configuration == 'Debug': _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True) + else: + _CopyPGORuntime(target_dir, target_cpu) + + _CopyDebugger(target_dir, target_cpu) + + +def _CopyDebugger(target_dir, target_cpu): + """Copy dbghelp.dll and dbgcore.dll into the requested directory as needed. + + target_cpu is one of 'x86' or 'x64'. + + dbghelp.dll is used when Chrome needs to symbolize stacks. Copying this file + from the SDK directory avoids using the system copy of dbghelp.dll which then + ensures compatibility with recent debug information formats, such as VS + 2017 /debug:fastlink PDBs. + + dbgcore.dll is needed when using some functions from dbghelp.dll (like + MinidumpWriteDump). + """ + win_sdk_dir = SetEnvironmentAndGetSDKDir() + if not win_sdk_dir: + return + + # List of debug files that should be copied, the first element of the tuple is + # the name of the file and the second indicates if it's optional. + debug_files = [('dbghelp.dll', False), ('dbgcore.dll', True)] + for debug_file, is_optional in debug_files: + full_path = os.path.join(win_sdk_dir, 'Debuggers', target_cpu, debug_file) + if not os.path.exists(full_path): + if is_optional: + continue + else: + raise Exception('%s not found in "%s"\r\nYou must install the ' + '"Debugging Tools for Windows" feature from the Windows' + ' 10 SDK.' % (debug_file, full_path)) + target_path = os.path.join(target_dir, debug_file) + _CopyRuntimeImpl(target_path, full_path) def _GetDesiredVsToolchainHashes(): """Load a list of SHA1s corresponding to the toolchains that we want installed to build with.""" - if GetVisualStudioVersion() == '2015': - # Update 3 final with patches with 10.0.10586.0 SDK. - return ['d5dc33b15d1b2c086f2f6632e2fd15882f80dbd3'] - else: - return ['03a4e939cd325d6bc5216af41b92d02dda1366a6'] + env_version = GetVisualStudioVersion() + if env_version == '2015': + # Update 3 final with 10.0.15063.468 SDK and no vctip.exe. + return ['f53e4598951162bad6330f7a167486c7ae5db1e5'] + if env_version == '2017': + # VS 2017 Update 3.2 with 10.0.15063.468 SDK. + return ['9bc7ccbf9f4bd50d4a3bd185e8ca94ff1618de0b'] + raise Exception('Unsupported VS version %s' % env_version) def ShouldUpdateToolchain(): @@ -353,10 +407,10 @@ def NormalizePath(path): return path -def GetToolchainDir(): - """Gets location information about the current toolchain (must have been +def SetEnvironmentAndGetSDKDir(): + """Gets location information about the current sdk (must have been previously updated by 'update'). This is used for the GN build.""" - runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() + SetEnvironmentAndGetRuntimeDllDirs() # If WINDOWSSDKDIR is not set, search the default SDK path and set it. if not 'WINDOWSSDKDIR' in os.environ: @@ -364,6 +418,15 @@ def GetToolchainDir(): if os.path.isdir(default_sdk_path): os.environ['WINDOWSSDKDIR'] = default_sdk_path + return NormalizePath(os.environ['WINDOWSSDKDIR']) + + +def GetToolchainDir(): + """Gets location information about the current toolchain (must have been + previously updated by 'update'). This is used for the GN build.""" + runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() + win_sdk_dir = SetEnvironmentAndGetSDKDir() + print '''vs_path = "%s" sdk_path = "%s" vs_version = "%s" @@ -371,7 +434,7 @@ wdk_dir = "%s" runtime_dirs = "%s" ''' % ( NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH']), - NormalizePath(os.environ['WINDOWSSDKDIR']), + win_sdk_dir, GetVisualStudioVersion(), NormalizePath(os.environ.get('WDK_DIR', '')), os.path.pathsep.join(runtime_dll_dirs or ['None']))