#!/usr/bin/env python3 # # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. import hashlib import imp import os import subprocess import sys DART_DIR = os.path.abspath( os.path.normpath(os.path.join(__file__, '..', '..', '..'))) def GetUtils(): '''Dynamically load the tools/utils.py python module.''' return imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py')) SYSTEM_RENAMES = { 'win32': 'windows', 'windows': 'windows', 'win': 'windows', 'linux': 'linux', 'linux2': 'linux', 'lucid32': 'linux', 'lucid64': 'linux', 'darwin': 'macos', 'mac': 'macos', 'macos': 'macos', } ARCH_RENAMES = { 'ia32': 'ia32', 'x64': 'x64', 'arm': 'arm', 'arm64': 'arm64', 'riscv64': 'riscv64', } SYSTEM_TO_CIPD = { 'win32': 'windows', 'windows': 'windows', 'linux': 'linux', 'macos': 'mac', } ARCH_TO_CIPD = { 'ia32': '386', 'x64': 'amd64', 'arm': 'arm6l', 'arm64': 'arm64', 'riscv64': 'riscv64', } class Channel(object): BETA = 'beta' BLEEDING_EDGE = 'be' DEV = 'dev' STABLE = 'stable' TRY = 'try' INTEGRATION = 'integration' ALL_CHANNELS = [BETA, BLEEDING_EDGE, DEV, STABLE, TRY, INTEGRATION] class ReleaseType(object): RAW = 'raw' SIGNED = 'signed' RELEASE = 'release' ALL_TYPES = [RAW, SIGNED, RELEASE] class Mode(object): RELEASE = 'release' DEBUG = 'debug' ALL_MODES = [RELEASE, DEBUG] class GCSNamer(object): """ This class is used for naming objects in our "gs://dart-archive/" GoogleCloudStorage bucket. It's structure is as follows: For every (channel,revision,release-type) tuple we have a base path: gs://dart-archive/channels/{be,beta,dev,stable,try,integration} /{raw,signed,release}/{revision,latest}/ Under every base path, the following structure is used: - /VERSION - /api-docs/dartdocs-gen-api.zip - /sdk/dartsdk-{linux,macos,windows}-{ia32,x64}-release.zip """ def __init__(self, channel=Channel.BLEEDING_EDGE, release_type=ReleaseType.RAW, internal=False): assert channel in Channel.ALL_CHANNELS assert release_type in ReleaseType.ALL_TYPES self.channel = channel self.release_type = release_type if internal: self.bucket = 'gs://dart-archive-internal' else: self.bucket = 'gs://dart-archive' # Functions for querying complete gs:// filepaths def version_filepath(self, revision): return '%s/channels/%s/%s/%s/VERSION' % (self.bucket, self.channel, self.release_type, revision) def sdk_zipfilepath(self, revision, system, arch, mode): return '/'.join([ self.sdk_directory(revision), self.sdk_zipfilename(system, arch, mode) ]) def unstripped_filepath(self, revision, system, arch): return '/'.join([ self._variant_directory('unstripped', revision), system, arch, self.unstripped_filename(system) ]) def apidocs_zipfilepath(self, revision): return '/'.join( [self.apidocs_directory(revision), self.dartdocs_zipfilename()]) # Functions for querying gs:// directories def base_directory(self, revision): return '%s/channels/%s/%s/%s' % (self.bucket, self.channel, self.release_type, revision) def sdk_directory(self, revision): return self._variant_directory('sdk', revision) def linux_packages_directory(self, revision): return '/'.join([self._variant_directory('linux_packages', revision)]) def src_directory(self, revision): return self._variant_directory('src', revision) def apidocs_directory(self, revision): return self._variant_directory('api-docs', revision) def misc_directory(self, revision): return self._variant_directory('misc', revision) def _variant_directory(self, name, revision): return '%s/%s' % (self.base_directory(revision), name) # Functions for querying filenames def dartdocs_zipfilename(self): return 'dartdocs-gen-api.zip' def sdk_zipfilename(self, system, arch, mode): assert mode in Mode.ALL_MODES return 'dartsdk-%s-%s-%s.zip' % (SYSTEM_RENAMES[system], ARCH_RENAMES[arch], mode) def unstripped_filename(self, system): return 'dart.exe' if system.startswith('win') else 'dart' class GCSNamerApiDocs(object): def __init__(self, channel=Channel.BLEEDING_EDGE): assert channel in Channel.ALL_CHANNELS self.channel = channel self.bucket = 'gs://dartlang-api-docs' def dartdocs_dirpath(self, revision): assert len('%s' % revision) > 0 if self.channel == Channel.BLEEDING_EDGE: return '%s/gen-dartdocs/builds/%s' % (self.bucket, revision) return '%s/gen-dartdocs/%s/%s' % (self.bucket, self.channel, revision) def docs_latestpath(self, revision): assert len('%s' % revision) > 0 return '%s/channels/%s/latest.txt' % (self.bucket, self.channel) def run(command, env=None, shell=False, throw_on_error=True): print("Running command: ", command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, shell=shell, universal_newlines=True) (stdout, stderr) = p.communicate() if throw_on_error and p.returncode != 0: print("Failed to execute '%s'. Exit code: %s." % (command, p.returncode), file=sys.stderr) print("stdout: ", stdout, file=sys.stderr) print("stderr: ", stderr, file=sys.stderr) raise Exception("Failed to execute %s." % command) return (stdout, stderr, p.returncode) class GSUtil(object): GSUTIL_PATH = None USE_DART_REPO_VERSION = False def _layzCalculateGSUtilPath(self): if not GSUtil.GSUTIL_PATH: dart_gsutil = os.path.join(DART_DIR, 'third_party', 'gsutil', 'gsutil') if os.path.isfile(dart_gsutil): GSUtil.GSUTIL_PATH = dart_gsutil elif GSUtil.USE_DART_REPO_VERSION: raise Exception("Dart repository version of gsutil required, " "but not found.") else: # We did not find gsutil, look in path possible_locations = list(os.environ['PATH'].split(os.pathsep)) for directory in possible_locations: location = os.path.join(directory, 'gsutil') if os.path.isfile(location): GSUtil.GSUTIL_PATH = location break assert GSUtil.GSUTIL_PATH def execute(self, gsutil_args): self._layzCalculateGSUtilPath() gsutil_command = [sys.executable, GSUtil.GSUTIL_PATH] return run(gsutil_command + gsutil_args) def upload(self, local_path, remote_path, recursive=False, multithread=False): assert remote_path.startswith('gs://') if multithread: args = ['-m', 'cp'] else: args = ['cp'] if recursive: args += ['-R'] args += [local_path, remote_path] self.execute(args) def cat(self, remote_path): assert remote_path.startswith('gs://') args = ['cat', remote_path] (stdout, _, _) = self.execute(args) return stdout def setGroupReadACL(self, remote_path, group): args = ['acl', 'ch', '-g', '%s:R' % group, remote_path] self.execute(args) def setContentType(self, remote_path, content_type): args = ['setmeta', '-h', 'Content-Type:%s' % content_type, remote_path] self.execute(args) def remove(self, remote_path, recursive=False): assert remote_path.startswith('gs://') args = ['rm'] if recursive: args += ['-R'] args += [remote_path] self.execute(args) def GetChannelFromName(name): """Get the channel from the name. Bleeding edge builders don't have a suffix.""" channel_name = name.split('-').pop() if channel_name in Channel.ALL_CHANNELS: return channel_name return Channel.BLEEDING_EDGE