diff --git a/tools/bots/dart_sdk.py b/tools/bots/dart_sdk.py deleted file mode 100755 index 87ab3b67b08a..000000000000 --- a/tools/bots/dart_sdk.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file -# for details. All rights reserved. Use of this source code is governed by a -# BSD-style license that can be found in the LICENSE file. - -import argparse -import os -import os.path -import shutil -import sys -import subprocess - -import bot_utils - - -def BuildRootPath(path, arch='x64', build_mode='release'): - return os.path.join(bot_utils.DART_DIR, - utils.GetBuildRoot(BUILD_OS, build_mode, arch), path) - - -def UploadFlutterCipd(arch, sdk_path, channel): - cipd_os = bot_utils.SYSTEM_TO_CIPD[BUILD_OS] - cipd_arch = bot_utils.ARCH_TO_CIPD[arch] - cipd_platform = cipd_os + '-' + cipd_arch - version = utils.GetVersion() - name = 'flutter/dart-sdk/%s' % cipd_platform - version_tag = 'version:%s' % version - git_tag = 'git_revision:%s' % utils.GetGitRevision() - bot_utils.run([ - 'cipd', 'create', '-name', name, '-in', sdk_path, '-install-mode', - 'copy', '-tag', version_tag, '-tag', git_tag, '-ref', channel, - '-preserve-writable' - ]) - - -def BuildDartdocAPIDocs(dirname): - dart_sdk = BuildRootPath('dart-sdk') - dart_exe = os.path.join(dart_sdk, 'bin', 'dart') - dartdoc_dart = os.path.join(bot_utils.DART_DIR, 'third_party', 'pkg', - 'dartdoc', 'bin', 'dartdoc.dart') - footer_text_file = os.path.join(bot_utils.DART_DIR, 'tools', 'bots', - 'dartdoc_footer_text.html') - footer_file = os.path.join(bot_utils.DART_DIR, 'tools', 'bots', - 'dartdoc_footer.html') - url = 'https://api.dartlang.org/stable' - print('Build API docs by dartdoc') - bot_utils.run([ - dart_exe, dartdoc_dart, '--sdk-docs', '--output', dirname, - '--footer-text', footer_text_file, '--footer', footer_file, - '--rel-canonical-prefix=' + url - ]) - - -def CreateUploadVersionFile(): - file_path = BuildRootPath('VERSION') - with open(file_path, 'w') as fd: - fd.write(utils.GetVersionFileContent()) - DartArchiveUploadVersionFile(file_path) - - -def DartArchiveUploadVersionFile(version_file): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - revision = utils.GetArchiveVersion() - for revision in [revision, 'latest']: - destination = namer.version_filepath(revision) - DartArchiveFile(version_file, destination, checksum_files=False) - - -def CreateAndUploadSDKZip(arch, sdk_path): - sdk_zip = BuildRootPath('dartsdk-%s-%s.zip' % (BUILD_OS, arch), arch=arch) - FileDelete(sdk_zip) - CreateZip(sdk_path, sdk_zip) - DartArchiveUploadSDKs(BUILD_OS, arch, sdk_zip) - - -def DartArchiveUploadSDKs(system, arch, sdk_zip): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - git_number = utils.GetArchiveVersion() - git_hash = 'hash/%s' % utils.GetGitRevision() - for revision in [git_number, git_hash, 'latest']: - path = namer.sdk_zipfilepath(revision, system, arch, 'release') - DartArchiveFile(sdk_zip, path, checksum_files=True) - - -def DartArchiveUnstrippedBinaries(arch): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - revision = utils.GetArchiveVersion() - binary = BuildRootPath(namer.unstripped_filename(BUILD_OS), arch=arch) - gs_path = namer.unstripped_filepath(revision, BUILD_OS, arch) - DartArchiveFile(binary, gs_path) - - -def CreateUploadAPIDocs(): - dartdoc_dir = BuildRootPath('gen-dartdocs') - dartdoc_zip = BuildRootPath('dartdocs-api.zip') - if CHANNEL == bot_utils.Channel.TRY or DART_EXPERIMENTAL_BUILD == '1': - BuildDartdocAPIDocs(dartdoc_dir) - else: - UploadApiLatestFile() - BuildDartdocAPIDocs(dartdoc_dir) - UploadDartdocApiDocs(dartdoc_dir) - CreateZip(dartdoc_dir, dartdoc_zip) - DartArchiveUploadDartdocAPIDocs(dartdoc_zip) - - -def DartArchiveUploadDartdocAPIDocs(api_zip): - namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) - revision = utils.GetArchiveVersion() - for revision in [revision, 'latest']: - destination = (namer.apidocs_directory(revision) + '/' + - namer.dartdocs_zipfilename()) - DartArchiveFile(api_zip, destination, checksum_files=False) - - -def UploadDartdocApiDocs(dir_name): - apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL) - revision = utils.GetArchiveVersion() - dartdocs_destination_gcsdir = apidocs_namer.dartdocs_dirpath(revision) - - # Return early if the documents have already been uploaded. - # This can happen if a build was forced, or a commit had no changes in the - # dart repository (e.g. DEPS file update). - if GsutilExists(dartdocs_destination_gcsdir): - print("Not uploading api docs, since %s is already present." % - dartdocs_destination_gcsdir) - return - - # Upload everything inside the built apidocs directory. - gsutil = bot_utils.GSUtil() - gsutil.upload( - dir_name, - dartdocs_destination_gcsdir, - recursive=True, - multithread=True) - - -def UploadApiLatestFile(): - apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL) - revision = utils.GetArchiveVersion() - apidocs_destination_latestfile = apidocs_namer.docs_latestpath(revision) - # Update latest.txt to contain the newest revision. - with utils.TempDir('latest_file') as temp_dir: - latest_file = os.path.join(temp_dir, 'latest.txt') - with open(latest_file, 'w') as f: - f.write('%s' % revision) - DartArchiveFile(latest_file, apidocs_destination_latestfile) - - -def GsutilExists(gsu_path): - # This is a little hackish, but it is basically a one off doing very - # specialized check that we don't use elsewhere. - gsutilTool = os.path.join(bot_utils.DART_DIR, 'third_party', 'gsutil', - 'gsutil') - (_, stderr, returncode) = bot_utils.run([gsutilTool, 'ls', gsu_path], - throw_on_error=False) - # If the returncode is nonzero and we can find a specific error message, - # we know there are no objects with a prefix of [gsu_path]. - missing = ( - returncode and - 'CommandException: One or more URLs matched no objects.' in stderr) - # Either the returncode has to be zero or the object must be missing, - # otherwise throw an exception. - if not missing and returncode: - raise Exception("Failed to determine whether %s exists" % gsu_path) - return not missing - - -def CreateZip(directory, target_file): - root = os.path.dirname(directory) - base = os.path.basename(directory) - f = shutil.make_archive(target_file, 'zip', root, base) - # make_archive will appened '.zip' to the filename, so we have to rename - # to avoid having it being '.zip.zip' - shutil.move(f, target_file) - - -def FileDelete(f): - if os.path.exists(f): - os.remove(f) - - -def CopyBetween(src_path, dst_path, *relatives): - try: - os.makedirs(os.path.join(dst_path, *relatives[:-1])) - except OSError: - # This is fine. - pass - shutil.copy2( - os.path.join(src_path, *relatives), - os.path.join(dst_path, *relatives[:-1])) - - -def GuessExtension(binary): - if 'win' in BUILD_OS: - return binary + '.exe' - return binary - - -def DartArchiveFile(local_path, remote_path, checksum_files=False): - gsutil = bot_utils.GSUtil() - gsutil.upload(local_path, remote_path) - if checksum_files: - # 'local_path' may have a different filename than 'remote_path'. So we need - # to make sure the *.md5sum file contains the correct name. - assert '/' in remote_path and not remote_path.endswith('/') - - mangled_filename = remote_path[remote_path.rfind('/') + 1:] - local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path, - mangled_filename) - gsutil.upload(local_md5sum, remote_path + '.md5sum') - local_sha256 = bot_utils.CreateSha256ChecksumFile( - local_path, mangled_filename) - gsutil.upload(local_sha256, remote_path + '.sha256sum') - - -def Run(command, env=None): - print("Running %s" % ' '.join(command)) - print("Environment %s" % env) - sys.stdout.flush() - exit_code = subprocess.call(command) - if exit_code != 0: - raise OSError(exit_code) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--arch', - default='x64', - help="comma separated list of architectures") - parser.add_argument('command', choices=['api_docs'], nargs='?') - args = parser.parse_args() - archs = args.arch.split(',') - command = args.command - - utils = bot_utils.GetUtils() - - BUILD_OS = utils.GuessOS() - BUILDER_NAME = os.environ.get('BUILDBOT_BUILDERNAME') - DART_EXPERIMENTAL_BUILD = os.environ.get('DART_EXPERIMENTAL_BUILD') - CHANNEL = bot_utils.GetChannelFromName(BUILDER_NAME) - - if command == 'api_docs': - if BUILD_OS == 'linux': - CreateUploadAPIDocs() - elif CHANNEL != bot_utils.Channel.TRY and DART_EXPERIMENTAL_BUILD != '1': - for arch in archs: - print('Create and upload sdk zip for ' + arch) - sdk_path = BuildRootPath('dart-sdk', arch=arch) - CreateAndUploadSDKZip(arch, sdk_path) - DartArchiveUnstrippedBinaries(arch) - if CHANNEL != bot_utils.Channel.BLEEDING_EDGE: - try: - UploadFlutterCipd(arch, sdk_path, CHANNEL) - except Exception as error: - print('Error uploading to CIPD:') - print(repr(error)) - if BUILD_OS == 'linux': - CreateUploadVersionFile() - else: - print('Skipping upload on tryjobs for archs: %s' % archs) diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 2f9a5b828748..d7aac9192499 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -3149,51 +3149,6 @@ } ] }, - { - "builders": [ - "dart-sdk-linux" - ], - "meta": { - "description": "This configuration is used by the linux sdk builder. The difference between this configuration and the other sdk builder configurations is that this one also generates the api docs." - }, - "steps": [ - { - "name": "build dart (clang builds)", - "script": "tools/build.py", - "arguments": [ - "--arch=ia32,x64,arm,arm64", - "--mode=release", - "--check-clean", - "create_sdk" - ] - }, - { - "name": "build dart (gcc builds)", - "script": "tools/build.py", - "arguments": [ - "--no-clang", - "--arch=riscv64", - "--mode=release", - "--check-clean", - "create_sdk" - ] - }, - { - "name": "upload sdk", - "script": "tools/bots/dart_sdk.py", - "arguments": [ - "--arch=ia32,x64,arm,arm64,riscv64" - ] - }, - { - "name": "build api docs", - "script": "tools/bots/dart_sdk.py", - "arguments": [ - "api_docs" - ] - } - ] - }, { "builders": [ "debianpackage-linux"