diff --git a/taskcluster/kinds/beetmover/kind.yml b/taskcluster/kinds/beetmover-promote/kind.yml similarity index 50% rename from taskcluster/kinds/beetmover/kind.yml rename to taskcluster/kinds/beetmover-promote/kind.yml index bfd97717b2..28fd66b718 100644 --- a/taskcluster/kinds/beetmover/kind.yml +++ b/taskcluster/kinds/beetmover-promote/kind.yml @@ -17,149 +17,66 @@ kind-dependencies: - mac-notarization - repackage-signing +task-defaults: + beetmover-action: "push-to-candidates" + run-on-tasks-for: [action] + worker-type: beetmover + tasks: android-arm64: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-arm64-v8a-release.apk] dependencies: signing: signing-android-arm64/release - if-dependencies: [signing] attributes: build-type: android/arm64-v8a - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/arm64-v8a android-armv7: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-armeabi-v7a-release.apk] dependencies: signing: signing-android-armv7/release - if-dependencies: [signing] attributes: build-type: android/armv7 - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/armv7 android-x86: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-x86-release.apk] dependencies: signing: signing-android-x86/release - if-dependencies: [signing] attributes: build-type: android/x86 - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/x86 android-x64: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-x86_64-release.apk] dependencies: signing: signing-android-x64/release - if-dependencies: [signing] attributes: build-type: android/x64 - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/x64 macos: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [MozillaVPN.pkg] dependencies: signing: signing-macos/opt mac-notarization: mac-notarization-macos/opt - if-dependencies: [signing] attributes: build-type: macos/opt - treeherder: - symbol: BM(macos) - kind: build - tier: 1 - platform: macos/opt windows: - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [MozillaVPN.msi] dependencies: repackage-signing: repackage-signing-msi - if-dependencies: [repackage-signing] attributes: build-type: windows/opt - treeherder: - symbol: BM(windows) - kind: build - tier: 1 - platform: windows/x86_64 addons-bundle: - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] # The addons-bundle release-artifacts are dynamically generated in the beetmover transform release-artifacts: [] dependencies: build: build-addons-bundle - if-dependencies: [build] attributes: build-type: addons/opt - treeherder: - symbol: BM(addons-bundle) - kind: build - tier: 1 - platform: addons/opt addons-manifest: - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: - manifest.json - manifest.json.sig dependencies: signing: signing-addons-bundle - if-dependencies: [signing] attributes: build-type: addons/opt - treeherder: - symbol: BM(addons-manifest) - kind: build - tier: 1 - platform: addons/opt diff --git a/taskcluster/kinds/beetmover-ship/kind.yml b/taskcluster/kinds/beetmover-ship/kind.yml new file mode 100644 index 0000000000..c191b8734e --- /dev/null +++ b/taskcluster/kinds/beetmover-ship/kind.yml @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - mozilla_taskgraph.transforms.scriptworker.release_artifacts + - mozillavpn_taskgraph.transforms.beetmover + - taskgraph.transforms.task + +kind-dependencies: + - beetmover-promote + - build + - signing + +task-defaults: + from-deps: + kinds: [beetmover-promote] + group-by: all + set-name: false + unique-kinds: false + worker-type: beetmover + worker: + chain-of-trust: true + max-run-time: 1800 + run-on-tasks-for: [action] + +tasks: + client: + beetmover-action: "push-to-releases" + from-deps: + copy-attributes: true + with-attributes: + shipping-phase: promote-client + + # Beetmoverscript doesn't support the `push-to-release` action for VPN + # addons yet, as they use a slightly different directory structure on + # archive.mozilla.org. + # + # For that reason, we need to use the `direct-push-to-bucket` action to + # re-upload the build artifacts rather than copying them over from the + # candidates dir. This means we need to depend on the build dependencies + # even in the `ship` phase. We also depend on the `beetmover-promote` tasks + # just to ensure we don't skip uploading to the candidates dir. + addons-bundle: + beetmover-action: "direct-push-to-bucket" + attributes: + build-type: "addons/opt" + from-deps: + with-attributes: + shipping-phase: promote-addons + dependencies: + build: build-addons-bundle + # The addons-bundle release-artifacts are dynamically generated in the beetmover transform + release-artifacts: [] + + addons-manifest: + beetmover-action: "direct-push-to-bucket" + attributes: + build-type: "addons/opt" + from-deps: + with-attributes: + shipping-phase: promote-addons + dependencies: + signing: signing-addons-bundle + release-artifacts: + - manifest.json + - manifest.json.sig diff --git a/taskcluster/kinds/release-notify/kind.yml b/taskcluster/kinds/release-notify/kind.yml index aec2e775a1..6ef8ec8c45 100644 --- a/taskcluster/kinds/release-notify/kind.yml +++ b/taskcluster/kinds/release-notify/kind.yml @@ -10,7 +10,8 @@ transforms: - taskgraph.transforms.task:transforms kind-dependencies: - - beetmover + - beetmover-promote + - beetmover-ship task-defaults: description: "Sends notifications to #mozilla-vpn-release in Slack" diff --git a/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py b/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py index 807c0f152d..d677770ed2 100644 --- a/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py +++ b/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py @@ -6,10 +6,43 @@ import os.path from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.task import task_description_schema +from taskgraph.util.schema import Schema +from voluptuous import Extra, Optional, Required transforms = TransformSequence() +beetmover_schema = Schema( + { + Required("beetmover-action"): str, + Required("attributes"): { + Required("build-type"): str, + Required("release-artifacts"): [dict], + Extra: object, + }, + Required("dependencies"): task_description_schema["dependencies"], + Required("name"): str, + Required("run-on-tasks-for"): task_description_schema["run-on-tasks-for"], + Required("worker-type"): task_description_schema["worker-type"], + Optional("task-from"): task_description_schema["task-from"], + } +) + + +@transforms.add +def remove_worker(config, tasks): + """The `release_artifacts` transforms add a key to 'worker' which we don't + use here, remove it.""" + for task in tasks: + if "worker" in task: + del task["worker"] + yield task + + +transforms.add_validate(beetmover_schema) + + @transforms.add def add_addons_release_artifacts(config, tasks): for task in tasks: @@ -36,46 +69,92 @@ def add_addons_release_artifacts(config, tasks): @transforms.add def add_beetmover_worker_config(config, tasks): + build_id = config.params["moz_build_date"] + build_type_os = { + "macos/opt": "mac", + "windows/opt": "windows", + "android/x86": "android", + "android/x64": "android", + "android/armv7": "android", + "android/arm64-v8a": "android", + } + + if config.params["version"]: + app_version = config.params["version"] + elif "releases" in config.params["head_ref"]: + app_version = config.params["head_ref"].split("/")[-1] + else: + app_version = "" # addons are not versioned + + is_production = ( + config.params["level"] == "3" and config.params["tasks_for"] == "action" + ) + bucket = "release" if is_production else "dep" + archive_url = ( + "https://ftp.mozilla.org/" if is_production else "https://ftp.stage.mozaws.net/" + ) + short_phase = config.kind[len("beetmover-"):] + for task in tasks: worker_type = task["worker-type"] - is_relpro = ( - config.params["level"] == "3" - and config.params["tasks_for"] in task["run-on-tasks-for"] - ) - bucket = "release" if is_relpro else "dep" - build_id = config.params["moz_build_date"] build_type = task["attributes"]["build-type"] - build_type_os = { - "macos/opt": "mac", - "windows/opt": "windows", - "android/x86": "android", - "android/x64": "android", - "android/armv7": "android", - "android/arm64-v8a": "android", - } build_os = build_type_os.get(build_type) - shipping_phase = config.params.get("shipping_phase", "") + phase = f"{short_phase}-{'addons' if task['name'].startswith('addons') else 'client'}" - if config.params["version"]: - app_version = config.params["version"] - elif "releases" in config.params["head_ref"]: - app_version = config.params["head_ref"].split("/")[-1] - else: - app_version = "" # addons are not versioned + upstream_artifacts = [] + for dep in task["dependencies"]: + if dep not in ("build", "signing"): + continue + upstream_artifacts.append( + { + "taskId": {"task-reference": f"<{dep}>"}, + "taskType": dep if dep == "build" else "scriptworker", + "paths": [ + release_artifact["name"] + for release_artifact in task["attributes"]["release-artifacts"] + ], + } + ) + + worker = { + "action": task["beetmover-action"], + "artifact-map": [], + "bucket": bucket, + "build-number": int(build_id), + "release-properties": { + "app-name": "vpn", + "app-version": app_version, + "branch": config.params["head_ref"], + "build-id": build_id, + "platform": build_type, + }, + "upstream-artifacts": upstream_artifacts + } destination_paths = [] - if build_type == "addons/opt": + if task["name"].startswith("addons"): destination_paths.append( os.path.join( "pub", "vpn", "addons", - "releases" if shipping_phase.startswith("ship") else "candidates", + "releases" if short_phase == "ship" else "candidates", build_id, ) ) - elif shipping_phase == "promote-client": + if phase == "ship-addons": + destination_paths.append( + os.path.join( + "pub", + "vpn", + "addons", + "releases", + "latest", + ) + ) + elif phase == "promote-client": + assert build_os destination_paths.append( os.path.join( "pub", @@ -86,7 +165,8 @@ def add_beetmover_worker_config(config, tasks): build_os, ) ) - elif shipping_phase == "ship-client": + elif phase == "ship-client": + assert build_os destination_paths.append( os.path.join( "pub", @@ -97,37 +177,8 @@ def add_beetmover_worker_config(config, tasks): ) ) - if shipping_phase == "ship-addons": - destination_paths.append( - os.path.join( - "pub", - "vpn", - "addons", - "releases", - "latest", - ) - ) - - archive_url = ( - "https://ftp.mozilla.org/" if is_relpro else "https://ftp.stage.mozaws.net/" - ) - - upstream_artifacts = [] - for dep in task["dependencies"]: - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "taskType": dep if dep == "build" else "scriptworker", - "paths": [ - release_artifact["name"] - for release_artifact in task["attributes"]["release-artifacts"] - ], - } - ) - - artifact_map = [] for artifact in upstream_artifacts: - artifact_map.append( + worker["artifact-map"].append( { "taskId": artifact["taskId"], "paths": { @@ -147,49 +198,28 @@ def add_beetmover_worker_config(config, tasks): attributes = { **task["attributes"], - "shipping-phase": shipping_phase, + "shipping-phase": phase, } dest = ( - f"{archive_url}{destination_paths[0]}" if destination_paths else archive_url + f"{archive_url}{destination_paths[0]}" + if destination_paths + else archive_url ) if build_type == "addons/opt": task_description = ( f"This {worker_type} task will upload the {task['name']} to {dest}/" ) - elif shipping_phase == "ship-client": - task_description = f"This {worker_type} task will copy build {build_id} from candidates to releases" + elif phase == "ship-client": + task_description = f"This {worker_type} task will copy the v{app_version} build {build_id} candidate to releases" else: task_description = f"This {worker_type} task will upload a {build_os} release candidate for v{app_version} to {dest}/" - if not shipping_phase or shipping_phase.startswith("promote"): - action = "push-to-candidates" - elif shipping_phase == "ship-addons": - action = "direct-push-to-bucket" - elif shipping_phase == "ship-client": - action = "push-to-releases" - else: - raise Exception(f"Invalid shipping_phase `{shipping_phase}`") - extra = { "release_destinations": [ f"{archive_url}{dest}/" for dest in destination_paths ] } - worker = { - "upstream-artifacts": upstream_artifacts, - "bucket": bucket, - "action": action, - "release-properties": { - "app-name": "vpn", - "app-version": app_version, - "branch": config.params["head_ref"], - "build-id": build_id, - "platform": build_type, - }, - "artifact-map": artifact_map, - "build-number": int(build_id), - } task_def = { "name": task["name"], "description": task_description, diff --git a/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py b/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py index 5069f9677c..573fd701e2 100644 --- a/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py +++ b/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py @@ -131,7 +131,7 @@ def format_message(config, tasks): dirs = set() for label, dep_task in config.kind_dependencies_tasks.items(): - if label not in task["dependencies"] or dep_task.kind != "beetmover": + if label not in task["dependencies"] or not dep_task.kind.startswith("beetmover"): continue platform = dep_task.attributes["build-type"].rsplit("/")[0]