From 492001106f37736ba7ca9a3e993ae1bcb16e50c4 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Mon, 15 Jan 2024 10:03:19 -0500 Subject: [PATCH] [VPN-6080] Refactor beetmover tasks to be distinct to each shipping phase (#8931) * ci: Refactor some beetmover logic out of the 'tasks' hot loop * ci: Add version to beetmover ship-client task's description * ci: Remove 'if-dependencies', 'worker' and 'treeherder' keys from beetmover tasks The `beetmover` transforms weren't using these keys in any way, nor were they forwarding these keys on to the next set of transforms. So the configs defined here were no-ops. * ci: Consolidate beetmover logic to 'task-defaults' * ci: Add a schema for beetmover tasks * [VPN-6080] create distinct beetmover tasks for each shipping phase This refactors the beetmover transform to generate distinct tasks for each shipping phase. This is more inline with how other projects work and is a pre-requisite for integrating VPN with Shipit, which has assumptions around how the tasks are set up. Issue: #8919 --- .../{beetmover => beetmover-promote}/kind.yml | 93 +-------- taskcluster/kinds/beetmover-ship/kind.yml | 70 +++++++ taskcluster/kinds/release-notify/kind.yml | 3 +- .../transforms/beetmover.py | 194 ++++++++++-------- .../transforms/release_notify.py | 2 +- 5 files changed, 190 insertions(+), 172 deletions(-) rename taskcluster/kinds/{beetmover => beetmover-promote}/kind.yml (50%) create mode 100644 taskcluster/kinds/beetmover-ship/kind.yml diff --git a/taskcluster/kinds/beetmover/kind.yml b/taskcluster/kinds/beetmover-promote/kind.yml similarity index 50% rename from taskcluster/kinds/beetmover/kind.yml rename to taskcluster/kinds/beetmover-promote/kind.yml index bfd97717b2..28fd66b718 100644 --- a/taskcluster/kinds/beetmover/kind.yml +++ b/taskcluster/kinds/beetmover-promote/kind.yml @@ -17,149 +17,66 @@ kind-dependencies: - mac-notarization - repackage-signing +task-defaults: + beetmover-action: "push-to-candidates" + run-on-tasks-for: [action] + worker-type: beetmover + tasks: android-arm64: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-arm64-v8a-release.apk] dependencies: signing: signing-android-arm64/release - if-dependencies: [signing] attributes: build-type: android/arm64-v8a - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/arm64-v8a android-armv7: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-armeabi-v7a-release.apk] dependencies: signing: signing-android-armv7/release - if-dependencies: [signing] attributes: build-type: android/armv7 - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/armv7 android-x86: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-x86-release.apk] dependencies: signing: signing-android-x86/release - if-dependencies: [signing] attributes: build-type: android/x86 - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/x86 android-x64: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [mozillavpn-x86_64-release.apk] dependencies: signing: signing-android-x64/release - if-dependencies: [signing] attributes: build-type: android/x64 - treeherder: - symbol: BM(android) - kind: build - tier: 1 - platform: android/x64 macos: requires-level: 3 - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [MozillaVPN.pkg] dependencies: signing: signing-macos/opt mac-notarization: mac-notarization-macos/opt - if-dependencies: [signing] attributes: build-type: macos/opt - treeherder: - symbol: BM(macos) - kind: build - tier: 1 - platform: macos/opt windows: - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: [MozillaVPN.msi] dependencies: repackage-signing: repackage-signing-msi - if-dependencies: [repackage-signing] attributes: build-type: windows/opt - treeherder: - symbol: BM(windows) - kind: build - tier: 1 - platform: windows/x86_64 addons-bundle: - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] # The addons-bundle release-artifacts are dynamically generated in the beetmover transform release-artifacts: [] dependencies: build: build-addons-bundle - if-dependencies: [build] attributes: build-type: addons/opt - treeherder: - symbol: BM(addons-bundle) - kind: build - tier: 1 - platform: addons/opt addons-manifest: - worker-type: beetmover - worker: - chain-of-trust: true - max-run-time: 1800 - run-on-tasks-for: [action] release-artifacts: - manifest.json - manifest.json.sig dependencies: signing: signing-addons-bundle - if-dependencies: [signing] attributes: build-type: addons/opt - treeherder: - symbol: BM(addons-manifest) - kind: build - tier: 1 - platform: addons/opt diff --git a/taskcluster/kinds/beetmover-ship/kind.yml b/taskcluster/kinds/beetmover-ship/kind.yml new file mode 100644 index 0000000000..c191b8734e --- /dev/null +++ b/taskcluster/kinds/beetmover-ship/kind.yml @@ -0,0 +1,70 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.from_deps + - mozilla_taskgraph.transforms.scriptworker.release_artifacts + - mozillavpn_taskgraph.transforms.beetmover + - taskgraph.transforms.task + +kind-dependencies: + - beetmover-promote + - build + - signing + +task-defaults: + from-deps: + kinds: [beetmover-promote] + group-by: all + set-name: false + unique-kinds: false + worker-type: beetmover + worker: + chain-of-trust: true + max-run-time: 1800 + run-on-tasks-for: [action] + +tasks: + client: + beetmover-action: "push-to-releases" + from-deps: + copy-attributes: true + with-attributes: + shipping-phase: promote-client + + # Beetmoverscript doesn't support the `push-to-release` action for VPN + # addons yet, as they use a slightly different directory structure on + # archive.mozilla.org. + # + # For that reason, we need to use the `direct-push-to-bucket` action to + # re-upload the build artifacts rather than copying them over from the + # candidates dir. This means we need to depend on the build dependencies + # even in the `ship` phase. We also depend on the `beetmover-promote` tasks + # just to ensure we don't skip uploading to the candidates dir. + addons-bundle: + beetmover-action: "direct-push-to-bucket" + attributes: + build-type: "addons/opt" + from-deps: + with-attributes: + shipping-phase: promote-addons + dependencies: + build: build-addons-bundle + # The addons-bundle release-artifacts are dynamically generated in the beetmover transform + release-artifacts: [] + + addons-manifest: + beetmover-action: "direct-push-to-bucket" + attributes: + build-type: "addons/opt" + from-deps: + with-attributes: + shipping-phase: promote-addons + dependencies: + signing: signing-addons-bundle + release-artifacts: + - manifest.json + - manifest.json.sig diff --git a/taskcluster/kinds/release-notify/kind.yml b/taskcluster/kinds/release-notify/kind.yml index aec2e775a1..6ef8ec8c45 100644 --- a/taskcluster/kinds/release-notify/kind.yml +++ b/taskcluster/kinds/release-notify/kind.yml @@ -10,7 +10,8 @@ transforms: - taskgraph.transforms.task:transforms kind-dependencies: - - beetmover + - beetmover-promote + - beetmover-ship task-defaults: description: "Sends notifications to #mozilla-vpn-release in Slack" diff --git a/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py b/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py index 807c0f152d..d677770ed2 100644 --- a/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py +++ b/taskcluster/mozillavpn_taskgraph/transforms/beetmover.py @@ -6,10 +6,43 @@ import os.path from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.task import task_description_schema +from taskgraph.util.schema import Schema +from voluptuous import Extra, Optional, Required transforms = TransformSequence() +beetmover_schema = Schema( + { + Required("beetmover-action"): str, + Required("attributes"): { + Required("build-type"): str, + Required("release-artifacts"): [dict], + Extra: object, + }, + Required("dependencies"): task_description_schema["dependencies"], + Required("name"): str, + Required("run-on-tasks-for"): task_description_schema["run-on-tasks-for"], + Required("worker-type"): task_description_schema["worker-type"], + Optional("task-from"): task_description_schema["task-from"], + } +) + + +@transforms.add +def remove_worker(config, tasks): + """The `release_artifacts` transforms add a key to 'worker' which we don't + use here, remove it.""" + for task in tasks: + if "worker" in task: + del task["worker"] + yield task + + +transforms.add_validate(beetmover_schema) + + @transforms.add def add_addons_release_artifacts(config, tasks): for task in tasks: @@ -36,46 +69,92 @@ def add_addons_release_artifacts(config, tasks): @transforms.add def add_beetmover_worker_config(config, tasks): + build_id = config.params["moz_build_date"] + build_type_os = { + "macos/opt": "mac", + "windows/opt": "windows", + "android/x86": "android", + "android/x64": "android", + "android/armv7": "android", + "android/arm64-v8a": "android", + } + + if config.params["version"]: + app_version = config.params["version"] + elif "releases" in config.params["head_ref"]: + app_version = config.params["head_ref"].split("/")[-1] + else: + app_version = "" # addons are not versioned + + is_production = ( + config.params["level"] == "3" and config.params["tasks_for"] == "action" + ) + bucket = "release" if is_production else "dep" + archive_url = ( + "https://ftp.mozilla.org/" if is_production else "https://ftp.stage.mozaws.net/" + ) + short_phase = config.kind[len("beetmover-"):] + for task in tasks: worker_type = task["worker-type"] - is_relpro = ( - config.params["level"] == "3" - and config.params["tasks_for"] in task["run-on-tasks-for"] - ) - bucket = "release" if is_relpro else "dep" - build_id = config.params["moz_build_date"] build_type = task["attributes"]["build-type"] - build_type_os = { - "macos/opt": "mac", - "windows/opt": "windows", - "android/x86": "android", - "android/x64": "android", - "android/armv7": "android", - "android/arm64-v8a": "android", - } build_os = build_type_os.get(build_type) - shipping_phase = config.params.get("shipping_phase", "") + phase = f"{short_phase}-{'addons' if task['name'].startswith('addons') else 'client'}" - if config.params["version"]: - app_version = config.params["version"] - elif "releases" in config.params["head_ref"]: - app_version = config.params["head_ref"].split("/")[-1] - else: - app_version = "" # addons are not versioned + upstream_artifacts = [] + for dep in task["dependencies"]: + if dep not in ("build", "signing"): + continue + upstream_artifacts.append( + { + "taskId": {"task-reference": f"<{dep}>"}, + "taskType": dep if dep == "build" else "scriptworker", + "paths": [ + release_artifact["name"] + for release_artifact in task["attributes"]["release-artifacts"] + ], + } + ) + + worker = { + "action": task["beetmover-action"], + "artifact-map": [], + "bucket": bucket, + "build-number": int(build_id), + "release-properties": { + "app-name": "vpn", + "app-version": app_version, + "branch": config.params["head_ref"], + "build-id": build_id, + "platform": build_type, + }, + "upstream-artifacts": upstream_artifacts + } destination_paths = [] - if build_type == "addons/opt": + if task["name"].startswith("addons"): destination_paths.append( os.path.join( "pub", "vpn", "addons", - "releases" if shipping_phase.startswith("ship") else "candidates", + "releases" if short_phase == "ship" else "candidates", build_id, ) ) - elif shipping_phase == "promote-client": + if phase == "ship-addons": + destination_paths.append( + os.path.join( + "pub", + "vpn", + "addons", + "releases", + "latest", + ) + ) + elif phase == "promote-client": + assert build_os destination_paths.append( os.path.join( "pub", @@ -86,7 +165,8 @@ def add_beetmover_worker_config(config, tasks): build_os, ) ) - elif shipping_phase == "ship-client": + elif phase == "ship-client": + assert build_os destination_paths.append( os.path.join( "pub", @@ -97,37 +177,8 @@ def add_beetmover_worker_config(config, tasks): ) ) - if shipping_phase == "ship-addons": - destination_paths.append( - os.path.join( - "pub", - "vpn", - "addons", - "releases", - "latest", - ) - ) - - archive_url = ( - "https://ftp.mozilla.org/" if is_relpro else "https://ftp.stage.mozaws.net/" - ) - - upstream_artifacts = [] - for dep in task["dependencies"]: - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "taskType": dep if dep == "build" else "scriptworker", - "paths": [ - release_artifact["name"] - for release_artifact in task["attributes"]["release-artifacts"] - ], - } - ) - - artifact_map = [] for artifact in upstream_artifacts: - artifact_map.append( + worker["artifact-map"].append( { "taskId": artifact["taskId"], "paths": { @@ -147,49 +198,28 @@ def add_beetmover_worker_config(config, tasks): attributes = { **task["attributes"], - "shipping-phase": shipping_phase, + "shipping-phase": phase, } dest = ( - f"{archive_url}{destination_paths[0]}" if destination_paths else archive_url + f"{archive_url}{destination_paths[0]}" + if destination_paths + else archive_url ) if build_type == "addons/opt": task_description = ( f"This {worker_type} task will upload the {task['name']} to {dest}/" ) - elif shipping_phase == "ship-client": - task_description = f"This {worker_type} task will copy build {build_id} from candidates to releases" + elif phase == "ship-client": + task_description = f"This {worker_type} task will copy the v{app_version} build {build_id} candidate to releases" else: task_description = f"This {worker_type} task will upload a {build_os} release candidate for v{app_version} to {dest}/" - if not shipping_phase or shipping_phase.startswith("promote"): - action = "push-to-candidates" - elif shipping_phase == "ship-addons": - action = "direct-push-to-bucket" - elif shipping_phase == "ship-client": - action = "push-to-releases" - else: - raise Exception(f"Invalid shipping_phase `{shipping_phase}`") - extra = { "release_destinations": [ f"{archive_url}{dest}/" for dest in destination_paths ] } - worker = { - "upstream-artifacts": upstream_artifacts, - "bucket": bucket, - "action": action, - "release-properties": { - "app-name": "vpn", - "app-version": app_version, - "branch": config.params["head_ref"], - "build-id": build_id, - "platform": build_type, - }, - "artifact-map": artifact_map, - "build-number": int(build_id), - } task_def = { "name": task["name"], "description": task_description, diff --git a/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py b/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py index 5069f9677c..573fd701e2 100644 --- a/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py +++ b/taskcluster/mozillavpn_taskgraph/transforms/release_notify.py @@ -131,7 +131,7 @@ def format_message(config, tasks): dirs = set() for label, dep_task in config.kind_dependencies_tasks.items(): - if label not in task["dependencies"] or dep_task.kind != "beetmover": + if label not in task["dependencies"] or not dep_task.kind.startswith("beetmover"): continue platform = dep_task.attributes["build-type"].rsplit("/")[0]