Taskgraph 7.0 changes

mozilla · Dec 20, 2023 · 084cd68 · 084cd68
1 parent 54e05e8
commit 084cd68
Show file tree

Hide file tree

Showing 44 changed files with 38 additions and 38 deletions.
diff --git a/.taskcluster.yml b/.taskcluster.yml
@@ -217,7 +217,7 @@ tasks:
                               features:
                                   taskclusterProxy: true
 
-                              image: mozillareleases/taskgraph:decision-5483484ad45a3d27a0f5bd05f1c87d90e08df67a3713605d812b851a8a5bd854@sha256:ef132cc5741539f846a85bbe0cebc3c9ead30b8f24c1da46c55363f2170c3993
+                              image: mozillareleases/taskgraph:decision-5483484ad45a3d27a0f5bd05f1c87d90e08df67a3713605d812b851a8a5bd854@sha256:2dd667994aa13fccdcdcede85c570a5eb4a5247e42875a9e95a135ef774ee469
                               maxRunTime: 1800
 
                               command:

diff --git a/taskcluster/ci/config.yml → taskcluster/config.yml b/taskcluster/ci/config.yml → taskcluster/config.yml
diff --git a/taskcluster/ci/alignments/kind.yml → taskcluster/kinds/alignments/kind.yml b/taskcluster/ci/alignments/kind.yml → taskcluster/kinds/alignments/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/all-pr/kind.yml → taskcluster/kinds/all-pr/kind.yml b/taskcluster/ci/all-pr/kind.yml → taskcluster/kinds/all-pr/kind.yml
diff --git a/taskcluster/ci/all/kind.yml → taskcluster/kinds/all/kind.yml b/taskcluster/ci/all/kind.yml → taskcluster/kinds/all/kind.yml
diff --git a/taskcluster/ci/bicleaner/kind.yml → taskcluster/kinds/bicleaner/kind.yml b/taskcluster/ci/bicleaner/kind.yml → taskcluster/kinds/bicleaner/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - translations_taskgraph.transforms.from_datasets:per_dataset
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/cefilter/kind.yml → taskcluster/kinds/cefilter/kind.yml b/taskcluster/ci/cefilter/kind.yml → taskcluster/kinds/cefilter/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/clean-corpus/kind.yml → taskcluster/kinds/clean-corpus/kind.yml b/taskcluster/ci/clean-corpus/kind.yml → taskcluster/kinds/clean-corpus/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - translations_taskgraph.transforms.from_datasets:per_dataset
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/clean-mono/kind.yml → taskcluster/kinds/clean-mono/kind.yml b/taskcluster/ci/clean-mono/kind.yml → taskcluster/kinds/clean-mono/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - translations_taskgraph.transforms.from_datasets:mono
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/collect-corpus/kind.yml → taskcluster/kinds/collect-corpus/kind.yml b/taskcluster/ci/collect-corpus/kind.yml → taskcluster/kinds/collect-corpus/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - taskgraph.transforms.task_context
     - taskgraph.transforms.from_deps
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/collect-mono-src/kind.yml → taskcluster/kinds/collect-mono-src/kind.yml b/taskcluster/ci/collect-mono-src/kind.yml → taskcluster/kinds/collect-mono-src/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - taskgraph.transforms.from_deps
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/collect-mono-trg/kind.yml → taskcluster/kinds/collect-mono-trg/kind.yml b/taskcluster/ci/collect-mono-trg/kind.yml → taskcluster/kinds/collect-mono-trg/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - taskgraph.transforms.from_deps
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/dataset/kind.yml → taskcluster/kinds/dataset/kind.yml b/taskcluster/ci/dataset/kind.yml → taskcluster/kinds/dataset/kind.yml
@@ -11,7 +11,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - translations_taskgraph.transforms.from_datasets:per_dataset
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/docker-image/kind.yml → taskcluster/kinds/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml → taskcluster/kinds/docker-image/kind.yml
diff --git a/taskcluster/ci/evaluate-quantized/kind.yml → ...cluster/kinds/evaluate-quantized/kind.yml b/taskcluster/ci/evaluate-quantized/kind.yml → ...cluster/kinds/evaluate-quantized/kind.yml
@@ -9,7 +9,7 @@ transforms:
     - translations_taskgraph.transforms.from_datasets:per_dataset
     - translations_taskgraph.transforms.worker_env
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/...ter/ci/evaluate-teacher-ensemble/kind.yml → .../kinds/evaluate-teacher-ensemble/kind.yml b/...ter/ci/evaluate-teacher-ensemble/kind.yml → .../kinds/evaluate-teacher-ensemble/kind.yml
@@ -10,7 +10,7 @@ transforms:
     - translations_taskgraph.transforms.worker_env
     - taskgraph.transforms.from_deps
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/evaluate/kind.yml → taskcluster/kinds/evaluate/kind.yml b/taskcluster/ci/evaluate/kind.yml → taskcluster/kinds/evaluate/kind.yml
@@ -11,7 +11,7 @@ transforms:
     - taskgraph.transforms.task_context
     - translations_taskgraph.transforms.cast_to
     - taskgraph.transforms.chunking
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/export/kind.yml → taskcluster/kinds/export/kind.yml b/taskcluster/ci/export/kind.yml → taskcluster/kinds/export/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/extract-best/kind.yml → taskcluster/kinds/extract-best/kind.yml b/taskcluster/ci/extract-best/kind.yml → taskcluster/kinds/extract-best/kind.yml
@@ -9,7 +9,7 @@ transforms:
     - taskgraph.transforms.task_context
     - translations_taskgraph.transforms.cast_to
     - taskgraph.transforms.chunking
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/fetch/bicleaner.yml → taskcluster/kinds/fetch/bicleaner.yml b/taskcluster/ci/fetch/bicleaner.yml → taskcluster/kinds/fetch/bicleaner.yml
diff --git a/taskcluster/ci/fetch/kind.yml → taskcluster/kinds/fetch/kind.yml b/taskcluster/ci/fetch/kind.yml → taskcluster/kinds/fetch/kind.yml
@@ -6,7 +6,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - translations_taskgraph.transforms.fetch:transforms
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - taskgraph.transforms.task:transforms
 
 task-defaults:

diff --git a/taskcluster/ci/fetch/python.yml → taskcluster/kinds/fetch/python.yml b/taskcluster/ci/fetch/python.yml → taskcluster/kinds/fetch/python.yml
diff --git a/taskcluster/ci/fetch/toolchains.yml → taskcluster/kinds/fetch/toolchains.yml b/taskcluster/ci/fetch/toolchains.yml → taskcluster/kinds/fetch/toolchains.yml
diff --git a/taskcluster/ci/finetune-student/kind.yml → taskcluster/kinds/finetune-student/kind.yml b/taskcluster/ci/finetune-student/kind.yml → taskcluster/kinds/finetune-student/kind.yml
@@ -9,7 +9,7 @@ transforms:
     - translations_taskgraph.transforms.marian_args:transforms
     - translations_taskgraph.transforms.worker_env
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/merge-corpus/kind.yml → taskcluster/kinds/merge-corpus/kind.yml b/taskcluster/ci/merge-corpus/kind.yml → taskcluster/kinds/merge-corpus/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - translations_taskgraph.transforms.find_upstreams:by_locales
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/merge-devset/kind.yml → taskcluster/kinds/merge-devset/kind.yml b/taskcluster/ci/merge-devset/kind.yml → taskcluster/kinds/merge-devset/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - translations_taskgraph.transforms.find_upstreams:by_locales
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/merge-mono/kind.yml → taskcluster/kinds/merge-mono/kind.yml b/taskcluster/ci/merge-mono/kind.yml → taskcluster/kinds/merge-mono/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - taskgraph.transforms.task_context
     - translations_taskgraph.transforms.find_upstreams:mono
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/merge-translated/kind.yml → taskcluster/kinds/merge-translated/kind.yml b/taskcluster/ci/merge-translated/kind.yml → taskcluster/kinds/merge-translated/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/quantize/kind.yml → taskcluster/kinds/quantize/kind.yml b/taskcluster/ci/quantize/kind.yml → taskcluster/kinds/quantize/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/score/kind.yml → taskcluster/kinds/score/kind.yml b/taskcluster/ci/score/kind.yml → taskcluster/kinds/score/kind.yml
@@ -8,7 +8,7 @@ loader: taskgraph.loader.transform:loader
 transforms:
     - translations_taskgraph.transforms.worker_env
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/split-corpus/kind.yml → taskcluster/kinds/split-corpus/kind.yml b/taskcluster/ci/split-corpus/kind.yml → taskcluster/kinds/split-corpus/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/split-mono-src/kind.yml → taskcluster/kinds/split-mono-src/kind.yml b/taskcluster/ci/split-mono-src/kind.yml → taskcluster/kinds/split-mono-src/kind.yml
@@ -6,7 +6,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/split-mono-trg/kind.yml → taskcluster/kinds/split-mono-trg/kind.yml b/taskcluster/ci/split-mono-trg/kind.yml → taskcluster/kinds/split-mono-trg/kind.yml
@@ -6,7 +6,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/tests/kind.yml → taskcluster/kinds/tests/kind.yml b/taskcluster/ci/tests/kind.yml → taskcluster/kinds/tests/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - taskgraph.transforms.task:transforms
 
 task-defaults:

diff --git a/taskcluster/ci/toolchain/kind.yml → taskcluster/kinds/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml → taskcluster/kinds/toolchain/kind.yml
@@ -8,7 +8,7 @@ kind-dependencies:
     - fetch
 
 transforms:
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/train-backwards/kind.yml → taskcluster/kinds/train-backwards/kind.yml b/taskcluster/ci/train-backwards/kind.yml → taskcluster/kinds/train-backwards/kind.yml
@@ -10,7 +10,7 @@ transforms:
     - translations_taskgraph.transforms.marian_args:transforms
     - translations_taskgraph.transforms.worker_env
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/train-student/kind.yml → taskcluster/kinds/train-student/kind.yml b/taskcluster/ci/train-student/kind.yml → taskcluster/kinds/train-student/kind.yml
@@ -9,7 +9,7 @@ transforms:
     - translations_taskgraph.transforms.marian_args:transforms
     - translations_taskgraph.transforms.worker_env
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/train-teacher/kind.yml → taskcluster/kinds/train-teacher/kind.yml b/taskcluster/ci/train-teacher/kind.yml → taskcluster/kinds/train-teacher/kind.yml
@@ -12,7 +12,7 @@ transforms:
     - taskgraph.transforms.task_context
     - translations_taskgraph.transforms.cast_to
     - taskgraph.transforms.chunking
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/train-vocab/kind.yml → taskcluster/kinds/train-vocab/kind.yml b/taskcluster/ci/train-vocab/kind.yml → taskcluster/kinds/train-vocab/kind.yml
@@ -7,7 +7,7 @@ loader: taskgraph.loader.transform:loader
 
 transforms:
     - taskgraph.transforms.task_context
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/translate-corpus/kind.yml → taskcluster/kinds/translate-corpus/kind.yml b/taskcluster/ci/translate-corpus/kind.yml → taskcluster/kinds/translate-corpus/kind.yml
@@ -11,7 +11,7 @@ transforms:
     - translations_taskgraph.transforms.cast_to
     - taskgraph.transforms.chunking
     - taskgraph.transforms.from_deps
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/translate-mono-src/kind.yml → ...cluster/kinds/translate-mono-src/kind.yml b/taskcluster/ci/translate-mono-src/kind.yml → ...cluster/kinds/translate-mono-src/kind.yml
@@ -10,7 +10,7 @@ transforms:
     - translations_taskgraph.transforms.cast_to
     - taskgraph.transforms.chunking
     - taskgraph.transforms.from_deps
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/ci/translate-mono-trg/kind.yml → ...cluster/kinds/translate-mono-trg/kind.yml b/taskcluster/ci/translate-mono-trg/kind.yml → ...cluster/kinds/translate-mono-trg/kind.yml
@@ -10,7 +10,7 @@ transforms:
     - taskgraph.transforms.task_context
     - translations_taskgraph.transforms.cast_to
     - taskgraph.transforms.chunking
-    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.run:transforms
     - translations_taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 

diff --git a/taskcluster/translations_taskgraph/target_tasks.py b/taskcluster/translations_taskgraph/target_tasks.py
@@ -1,7 +1,7 @@
-from taskgraph.target_tasks import _target_task
+from taskgraph.target_tasks import register_target_task
 
 
-@_target_task("train-target-tasks")
+@register_target_task("train-target-tasks")
 def train_target_tasks(full_task_graph, parameters, graph_config):
     training_config = parameters["training_config"]
     stage = training_config["target-stage"]

diff --git a/taskcluster/translations_taskgraph/transforms/skip_unless_pipeline_changed.py b/taskcluster/translations_taskgraph/transforms/skip_unless_pipeline_changed.py
@@ -16,7 +16,7 @@
 from taskgraph import files_changed
 from taskgraph.transforms.base import TransformSequence
 
-KIND_DIR = Path(__file__).parent.parent.parent / "ci"
+KIND_DIR = Path(__file__).parent.parent.parent / "kinds"
 
 # Kinds are slightly special - there are some kinds that don't affect the pipeline,
 # and changing them shouldn't force the pipeline to run.