Refactor b-cpu-xlargedisk worker pools to allow for experimentation w…

…ith different configurations (mozilla#686) Same as mozilla#674, except for main. At the moment the pipeline doesn't run there because there's no `b-linux-large-gcp-1tb` workers anymore.
bhearsum · Jun 19, 2024 · 9ad9aba · 9ad9aba
1 parent a5dd406
commit 9ad9aba
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 9 deletions.
diff --git a/taskcluster/config.yml b/taskcluster/config.yml
@@ -70,12 +70,22 @@ workers:
             implementation: docker-worker
             os: linux
             worker-type: '{alias}'
-        b-linux-large-gcp-1tb:
+        b-linux-large-gcp-1tb-32-256:
             provisioner: '{trust-domain}-{level}'
             implementation: docker-worker
             os: linux
             worker-type: '{alias}'
-        b-linux-large-gcp-1tb-standard:
+        b-linux-large-gcp-1tb-32-256-standard:
+            provisioner: '{trust-domain}-{level}'
+            implementation: docker-worker
+            os: linux
+            worker-type: '{alias}'
+        b-linux-large-gcp-1tb-64-512:
+            provisioner: '{trust-domain}-{level}'
+            implementation: docker-worker
+            os: linux
+            worker-type: '{alias}'
+        b-linux-large-gcp-1tb-64-512-standard:
             provisioner: '{trust-domain}-{level}'
             implementation: docker-worker
             os: linux
@@ -129,12 +139,20 @@ local-worker-aliases:
         by-worker-class:
             gcp-standard: 'b-linux-large-gcp-300gb'
             default: 'b-linux-large-gcp-300gb'
+    # VM workers with 1 TB of disk space, 32 vCPUs, and 256 GiB of memory
+    # Use for tasks that don't require GPUs, but need immense amounts of disk space
+    # eg: alignments
+    b-cpu-xlargedisk-32-256:
+        by-worker-class:
+            gcp-standard: 'b-linux-large-gcp-1tb-32-256-standard'
+            default: 'b-linux-large-gcp-1tb-32-256'
+    # VM workers with 1 TB of disk space, 64 vCPUs, and 512 GiB of memory
     # Use for tasks that don't require GPUs, but need immense amounts of disk space
     # eg: alignments
-    b-cpu-xlargedisk:
+    b-cpu-xlargedisk-64-512:
         by-worker-class:
-            gcp-standard: 'b-linux-large-gcp-1tb-standard'
-            default: 'b-linux-large-gcp-1tb'
+            gcp-standard: 'b-linux-large-gcp-1tb-64-512-standard'
+            default: 'b-linux-large-gcp-1tb-64-512'
     # Use for quick tasks that need a GPU, eg: evaluate
     b-gpu:
         by-worker-class:

diff --git a/taskcluster/kinds/alignments-backtranslated/kind.yml b/taskcluster/kinds/alignments-backtranslated/kind.yml
@@ -44,7 +44,7 @@ tasks:
                 - dependencies
                 - worker.env
                 - attributes
-        worker-type: b-cpu-xlargedisk
+        worker-type: b-cpu-xlargedisk-64-512
         expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}

diff --git a/taskcluster/kinds/alignments-original/kind.yml b/taskcluster/kinds/alignments-original/kind.yml
@@ -43,7 +43,7 @@ tasks:
                 - dependencies
                 - worker.env
                 - attributes
-        worker-type: b-cpu-xlargedisk
+        worker-type: b-cpu-xlargedisk-64-512
         expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}

diff --git a/taskcluster/kinds/alignments-student/kind.yml b/taskcluster/kinds/alignments-student/kind.yml
@@ -42,7 +42,7 @@ tasks:
                 - dependencies
                 - worker.env
                 - attributes
-        worker-type: b-cpu-xlargedisk
+        worker-type: b-cpu-xlargedisk-64-512
         expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}

diff --git a/taskcluster/kinds/shortlist/kind.yml b/taskcluster/kinds/shortlist/kind.yml
@@ -43,7 +43,7 @@ tasks:
                 - dependencies
                 - worker.env
                 - attributes
-        worker-type: b-cpu-xlargedisk
+        worker-type: b-cpu-xlargedisk-32-256
         expires-after: "90 days"
         worker:
             docker-image: {"in-tree": "train"}