validate worker secret file

buildbot_nix/buildbot_nix/__init__.py

@@ -1761,18 +1761,15 @@ def configure(self, config: dict[str, Any]) -> None:
         for backend in backends.values():
             projects += backend.load_projects()
 
-        worker_config = json.loads(self.config.nix_workers_secret)
-        worker_names = []
-
         config.setdefault("projects", [])
         config.setdefault("secretsProviders", [])
         config.setdefault("www", {})
 
-        for item in worker_config:
-            cores = item.get("cores", 0)
-            for i in range(cores):
-                worker_name = f"{item['name']}-{i:03}"
-                config["workers"].append(worker.Worker(worker_name, item["pass"]))
+        worker_names = []
+        for w in self.config.nix_worker_secrets().workers:
+            for i in range(w.cores):
+                worker_name = f"{w.name}-{i:03}"
+                config["workers"].append(worker.Worker(worker_name, w.password))
                 worker_names.append(worker_name)
 
         if worker_names == []:
@@ -1801,25 +1798,31 @@ def configure(self, config: dict[str, Any]) -> None:
             # Hacky but we have no other hooks just now to run code on shutdown.
             atexit.register(lambda: DB.close() if DB is not None else None)
 
+        succeeded_projects = []
         for project in projects:
-            config_for_project(
-                config=config,
-                project=project,
-                worker_names=worker_names,
-                nix_supported_systems=self.config.build_systems,
-                nix_eval_worker_count=self.config.eval_worker_count
-                or multiprocessing.cpu_count(),
-                nix_eval_max_memory_size=self.config.eval_max_memory_size,
-                eval_lock=eval_lock,
-                post_build_steps=[
-                    x.to_buildstep() for x in self.config.post_build_steps
-                ],
-                job_report_limit=self.config.job_report_limit,
-                per_repo_effects_secrets=self.config.effects_per_repo_secrets,
-                failed_builds_db=DB,
-                branch_config_dict=self.config.branches,
-                outputs_path=self.config.outputs_path,
-            )
+            try:
+                config_for_project(
+                    config=config,
+                    project=project,
+                    worker_names=worker_names,
+                    nix_supported_systems=self.config.build_systems,
+                    nix_eval_worker_count=self.config.eval_worker_count
+                    or multiprocessing.cpu_count(),
+                    nix_eval_max_memory_size=self.config.eval_max_memory_size,
+                    eval_lock=eval_lock,
+                    post_build_steps=[
+                        x.to_buildstep() for x in self.config.post_build_steps
+                    ],
+                    job_report_limit=self.config.job_report_limit,
+                    per_repo_effects_secrets=self.config.effects_per_repo_secrets,
+                    failed_builds_db=DB,
+                    branch_config_dict=self.config.branches,
+                    outputs_path=self.config.outputs_path,
+                )
+            except Exception:  # noqa: BLE001
+                log.failure(f"Failed to configure project {project.name}")
+            else:
+                succeeded_projects.append(project)
 
         config["workers"].extend(worker.LocalWorker(w) for w in SKIPPED_BUILDER_NAMES)
 
@@ -1862,7 +1865,7 @@ def configure(self, config: dict[str, Any]) -> None:
             )
 
         config.setdefault("change_source", [])
-        for project in projects:
+        for project in succeeded_projects:
             change_source = project.create_change_source()
             if change_source is not None:
                 config["change_source"].append(change_source)
@@ -1882,5 +1885,5 @@ def configure(self, config: dict[str, Any]) -> None:
             config["www"]["authz"] = setup_authz(
                 admins=self.config.admins,
                 backends=list(backends.values()),
-                projects=projects,
+                projects=succeeded_projects,
             )

buildbot_nix/buildbot_nix/models.py

@@ -1,3 +1,4 @@
+import json
 import re
 from collections.abc import Callable, Mapping
 from enum import Enum
@@ -8,6 +9,7 @@
 from pydantic import BaseModel, ConfigDict, Field, GetCoreSchemaHandler, TypeAdapter
 from pydantic_core import CoreSchema, core_schema
 
+from .errors import BuildbotNixError
 from .secrets import read_secret_file
 
 
@@ -270,6 +272,16 @@ def do_update_outputs(self, default_branch: str, branch: str) -> bool:
         return self.check_lookup(default_branch, branch, lambda bc: bc.update_outputs)
 
 
+class Worker(BaseModel):
+    name: str
+    cores: int
+    password: str = Field(alias="pass")
+
+
+class WorkerConfig(BaseModel):
+    workers: list[Worker]
+
+
 class BuildbotNixConfig(BaseModel):
     db_url: str
     auth_backend: AuthBackendConfig
@@ -294,9 +306,13 @@ class BuildbotNixConfig(BaseModel):
     effects_per_repo_secrets: dict[str, str]
     branches: BranchConfigDict
 
-    @property
-    def nix_workers_secret(self) -> str:
-        return read_secret_file(self.nix_workers_secret_file)
+    def nix_worker_secrets(self) -> WorkerConfig:
+        try:
+            data = json.loads(read_secret_file(self.nix_workers_secret_file))
+        except json.JSONDecodeError as e:
+            msg = f"Failed to decode JSON from {self.nix_workers_secret_file}"
+            raise BuildbotNixError(msg) from e
+        return WorkerConfig(workers=data)
 
     @property
     def http_basic_auth_password(self) -> str:

nix/checks/effects.nix

@@ -3,7 +3,7 @@
   nodes = {
     # `self` here is set by using specialArgs in `lib.nix`
     node1 =
-      { self, pkgs, ... }:
+      { pkgs, ... }:
       {
         environment.systemPackages = [
           (pkgs.python3.pkgs.callPackage ../../nix/buildbot-effects.nix { })

nix/checks/worker.nix

@@ -5,7 +5,6 @@
     node1 =
       {
         self,
-        config,
         pkgs,
         ...
       }:

nix/master.nix

@@ -2,7 +2,6 @@
   config,
   pkgs,
   lib,
-  options,
   ...
 }:
 let
@@ -20,13 +19,6 @@ let
     check = x: x ? "_type" && x._type == "interpolate" && x ? "value";
   };
 
-  interpolateToString =
-    value:
-    if lib.isAttrs value && value ? "_type" && value._type == "interpolate" then
-      "util.Interpolate(${builtins.toJSON value.value})"
-    else
-      builtins.toJSON value;
-
   cleanUpRepoName =
     name:
     builtins.replaceStrings

nix/nix-eval-jobs.nix

@@ -6,7 +6,7 @@
   curl,
   nlohmann_json,
 }:
-nix-eval-jobs.overrideAttrs (oldAttrs: {
+nix-eval-jobs.overrideAttrs (_oldAttrs: {
   src = fetchFromGitHub {
     owner = "nix-community";
     repo = "nix-eval-jobs";