diff --git a/buildbot_nix/buildbot_nix/__init__.py b/buildbot_nix/buildbot_nix/__init__.py index 31fa2f344..8d363543e 100644 --- a/buildbot_nix/buildbot_nix/__init__.py +++ b/buildbot_nix/buildbot_nix/__init__.py @@ -8,7 +8,7 @@ import re import urllib.parse from collections import defaultdict -from collections.abc import Callable, Coroutine, Generator +from collections.abc import Coroutine, Generator from dataclasses import dataclass from datetime import UTC, datetime from multiprocessing import cpu_count @@ -816,18 +816,51 @@ async def run(self) -> int: class CachedFailureStep(steps.BuildStep): """Shows a dependency failure.""" + project: GitProject + worker_names: list[str] + post_build_steps: list[models.PostBuildStep] + branch_config_dict: models.BranchConfigDict + outputs_path: Path | None + + def __init__( + self, + project: GitProject, + worker_names: list[str], + post_build_steps: list[models.PostBuildStep], + branch_config_dict: models.BranchConfigDict, + outputs_path: Path | None, + **kwargs: Any, + ) -> None: + self.project = project + self.worker_names = worker_names + self.post_build_steps = post_build_steps + self.branch_config_dict = branch_config_dict + + super().__init__(**kwargs) + async def run(self) -> int: - attr = self.getProperty("attr") - # show eval error - error_log: StreamLog = await self.addLog("nix_error") - msg = [ - f"{attr} was failed because it has failed previously and its failure has been cached.", - ] - url = self.getProperty("first_failure_url") - if url: - msg.append(f" failed build: {url}") - error_log.addStderr("\n".join(msg) + "\n") - return util.FAILURE + if self.build.reason != "rebuild": + attr = self.getProperty("attr") + # show eval error + error_log: StreamLog = await self.addLog("nix_error") + msg = [ + f"{attr} was failed because it has failed previously and its failure has been cached.", + ] + url = self.getProperty("first_failure_url") + if url: + msg.append(f" failed build: {url}") + error_log.addStderr("\n".join(msg) + "\n") + return util.FAILURE + self.build.addStepsAfterCurrentStep( + nix_build_steps( + self.project, + self.worker_names, + self.post_build_steps, + self.branch_config_dict, + self.outputs_path, + ) + ) + return util.SUCCESS class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): @@ -864,11 +897,20 @@ class UpdateBuildOutput(steps.BuildStep): """ project: GitProject + path: Path + branch_config: models.BranchConfigDict - def __init__(self, project: GitProject, path: Path, **kwargs: Any) -> None: + def __init__( + self, + project: GitProject, + path: Path, + branch_config: models.BranchConfigDict, + **kwargs: Any, + ) -> None: super().__init__(**kwargs) self.project = project self.path = path + self.branch_config = branch_config def join_traversalsafe(self, root: Path, joined: Path) -> Path: root = root.resolve() @@ -893,7 +935,12 @@ def join_all_traversalsafe(self, root: Path, *paths: str) -> Path: async def run(self) -> int: props = self.build.getProperties() - if props.getProperty("branch") != self.project.default_branch: + if ( + not self.branch_config.do_update_outputs( + self.project.default_branch, props.getProperty("branch") + ) + or props.getProperty("event") != "push" + ): return util.SKIPPED out_path = props.getProperty("out_path") @@ -1072,17 +1119,23 @@ def nix_eval_config( ) -async def do_register_gcroot_if(s: steps.BuildStep) -> bool: +async def do_register_gcroot_if( + s: steps.BuildStep, branch_config: models.BranchConfigDict +) -> bool: gc_root = await util.Interpolate( "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s" ).getRenderingFor(s.getProperties()) - out_path = await util.Property("out_path").getRenderingFor(s.getProperties()) - default_branch = await util.Property("default_branch").getRenderingFor( - s.getProperties() - ) + out_path = s.getProperty("out_path") - return s.getProperty("branch") == str(default_branch) and not ( - Path(str(gc_root)).exists() and Path(str(gc_root)).readlink() == str(out_path) + return ( + branch_config.do_register_gcroot( + s.getProperty("default_branch"), s.getProperty("branch") + ) + and s.getProperty("event") == "push" + and not ( + Path(str(gc_root)).exists() + and Path(str(gc_root)).readlink() == Path(out_path) + ) ) @@ -1090,8 +1143,8 @@ def nix_build_steps( project: GitProject, worker_names: list[str], post_build_steps: list[steps.BuildStep], + branch_config: models.BranchConfigDict, outputs_path: Path | None = None, - global_do_step_if: Callable[[steps.BuildStep], bool] | bool = True, ) -> list[steps.BuildStep]: out_steps = [ NixBuildCommand( @@ -1116,7 +1169,6 @@ def nix_build_steps( # We increase this over the default since the build output might end up in a different `nix build`. timeout=60 * 60 * 3, haltOnFailure=True, - doStepIf=global_do_step_if, ), *post_build_steps, Trigger( @@ -1128,19 +1180,13 @@ def nix_build_steps( sourceStamps=[], alwaysUseLatest=False, updateSourceStamp=False, - doStepIf=lambda buildstep: ( - global_do_step_if - if isinstance(global_do_step_if, bool) - else global_do_step_if(buildstep) - ) - and do_register_gcroot_if(buildstep), + doStepIf=lambda s: do_register_gcroot_if(s, branch_config), copy_properties=["out_path", "attr"], set_properties={"report_status": False}, ), steps.ShellCommand( name="Delete temporary gcroots", command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")], - doStepIf=global_do_step_if, ), ] @@ -1150,7 +1196,7 @@ def nix_build_steps( project=project, name="Update build output", path=outputs_path, - doStepIf=global_do_step_if, + branch_config=branch_config, ), ) @@ -1161,12 +1207,15 @@ def nix_build_config( project: GitProject, worker_names: list[str], post_build_steps: list[steps.BuildStep], + branch_config_dict: models.BranchConfigDict, outputs_path: Path | None = None, ) -> BuilderConfig: """Builds one nix flake attribute.""" factory = util.BuildFactory() factory.addSteps( - nix_build_steps(project, worker_names, post_build_steps, outputs_path) + nix_build_steps( + project, worker_names, post_build_steps, branch_config_dict, outputs_path + ) ) return util.BuilderConfig( @@ -1229,6 +1278,7 @@ def nix_cached_failure_config( project: GitProject, worker_names: list[str], skipped_worker_names: list[str], + branch_config_dict: models.BranchConfigDict, post_build_steps: list[steps.BuildStep], outputs_path: Path | None = None, ) -> BuilderConfig: @@ -1236,23 +1286,17 @@ def nix_cached_failure_config( factory = util.BuildFactory() factory.addStep( CachedFailureStep( + project=project, + worker_names=worker_names, + post_build_steps=post_build_steps, + branch_config_dict=branch_config_dict, + outputs_path=outputs_path, name="Cached failure", - doStepIf=lambda buildstep: buildstep.build.reason != "rebuild", haltOnFailure=True, flunkOnFailure=True, ), ) - factory.addSteps( - nix_build_steps( - project, - worker_names, - post_build_steps, - outputs_path, - global_do_step_if=lambda buildstep: buildstep.build.reason == "rebuild", - ) - ) - return util.BuilderConfig( name=f"{project.name}/nix-cached-failure", project=project.name, @@ -1266,6 +1310,7 @@ def nix_cached_failure_config( def nix_skipped_build_config( project: GitProject, worker_names: list[str], + branch_config_dict: models.BranchConfigDict, outputs_path: Path | None = None, ) -> BuilderConfig: """Dummy builder that is triggered when a build is skipped.""" @@ -1290,7 +1335,7 @@ def nix_skipped_build_config( sourceStamps=[], alwaysUseLatest=False, updateSourceStamp=False, - doStepIf=do_register_gcroot_if, + doStepIf=lambda s: do_register_gcroot_if(s, branch_config_dict), copy_properties=["out_path", "attr"], set_properties={"report_status": False}, ), @@ -1301,6 +1346,7 @@ def nix_skipped_build_config( project=project, name="Update build output", path=outputs_path, + branch_config=branch_config_dict, ), ) return util.BuilderConfig( @@ -1410,24 +1456,32 @@ def config_for_project( job_report_limit: int | None, failed_builds_db: FailedBuildDB, per_repo_effects_secrets: dict[str, str], + branch_config_dict: models.BranchConfigDict, outputs_path: Path | None = None, ) -> None: config["projects"].append(Project(project.name)) config["schedulers"].extend( [ schedulers.SingleBranchScheduler( - name=f"{project.project_id}-default-branch", + name=f"{project.project_id}-primary", change_filter=util.ChangeFilter( repository=project.url, - filter_fn=lambda c: c.branch == project.default_branch, + filter_fn=lambda c: branch_config_dict.do_run( + project.default_branch, c.branch + ), ), builderNames=[f"{project.name}/nix-eval"], treeStableTimer=5, - ), + ) + ] + ) + config["schedulers"].extend( + [ # this is compatible with bors or github's merge queue schedulers.SingleBranchScheduler( name=f"{project.project_id}-merge-queue", change_filter=util.ChangeFilter( + # TODO add filter repository=project.url, branch_re="(gh-readonly-queue/.*|staging|trying)", ), @@ -1442,6 +1496,10 @@ def config_for_project( ), builderNames=[f"{project.name}/nix-eval"], ), + ] + ) + config["schedulers"].extend( + [ # this is triggered from `nix-eval` schedulers.Triggerable( name=f"{project.project_id}-nix-build", @@ -1511,6 +1569,7 @@ def config_for_project( project, worker_names, outputs_path=outputs_path, + branch_config_dict=branch_config_dict, post_build_steps=post_build_steps, ), buildbot_effects_config( @@ -1519,15 +1578,21 @@ def config_for_project( worker_names=worker_names, secrets=effects_secrets_cred, ), - nix_skipped_build_config(project, SKIPPED_BUILDER_NAMES, outputs_path), + nix_skipped_build_config( + project=project, + worker_names=SKIPPED_BUILDER_NAMES, + branch_config_dict=branch_config_dict, + outputs_path=outputs_path, + ), nix_failed_eval_config(project, SKIPPED_BUILDER_NAMES), nix_dependency_failed_config(project, SKIPPED_BUILDER_NAMES), nix_cached_failure_config( - project, - worker_names, - SKIPPED_BUILDER_NAMES, - post_build_steps, - outputs_path, + project=project, + worker_names=worker_names, + skipped_worker_names=SKIPPED_BUILDER_NAMES, + branch_config_dict=branch_config_dict, + post_build_steps=post_build_steps, + outputs_path=outputs_path, ), nix_register_gcroot_config(project, worker_names), ], @@ -1744,6 +1809,7 @@ def configure(self, config: dict[str, Any]) -> None: job_report_limit=self.config.job_report_limit, per_repo_effects_secrets=self.config.effects_per_repo_secrets, failed_builds_db=DB, + branch_config_dict=self.config.branches, outputs_path=self.config.outputs_path, ) diff --git a/buildbot_nix/buildbot_nix/models.py b/buildbot_nix/buildbot_nix/models.py index 038815381..9a347307c 100644 --- a/buildbot_nix/buildbot_nix/models.py +++ b/buildbot_nix/buildbot_nix/models.py @@ -1,10 +1,12 @@ -from collections.abc import Mapping +import re +from collections.abc import Callable, Mapping from enum import Enum from pathlib import Path from typing import Any from buildbot.plugins import steps, util -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, GetCoreSchemaHandler, TypeAdapter +from pydantic_core import CoreSchema, core_schema from .secrets import read_secret_file @@ -172,6 +174,71 @@ def maybe_interpolate(value: str | Interpolate) -> str | util.Interpolate: ) +def glob_to_regex(glob: str) -> re.Pattern: + return re.compile(glob.replace("*", ".*").replace("?", ".")) + + +class BranchConfig(BaseModel): + match_glob: str = Field(validation_alias="matchGlob") + register_gcroots: bool = Field(validation_alias="registerGCRoots") + update_outputs: bool = Field(validation_alias="updateOutputs") + + match_regex: re.Pattern = Field(default=None, exclude=True) + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + + match_glob = kwargs.get("match_glob") or kwargs["matchGlob"] + self.match_regex = glob_to_regex(match_glob) + + def __or__(self, other: "BranchConfig") -> "BranchConfig": + assert self.match_glob == other.match_glob + assert self.match_regex == other.match_regex + + return BranchConfig( + match_glob=self.match_glob, + register_gcroots=self.register_gcroots or other.register_gcroots, + update_outputs=self.update_outputs or other.update_outputs, + ) + + +class BranchConfigDict(dict[str, BranchConfig]): + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_after_validator_function( + cls, handler(dict[str, BranchConfig]) + ) + + def lookup_branch_config(self, branch: str) -> BranchConfig | None: + ret = None + for branch_config in self.values(): + if branch_config.match_regex.fullmatch(branch): + if ret is None: + ret = branch_config + else: + ret |= branch_config + return ret + + def check_lookup( + self, default_branch: str, branch: str, fn: Callable[[BranchConfig], bool] + ) -> bool: + branch_config = self.lookup_branch_config(branch) + return branch == default_branch or ( + branch_config is not None and fn(branch_config) + ) + + def do_run(self, default_branch: str, branch: str) -> bool: + return self.check_lookup(default_branch, branch, lambda _: True) + + def do_register_gcroot(self, default_branch: str, branch: str) -> bool: + return self.check_lookup(default_branch, branch, lambda bc: bc.register_gcroots) + + def do_update_outputs(self, default_branch: str, branch: str) -> bool: + return self.check_lookup(default_branch, branch, lambda bc: bc.update_outputs) + + class BuildbotNixConfig(BaseModel): db_url: str auth_backend: AuthBackendConfig @@ -193,6 +260,7 @@ class BuildbotNixConfig(BaseModel): job_report_limit: int | None http_basic_auth_password_file: Path | None effects_per_repo_secrets: dict[str, str] + branches: BranchConfigDict @property def nix_workers_secret(self) -> str: diff --git a/examples/master.nix b/examples/master.nix index 2286dd436..5f00660dc 100644 --- a/examples/master.nix +++ b/examples/master.nix @@ -68,6 +68,11 @@ # auth.signingKey.file = "/var/lib/secrets/cachix-key"; # auth.authToken.file = "/var/lib/secrets/cachix-token"; #}; + + # By default we only build the "default" branch. Using the branches option, buildbot will also build other branches. + # branches = { + # releaseBranches.matchGlob = "release-*"; + # }; }; # Optional: Enable acme/TLS in nginx (recommended) diff --git a/nix/master.nix b/nix/master.nix index a5bf9ca2c..6f0640448 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -520,6 +520,77 @@ in default = { }; example = ''{ "github:nix-community/buildbot-nix" = config.agenix.secrets.buildbot-nix-effects-secrets.path; }''; }; + + branches = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + matchGlob = lib.mkOption { + type = lib.types.str; + description = '' + A glob specifying which branches to apply this rule to. + ''; + }; + + registerGCRoots = lib.mkOption { + type = lib.types.bool; + description = '' + Whether to register gcroots for branches matching this glob. + ''; + default = true; + }; + + updateOutputs = lib.mkOption { + type = lib.types.bool; + description = '' + Whether to update outputs for branches matching this glob. + ''; + default = false; + }; + }; + } + ); + default = { }; + description = '' + An attrset of branch rules, each rule specifies which branches it should apply to using the + `matchGlob` option and then the corresponding settings are applied to the matched branches. + If multiple rules match a given branch, the rules are `or`-ed together, by `or`-ing each + individual boolean option of all matching rules. Take the following as example: + ``` + { + rule1 = { + matchGlob = "f*"; + registerGCRroots = false; + updateOutputs = false; + } + rule2 = { + matchGlob = "foo"; + registerGCRroots = true; + updateOutputs = false; + } + } + ``` + This example will result in `registerGCRoots` both being considered `true`, + but `updateOutputs` being `false` for the branch `foo`. + + The default branches of all repos are considered to be matching of a rule setting all the options + to `true`. + ''; + example = lib.literalExpression '' + { + rule1 = { + matchGlob = "f*"; + registerGCRroots = false; + updateOutputs = false; + } + rule2 = { + matchGlob = "foo"; + registerGCRroots = true; + updateOutputs = false; + } + } + ''; + }; }; }; config = lib.mkMerge [ @@ -691,6 +762,7 @@ in inherit name; value = "effects-secret__${cleanUpRepoName name}"; }) cfg.effects.perRepoSecretFiles; + branches = cfg.branches; } }").read_text())) )