Skip to content

Commit

Permalink
Merge pull request #312 from MagicRB/configurable-branches
Browse files Browse the repository at this point in the history 
Configurable branches
  • Loading branch information
@Mic92
Mic92 authored Dec 6, 2024
2 parents 001114c + 6a03980 commit b57de59
Show file tree
Hide file tree
Showing 4 changed files with 266 additions and 55 deletions.
172 changes: 119 additions & 53 deletions buildbot_nix/buildbot_nix/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import re
import urllib.parse
from collections import defaultdict
from collections.abc import Callable, Coroutine, Generator
from collections.abc import Coroutine, Generator
from dataclasses import dataclass
from datetime import UTC, datetime
from multiprocessing import cpu_count
Expand Down Expand Up @@ -816,18 +816,51 @@ async def run(self) -> int:
class CachedFailureStep(steps.BuildStep):
"""Shows a dependency failure."""

project: GitProject
worker_names: list[str]
post_build_steps: list[models.PostBuildStep]
branch_config_dict: models.BranchConfigDict
outputs_path: Path | None

def __init__(
self,
project: GitProject,
worker_names: list[str],
post_build_steps: list[models.PostBuildStep],
branch_config_dict: models.BranchConfigDict,
outputs_path: Path | None,
**kwargs: Any,
) -> None:
self.project = project
self.worker_names = worker_names
self.post_build_steps = post_build_steps
self.branch_config_dict = branch_config_dict

super().__init__(**kwargs)

async def run(self) -> int:
attr = self.getProperty("attr")
# show eval error
error_log: StreamLog = await self.addLog("nix_error")
msg = [
f"{attr} was failed because it has failed previously and its failure has been cached.",
]
url = self.getProperty("first_failure_url")
if url:
msg.append(f" failed build: {url}")
error_log.addStderr("\n".join(msg) + "\n")
return util.FAILURE
if self.build.reason != "rebuild":
attr = self.getProperty("attr")
# show eval error
error_log: StreamLog = await self.addLog("nix_error")
msg = [
f"{attr} was failed because it has failed previously and its failure has been cached.",
]
url = self.getProperty("first_failure_url")
if url:
msg.append(f" failed build: {url}")
error_log.addStderr("\n".join(msg) + "\n")
return util.FAILURE
self.build.addStepsAfterCurrentStep(
nix_build_steps(
self.project,
self.worker_names,
self.post_build_steps,
self.branch_config_dict,
self.outputs_path,
)
)
return util.SUCCESS


class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
Expand Down Expand Up @@ -864,11 +897,20 @@ class UpdateBuildOutput(steps.BuildStep):
"""

project: GitProject
path: Path
branch_config: models.BranchConfigDict

def __init__(self, project: GitProject, path: Path, **kwargs: Any) -> None:
def __init__(
self,
project: GitProject,
path: Path,
branch_config: models.BranchConfigDict,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self.project = project
self.path = path
self.branch_config = branch_config

def join_traversalsafe(self, root: Path, joined: Path) -> Path:
root = root.resolve()
Expand All @@ -893,7 +935,12 @@ def join_all_traversalsafe(self, root: Path, *paths: str) -> Path:
async def run(self) -> int:
props = self.build.getProperties()

if props.getProperty("branch") != self.project.default_branch:
if (
not self.branch_config.do_update_outputs(
self.project.default_branch, props.getProperty("branch")
)
or props.getProperty("event") != "push"
):
return util.SKIPPED

out_path = props.getProperty("out_path")
Expand Down Expand Up @@ -1072,26 +1119,32 @@ def nix_eval_config(
)


async def do_register_gcroot_if(s: steps.BuildStep) -> bool:
async def do_register_gcroot_if(
s: steps.BuildStep, branch_config: models.BranchConfigDict
) -> bool:
gc_root = await util.Interpolate(
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s"
).getRenderingFor(s.getProperties())
out_path = await util.Property("out_path").getRenderingFor(s.getProperties())
default_branch = await util.Property("default_branch").getRenderingFor(
s.getProperties()
)
out_path = s.getProperty("out_path")

return s.getProperty("branch") == str(default_branch) and not (
Path(str(gc_root)).exists() and Path(str(gc_root)).readlink() == str(out_path)
return (
branch_config.do_register_gcroot(
s.getProperty("default_branch"), s.getProperty("branch")
)
and s.getProperty("event") == "push"
and not (
Path(str(gc_root)).exists()
and Path(str(gc_root)).readlink() == Path(out_path)
)
)


def nix_build_steps(
project: GitProject,
worker_names: list[str],
post_build_steps: list[steps.BuildStep],
branch_config: models.BranchConfigDict,
outputs_path: Path | None = None,
global_do_step_if: Callable[[steps.BuildStep], bool] | bool = True,
) -> list[steps.BuildStep]:
out_steps = [
NixBuildCommand(
Expand All @@ -1116,7 +1169,6 @@ def nix_build_steps(
# We increase this over the default since the build output might end up in a different `nix build`.
timeout=60 * 60 * 3,
haltOnFailure=True,
doStepIf=global_do_step_if,
),
*post_build_steps,
Trigger(
Expand All @@ -1128,19 +1180,13 @@ def nix_build_steps(
sourceStamps=[],
alwaysUseLatest=False,
updateSourceStamp=False,
doStepIf=lambda buildstep: (
global_do_step_if
if isinstance(global_do_step_if, bool)
else global_do_step_if(buildstep)
)
and do_register_gcroot_if(buildstep),
doStepIf=lambda s: do_register_gcroot_if(s, branch_config),
copy_properties=["out_path", "attr"],
set_properties={"report_status": False},
),
steps.ShellCommand(
name="Delete temporary gcroots",
command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")],
doStepIf=global_do_step_if,
),
]

Expand All @@ -1150,7 +1196,7 @@ def nix_build_steps(
project=project,
name="Update build output",
path=outputs_path,
doStepIf=global_do_step_if,
branch_config=branch_config,
),
)

Expand All @@ -1161,12 +1207,15 @@ def nix_build_config(
project: GitProject,
worker_names: list[str],
post_build_steps: list[steps.BuildStep],
branch_config_dict: models.BranchConfigDict,
outputs_path: Path | None = None,
) -> BuilderConfig:
"""Builds one nix flake attribute."""
factory = util.BuildFactory()
factory.addSteps(
nix_build_steps(project, worker_names, post_build_steps, outputs_path)
nix_build_steps(
project, worker_names, post_build_steps, branch_config_dict, outputs_path
)
)

return util.BuilderConfig(
Expand Down Expand Up @@ -1229,30 +1278,25 @@ def nix_cached_failure_config(
project: GitProject,
worker_names: list[str],
skipped_worker_names: list[str],
branch_config_dict: models.BranchConfigDict,
post_build_steps: list[steps.BuildStep],
outputs_path: Path | None = None,
) -> BuilderConfig:
"""Dummy builder that is triggered when a build is cached as failed."""
factory = util.BuildFactory()
factory.addStep(
CachedFailureStep(
project=project,
worker_names=worker_names,
post_build_steps=post_build_steps,
branch_config_dict=branch_config_dict,
outputs_path=outputs_path,
name="Cached failure",
doStepIf=lambda buildstep: buildstep.build.reason != "rebuild",
haltOnFailure=True,
flunkOnFailure=True,
),
)

factory.addSteps(
nix_build_steps(
project,
worker_names,
post_build_steps,
outputs_path,
global_do_step_if=lambda buildstep: buildstep.build.reason == "rebuild",
)
)

return util.BuilderConfig(
name=f"{project.name}/nix-cached-failure",
project=project.name,
Expand All @@ -1266,6 +1310,7 @@ def nix_cached_failure_config(
def nix_skipped_build_config(
project: GitProject,
worker_names: list[str],
branch_config_dict: models.BranchConfigDict,
outputs_path: Path | None = None,
) -> BuilderConfig:
"""Dummy builder that is triggered when a build is skipped."""
Expand All @@ -1290,7 +1335,7 @@ def nix_skipped_build_config(
sourceStamps=[],
alwaysUseLatest=False,
updateSourceStamp=False,
doStepIf=do_register_gcroot_if,
doStepIf=lambda s: do_register_gcroot_if(s, branch_config_dict),
copy_properties=["out_path", "attr"],
set_properties={"report_status": False},
),
Expand All @@ -1301,6 +1346,7 @@ def nix_skipped_build_config(
project=project,
name="Update build output",
path=outputs_path,
branch_config=branch_config_dict,
),
)
return util.BuilderConfig(
Expand Down Expand Up @@ -1410,24 +1456,32 @@ def config_for_project(
job_report_limit: int | None,
failed_builds_db: FailedBuildDB,
per_repo_effects_secrets: dict[str, str],
branch_config_dict: models.BranchConfigDict,
outputs_path: Path | None = None,
) -> None:
config["projects"].append(Project(project.name))
config["schedulers"].extend(
[
schedulers.SingleBranchScheduler(
name=f"{project.project_id}-default-branch",
name=f"{project.project_id}-primary",
change_filter=util.ChangeFilter(
repository=project.url,
filter_fn=lambda c: c.branch == project.default_branch,
filter_fn=lambda c: branch_config_dict.do_run(
project.default_branch, c.branch
),
),
builderNames=[f"{project.name}/nix-eval"],
treeStableTimer=5,
),
)
]
)
config["schedulers"].extend(
[
# this is compatible with bors or github's merge queue
schedulers.SingleBranchScheduler(
name=f"{project.project_id}-merge-queue",
change_filter=util.ChangeFilter(
# TODO add filter
repository=project.url,
branch_re="(gh-readonly-queue/.*|staging|trying)",
),
Expand All @@ -1442,6 +1496,10 @@ def config_for_project(
),
builderNames=[f"{project.name}/nix-eval"],
),
]
)
config["schedulers"].extend(
[
# this is triggered from `nix-eval`
schedulers.Triggerable(
name=f"{project.project_id}-nix-build",
Expand Down Expand Up @@ -1511,6 +1569,7 @@ def config_for_project(
project,
worker_names,
outputs_path=outputs_path,
branch_config_dict=branch_config_dict,
post_build_steps=post_build_steps,
),
buildbot_effects_config(
Expand All @@ -1519,15 +1578,21 @@ def config_for_project(
worker_names=worker_names,
secrets=effects_secrets_cred,
),
nix_skipped_build_config(project, SKIPPED_BUILDER_NAMES, outputs_path),
nix_skipped_build_config(
project=project,
worker_names=SKIPPED_BUILDER_NAMES,
branch_config_dict=branch_config_dict,
outputs_path=outputs_path,
),
nix_failed_eval_config(project, SKIPPED_BUILDER_NAMES),
nix_dependency_failed_config(project, SKIPPED_BUILDER_NAMES),
nix_cached_failure_config(
project,
worker_names,
SKIPPED_BUILDER_NAMES,
post_build_steps,
outputs_path,
project=project,
worker_names=worker_names,
skipped_worker_names=SKIPPED_BUILDER_NAMES,
branch_config_dict=branch_config_dict,
post_build_steps=post_build_steps,
outputs_path=outputs_path,
),
nix_register_gcroot_config(project, worker_names),
],
Expand Down Expand Up @@ -1744,6 +1809,7 @@ def configure(self, config: dict[str, Any]) -> None:
job_report_limit=self.config.job_report_limit,
per_repo_effects_secrets=self.config.effects_per_repo_secrets,
failed_builds_db=DB,
branch_config_dict=self.config.branches,
outputs_path=self.config.outputs_path,
)

Expand Down
Loading

0 comments on commit b57de59

Please sign in to comment.