From b7a09e2bbadc126127634daf19c7e78267507f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 27 Dec 2023 00:52:26 +0100 Subject: [PATCH 01/23] allow out-commented code --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c18b28829..7e7902af7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,9 @@ ignore = [ # fixmes "FIX", + # commented out code + "ERA001", + # Unused function argument "ARG001", "ARG002", From b3d36eb9f10d64d0894b1d260ca5d1e223d1f729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 27 Dec 2023 00:39:36 +0100 Subject: [PATCH 02/23] integrate buildbot-effects into buildbot --- buildbot_nix/__init__.py | 163 ++++++++++++++++++++++++++++++++++++++- flake.nix | 1 + nix/buildbot-effects.nix | 14 ++++ nix/worker.nix | 1 + 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 nix/buildbot-effects.nix diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 58231e281..9d07c3018 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -75,6 +75,54 @@ class BuildbotNixError(Exception): pass +class BuildbotEffectsTrigger(Trigger): + """Dynamic trigger that run buildbot effect defined in a flake.""" + + def __init__( + self, + project: GitProject, + effects_scheduler: str, + effects: list[dict[str, Any]], + **kwargs: Any, + ) -> None: + if "name" not in kwargs: + kwargs["name"] = "trigger" + self.effects = effects + self.config = None + self.effects_scheduler = effects_scheduler + self.project = project + super().__init__( + waitForFinish=True, + schedulerNames=[effects_scheduler], + haltOnFailure=True, + flunkOnFailure=True, + sourceStamps=[], + alwaysUseLatest=False, + updateSourceStamp=False, + **kwargs, + ) + + def createTriggerProperties(self, props: Any) -> Any: # noqa: N802 + return props + + def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 + repo_name = self.project.name + project_id = slugify_project_name(repo_name) + source = f"buildbot-run-effect-{project_id}" + + triggered_schedulers = [] + for effect in self.effects: + props = Properties() + props.setProperty("virtual_builder_name", effect, source) + props.setProperty("status_name", f"effects.{effect}", source) + props.setProperty("virtual_builder_tags", "", source) + props.setProperty("command", effect, source) + + triggered_schedulers.append((self.effects_scheduler, props)) + + return triggered_schedulers + + class BuildTrigger(buildstep.ShellMixin, steps.BuildStep): """Dynamic trigger that creates a build for every attribute.""" @@ -708,6 +756,42 @@ async def run(self) -> int: return result +class BuildbotEffectsCommand(buildstep.ShellMixin, steps.BuildStep): + """Evaluate the effects of a flake and run them on the default branch.""" + + def __init__(self, project: GitProject, **kwargs: Any) -> None: + kwargs = self.setupShellMixin(kwargs) + super().__init__(**kwargs) + self.project = project + self.observer = logobserver.BufferLogObserver() + self.addLogObserver("stdio", self.observer) + + @defer.inlineCallbacks + def run(self) -> Generator[Any, object, Any]: + # run nix-eval-jobs --flake .#checks to generate the dict of stages + cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() + yield self.runCommand(cmd) + + # if the command passes extract the list of stages + result = cmd.results() + if result == util.SUCCESS: + # create a ShellCommand for each stage and add them to the build + effects = json.loads(self.observer.getStdout()) + repo_name = self.project.name + project_id = slugify_project_name(repo_name) + + self.build.addStepsAfterCurrentStep( + [ + BuildbotEffectsTrigger( + project=self.project, + effects_scheduler=f"{project_id}-run-effect", + name="Buildbot effect", + effects=effects, + ), + ], + ) + + return result class EvalErrorStep(steps.BuildStep): """Shows the error message of a failed evaluation.""" @@ -940,7 +1024,7 @@ def nix_eval_config( NixEvalCommand( project=project, env={}, - name="evaluate flake", + name="Evaluate flake", supported_systems=supported_systems, job_report_limit=job_report_limit, failed_builds_db=failed_builds_db, @@ -962,6 +1046,28 @@ def nix_eval_config( ], ), ) + factory.addStep( + BuildbotEffectsCommand( + project=project, + env={}, + name="Evaluate effects", + command=[ + # fmt: off + "buildbot-effects", + "--rev", + util.Property("revision"), + "--branch", + util.Property("branch"), + "--repo", + util.Property("github.repository.html_url"), + "list", + # fmt: on + ], + # TODO: support other branches? + doStepIf=lambda c: c.build.getProperty("branch", "") + == project.default_branch, + ) + ) return util.BuilderConfig( name=f"{project.name}/nix-eval", @@ -1245,6 +1351,50 @@ def nix_register_gcroot_config( factory=factory, ) +def buildbot_effects_config( + project: Project, git_url: str, worker_names: list[str] +) -> util.BuilderConfig: + """Builds one nix flake attribute.""" + factory = util.BuildFactory() + url_with_secret = util.Interpolate(git_url) + factory.addStep( + GitLocalPrMerge( + repourl=url_with_secret, + method="clean", + submodules=True, + haltOnFailure=True, + ), + ) + factory.addStep( + steps.ShellCommand( + name="Run effects", + command=[ + # fmt: off + "buildbot-effects", + # TODO: + # "--secrets", util.Secret(""), + "--rev", + util.Property("revision"), + "--branch", + util.Property("branch"), + "--repo", + # TODO: gittea + util.Property("github.base.repo.full_name"), + "run", + util.Property("command"), + # fmt: on + ], + ), + ) + return util.BuilderConfig( + name=f"{project.name}/run-effect", + project=project.name, + workernames=worker_names, + collapseRequests=False, + env={}, + factory=factory, + ) + def config_for_project( config: dict[str, Any], @@ -1294,7 +1444,11 @@ def config_for_project( name=f"{project.project_id}-nix-build", builderNames=[f"{project.name}/nix-build"], ), - # this is triggered from `nix-build` when the build is skipped + schedulers.Triggerable( + name=f"{project.project_id}-run-effect", + builderNames=[f"{project.name}/run-effect"], + ), + # this is triggered from `nix-eval` when the build is skipped schedulers.Triggerable( name=f"{project.project_id}-nix-skipped-build", builderNames=[f"{project.name}/nix-skipped-build"], @@ -1353,6 +1507,11 @@ def config_for_project( outputs_path=outputs_path, post_build_steps=post_build_steps, ), + buildbot_effects_config( + project, + git_url=project.get_project_url(), + worker_names=worker_names, + ), nix_skipped_build_config(project, SKIPPED_BUILDER_NAMES, outputs_path), nix_failed_eval_config(project, SKIPPED_BUILDER_NAMES), nix_dependency_failed_config(project, SKIPPED_BUILDER_NAMES), diff --git a/flake.nix b/flake.nix index f54ad66b7..57e717ce5 100644 --- a/flake.nix +++ b/flake.nix @@ -84,6 +84,7 @@ ]; }; packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { }; + packages.buildbot-effects = pkgs.python3.pkgs.callPackage ./nix/buildbot-effects.nix { }; checks = let nixosMachines = lib.mapAttrs' ( diff --git a/nix/buildbot-effects.nix b/nix/buildbot-effects.nix new file mode 100644 index 000000000..ac51bb09d --- /dev/null +++ b/nix/buildbot-effects.nix @@ -0,0 +1,14 @@ +{ lib, python3, bubblewrap, setuptools, buildPythonApplication }: +buildPythonApplication { + name = "buildbot-effects"; + format = "pyproject"; + src = ./..; + nativeBuildInputs = [ + setuptools + ]; + makeWrapperArgs = [ "--prefix PATH : ${lib.makeBinPath [ bubblewrap ]}" ]; + + postFixup = '' + rm -rf $out/${python3.pkgs.python.sitePackages}/buildbot_nix + ''; +} diff --git a/nix/worker.nix b/nix/worker.nix index 230291a5a..bfb80cf5c 100644 --- a/nix/worker.nix +++ b/nix/worker.nix @@ -80,6 +80,7 @@ in pkgs.bash pkgs.coreutils cfg.nixEvalJobs.package + (pkgs.python3.pkgs.callPackage ./buildbot-effects.nix { }) ]; environment.PYTHONPATH = "${python.withPackages (_: [ cfg.package ])}/${python.sitePackages}"; environment.MASTER_URL = cfg.masterUrl; From 7d46b3bff7f027cc8c514468672569acdeab320a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 1 Jan 2024 07:59:55 +0100 Subject: [PATCH 03/23] hercules effects: build effects output instead of herculesCI output --- buildbot_effects/__init__.py | 6 +++--- buildbot_effects/cli.py | 34 +++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/buildbot_effects/__init__.py b/buildbot_effects/__init__.py index ea6da55ea..abd4e1379 100644 --- a/buildbot_effects/__init__.py +++ b/buildbot_effects/__init__.py @@ -96,7 +96,7 @@ def effect_function(opts: EffectsOptions) -> str: rev = args["rev"] escaped_args = json.dumps(json.dumps(args)) url = json.dumps(f"git+file://{opts.path}?rev={rev}#") - return f"""(((builtins.getFlake {url}).outputs.herculesCI (builtins.fromJSON {escaped_args})).onPush.default.outputs.hci-effects)""" + return f"""(((builtins.getFlake {url}).outputs.herculesCI (builtins.fromJSON {escaped_args})).onPush.default.outputs.effects)""" def list_effects(opts: EffectsOptions) -> list[str]: @@ -110,11 +110,11 @@ def list_effects(opts: EffectsOptions) -> list[str]: return json.loads(proc.stdout) -def instantiate_effects(opts: EffectsOptions) -> str: +def instantiate_effects(effect: str, opts: EffectsOptions) -> str: cmd = [ "nix-instantiate", "--expr", - f"{effect_function(opts)}.deploy.run", + f"({effect_function(opts)}.{effect}).run or []", ] proc = run(cmd, stdout=subprocess.PIPE) return proc.stdout.rstrip() diff --git a/buildbot_effects/cli.py b/buildbot_effects/cli.py index 556e4cc83..72876d567 100644 --- a/buildbot_effects/cli.py +++ b/buildbot_effects/cli.py @@ -1,18 +1,22 @@ import argparse import json -from collections.abc import Callable +import sys from pathlib import Path from . import instantiate_effects, list_effects, parse_derivation, run_effects from .options import EffectsOptions -def list_command(options: EffectsOptions) -> None: - print(list_effects(options)) +def list_command(args: argparse.Namespace, options: EffectsOptions) -> None: + json.dump(list_effects(options), fp=sys.stdout, indent=2) -def run_command(options: EffectsOptions) -> None: - drv_path = instantiate_effects(options) +def run_command(args: argparse.Namespace, options: EffectsOptions) -> None: + effect = args.effect + drv_path = instantiate_effects(effect, options) + if drv_path == "": + print(f"Effect {effect} not found or not runnable for {options}") + return drvs = parse_derivation(drv_path) drv = next(iter(drvs.values())) @@ -20,11 +24,11 @@ def run_command(options: EffectsOptions) -> None: run_effects(drv_path, drv, secrets=secrets) -def run_all_command(options: EffectsOptions) -> None: +def run_all_command(args: argparse.Namespace, options: EffectsOptions) -> None: print("TODO") -def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]: +def parse_args() -> tuple[argparse.Namespace, EffectsOptions]: parser = argparse.ArgumentParser(description="Run effects from a hercules-ci flake") parser.add_argument( "--secrets", @@ -48,7 +52,8 @@ def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]: ) parser.add_argument( "--path", - type=str, + type=Path, + default=Path(), help="Path to the repository", ) subparser = parser.add_subparsers( @@ -77,9 +82,16 @@ def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]: run_all_parser.set_defaults(command=run_all_command) args = parser.parse_args() - return args.command, EffectsOptions(secrets=args.secrets) + opts = EffectsOptions( + secrets=args.secrets, + branch=args.branch, + rev=args.rev, + repo=args.repo, + path=args.path.resolve(), + ) + return args, opts def main() -> None: - command, options = parse_args() - command(options) + args, options = parse_args() + args.command(args, options) From 4d5cacc899f0d03895bc5093d8e703d3e660eb34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 27 Dec 2023 00:35:53 +0100 Subject: [PATCH 04/23] fix buildbot-effects command --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7e7902af7..0a9782512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,8 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python" ] -version = "0.1.1" -scripts = { buildbot-effects = "hercules_effects.cli:main" } +version = "0.0.1" +scripts = { buildbot-effects = "buildbot_effects.cli:main" } [tool.setuptools] packages = [ From 1129486d17b409113aa5f611691da37575f3632d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 7 Jan 2024 08:42:04 +0100 Subject: [PATCH 05/23] fix nixos test names --- nix/checks/master.nix | 2 +- nix/checks/worker.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/checks/master.nix b/nix/checks/master.nix index a19dbef71..8ec233571 100644 --- a/nix/checks/master.nix +++ b/nix/checks/master.nix @@ -1,5 +1,5 @@ (import ./lib.nix) { - name = "from-nixos"; + name = "master"; nodes = { # `self` here is set by using specialArgs in `lib.nix` node1 = diff --git a/nix/checks/worker.nix b/nix/checks/worker.nix index bc05e0d4e..694425530 100644 --- a/nix/checks/worker.nix +++ b/nix/checks/worker.nix @@ -1,5 +1,5 @@ (import ./lib.nix) { - name = "from-nixos"; + name = "worker"; nodes = { # `self` here is set by using specialArgs in `lib.nix` node1 = From f9599f6d8805b5bfcc84da2437400901efd75dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 7 Jan 2024 08:39:38 +0100 Subject: [PATCH 06/23] buildbot-effects: add nixos test --- nix/checks/effects.nix | 16 ++++++++++++++++ nix/checks/flake-module.nix | 1 + 2 files changed, 17 insertions(+) create mode 100644 nix/checks/effects.nix diff --git a/nix/checks/effects.nix b/nix/checks/effects.nix new file mode 100644 index 000000000..0d1e578f2 --- /dev/null +++ b/nix/checks/effects.nix @@ -0,0 +1,16 @@ +(import ./lib.nix) { + name = "effects"; + nodes = { + # `self` here is set by using specialArgs in `lib.nix` + node1 = { self, pkgs, ... }: { + environment.systemPackages = [ + (pkgs.python3.pkgs.callPackage ../../nix/buildbot-effects.nix { }) + ]; + }; + }; + testScript = '' + start_all() + # wait for our service to start + node1.succeed("buildbot-effects --help") + ''; +} diff --git a/nix/checks/flake-module.nix b/nix/checks/flake-module.nix index 324ab2de5..99c7813aa 100644 --- a/nix/checks/flake-module.nix +++ b/nix/checks/flake-module.nix @@ -13,6 +13,7 @@ lib.mkIf (pkgs.hostPlatform.isLinux) { master = import ./master.nix checkArgs; worker = import ./worker.nix checkArgs; + effects = import ./effects.nix checkArgs; }; }; } From 2eb4b438ca096323380f76856b22cffa211877c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 14 Jan 2024 15:41:04 +0100 Subject: [PATCH 07/23] add features matrix --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 7d7d1bf66..d95c2131a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,21 @@ into the Nix ecosystem. This module is under active development, and while it's generally stable and widely used, please be aware that some APIs may change over time. +## Features + +- Fast, Parallel evaluation using + [nix-eval-jobs](https://github.com/nix-community/nix-eval-jobs) +- Github integration: + - Login with GitHub to control builds + - CI status notification in pull requests and on the default branch +- All builds share the same nix store for speed +- The last attribute of a build is protected from garbage collection +- Build matrix based on `.#checks` attributes +- No arbitrary code runs outside of the Nix sandbox +- _experimental_ + [hercules-ci effect](https://docs.hercules-ci.com/hercules-ci-effects/) to run + impure CI steps i.e. deploying NixOS + ## Getting Started with Buildbot Setup To set up Buildbot using Buildbot-nix, you can start by exploring the provided From 360579ed03a2755e4e5d018f2150147981caa99b Mon Sep 17 00:00:00 2001 From: magic_rb Date: Wed, 18 Sep 2024 22:39:12 +0200 Subject: [PATCH 08/23] Adapt to newer versions of `buildbot-nix` and make effects optional Signed-off-by: magic_rb --- buildbot_effects/__init__.py | 15 ++++++++++++--- buildbot_nix/__init__.py | 8 ++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/buildbot_effects/__init__.py b/buildbot_effects/__init__.py index abd4e1379..ecd449a95 100644 --- a/buildbot_effects/__init__.py +++ b/buildbot_effects/__init__.py @@ -96,7 +96,16 @@ def effect_function(opts: EffectsOptions) -> str: rev = args["rev"] escaped_args = json.dumps(json.dumps(args)) url = json.dumps(f"git+file://{opts.path}?rev={rev}#") - return f"""(((builtins.getFlake {url}).outputs.herculesCI (builtins.fromJSON {escaped_args})).onPush.default.outputs.effects)""" + return f""" + let + flake = builtins.getFlake {url}; + effects = flake.outputs.herculesCI (builtins.fromJSON {escaped_args}); + in + if flake.outputs ? herculesCI then + effects.onPush.default.outputs.effects or {{}} + else + {{}} + """ def list_effects(opts: EffectsOptions) -> list[str]: @@ -104,7 +113,7 @@ def list_effects(opts: EffectsOptions) -> list[str]: "eval", "--json", "--expr", - f"builtins.attrNames {effect_function(opts)}", + f"builtins.attrNames ({effect_function(opts)})", ) proc = run(cmd, stdout=subprocess.PIPE) return json.loads(proc.stdout) @@ -114,7 +123,7 @@ def instantiate_effects(effect: str, opts: EffectsOptions) -> str: cmd = [ "nix-instantiate", "--expr", - f"({effect_function(opts)}.{effect}).run or []", + f"(({effect_function(opts)}).{effect}).run or []", ] proc = run(cmd, stdout=subprocess.PIPE) return proc.stdout.rstrip() diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 9d07c3018..6ef79da64 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -106,9 +106,7 @@ def createTriggerProperties(self, props: Any) -> Any: # noqa: N802 return props def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802 - repo_name = self.project.name - project_id = slugify_project_name(repo_name) - source = f"buildbot-run-effect-{project_id}" + source = f"buildbot-run-effect-{self.project.project_id}" triggered_schedulers = [] for effect in self.effects: @@ -777,14 +775,12 @@ def run(self) -> Generator[Any, object, Any]: if result == util.SUCCESS: # create a ShellCommand for each stage and add them to the build effects = json.loads(self.observer.getStdout()) - repo_name = self.project.name - project_id = slugify_project_name(repo_name) self.build.addStepsAfterCurrentStep( [ BuildbotEffectsTrigger( project=self.project, - effects_scheduler=f"{project_id}-run-effect", + effects_scheduler=f"{self.project.project_id}-run-effect", name="Buildbot effect", effects=effects, ), From 07769f127d241ca6761d405f5e77fdee189ce795 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Tue, 1 Oct 2024 14:29:40 -0400 Subject: [PATCH 09/23] Properly finalize writing to the munged secrets.json A mere .write doesn't flush the write out (it's fully-buffered), so the bwrapped secrets remain empty. Tricky! --- buildbot_effects/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/buildbot_effects/__init__.py b/buildbot_effects/__init__.py index ecd449a95..af3d9e7cd 100644 --- a/buildbot_effects/__init__.py +++ b/buildbot_effects/__init__.py @@ -223,6 +223,7 @@ def run_effects( secrets = secrets.copy() secrets["hercules-ci"] = {"data": {"token": "dummy"}} tmp.write(json.dumps(secrets).encode()) + tmp.flush() bubblewrap_cmd.extend( [ "--ro-bind", From 754c69a0d02e509856dc18a0d411f9cf936287ac Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Mon, 30 Sep 2024 10:36:29 -0400 Subject: [PATCH 10/23] Configure & thread per-repo secrets through to the effects runner * Add a nixpkgs configuration setting effects.perRepoSecretFiles, which associates a repository with a path containing a hci secrets.json-style file (that could be sourced from sops-nix or agenix). * Configure that and pass the secrets to effects running in projects for the respective repositories. --- buildbot_nix/__init__.py | 79 +++++++++++++++++++++++++--------------- buildbot_nix/models.py | 1 + nix/buildbot-effects.nix | 8 +++- nix/checks/effects.nix | 12 +++--- nix/master.nix | 29 ++++++++++++++- 5 files changed, 92 insertions(+), 37 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 6ef79da64..3d059b083 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -754,6 +754,7 @@ async def run(self) -> int: return result + class BuildbotEffectsCommand(buildstep.ShellMixin, steps.BuildStep): """Evaluate the effects of a flake and run them on the default branch.""" @@ -789,6 +790,7 @@ def run(self) -> Generator[Any, object, Any]: return result + class EvalErrorStep(steps.BuildStep): """Shows the error message of a failed evaluation.""" @@ -1347,8 +1349,9 @@ def nix_register_gcroot_config( factory=factory, ) + def buildbot_effects_config( - project: Project, git_url: str, worker_names: list[str] + project: Project, git_url: str, worker_names: list[str], secrets: str | None ) -> util.BuilderConfig: """Builds one nix flake attribute.""" factory = util.BuildFactory() @@ -1361,26 +1364,33 @@ def buildbot_effects_config( haltOnFailure=True, ), ) - factory.addStep( - steps.ShellCommand( - name="Run effects", - command=[ - # fmt: off - "buildbot-effects", - # TODO: - # "--secrets", util.Secret(""), - "--rev", - util.Property("revision"), - "--branch", - util.Property("branch"), - "--repo", - # TODO: gittea - util.Property("github.base.repo.full_name"), - "run", - util.Property("command"), - # fmt: on - ], - ), + secrets_list = [] + secrets_args = [] + if secrets is not None: + secrets_list = [("../secrets.json", util.Secret(secrets))] + secrets_args = ["--secrets", "../secrets.json"] + factory.addSteps( + [ + steps.ShellCommand( + name="Run effects", + command=[ + # fmt: off + "buildbot-effects", + "--rev", + util.Property("revision"), + "--branch", + util.Property("branch"), + "--repo", + # TODO: gittea + util.Property("github.base.repo.full_name"), + *secrets_args, + "run", + util.Property("command"), + # fmt: on + ], + ), + ], + withSecrets=secrets_list, ) return util.BuilderConfig( name=f"{project.name}/run-effect", @@ -1403,6 +1413,7 @@ def config_for_project( post_build_steps: list[steps.BuildStep], job_report_limit: int | None, failed_builds_db: FailedBuildDB, + per_repo_effects_secrets: dict[str, str], outputs_path: Path | None = None, ) -> None: config["projects"].append(Project(project.name)) @@ -1482,6 +1493,9 @@ def config_for_project( ), ], ) + key = f"{project.type}:{project.owner}/{project.repo}" + effects_secrets_cred = per_repo_effects_secrets.get(key) + config["builders"].extend( [ # Since all workers run on the same machine, we only assign one of them to do the evaluation. @@ -1507,6 +1521,7 @@ def config_for_project( project, git_url=project.get_project_url(), worker_names=worker_names, + secrets=effects_secrets_cred, ), nix_skipped_build_config(project, SKIPPED_BUILDER_NAMES, outputs_path), nix_failed_eval_config(project, SKIPPED_BUILDER_NAMES), @@ -1719,15 +1734,19 @@ def configure(self, config: dict[str, Any]) -> None: for project in projects: config_for_project( - config, - project, - worker_names, - self.config.build_systems, - self.config.eval_worker_count or multiprocessing.cpu_count(), - self.config.eval_max_memory_size, - eval_lock, - [x.to_buildstep() for x in self.config.post_build_steps], - self.config.job_report_limit, + config=config, + project=project, + worker_names=worker_names, + nix_supported_systems=self.config.build_systems, + nix_eval_worker_count=self.config.eval_worker_count + or multiprocessing.cpu_count(), + nix_eval_max_memory_size=self.config.eval_max_memory_size, + eval_lock=eval_lock, + post_build_steps=[ + x.to_buildstep() for x in self.config.post_build_steps + ], + job_report_limit=self.config.job_report_limit, + per_repo_effects_secrets=self.config.effects_per_repo_secrets, failed_builds_db=DB, outputs_path=self.config.outputs_path, ) diff --git a/buildbot_nix/models.py b/buildbot_nix/models.py index 4792e3681..912f9ea5f 100644 --- a/buildbot_nix/models.py +++ b/buildbot_nix/models.py @@ -192,6 +192,7 @@ class BuildbotNixConfig(BaseModel): post_build_steps: list[PostBuildStep] job_report_limit: int | None http_basic_auth_password_file: Path | None + effects_per_repo_secrets: dict[str, str] @property def nix_workers_secret(self) -> str: diff --git a/nix/buildbot-effects.nix b/nix/buildbot-effects.nix index ac51bb09d..81b9f80a1 100644 --- a/nix/buildbot-effects.nix +++ b/nix/buildbot-effects.nix @@ -1,4 +1,10 @@ -{ lib, python3, bubblewrap, setuptools, buildPythonApplication }: +{ + lib, + python3, + bubblewrap, + setuptools, + buildPythonApplication, +}: buildPythonApplication { name = "buildbot-effects"; format = "pyproject"; diff --git a/nix/checks/effects.nix b/nix/checks/effects.nix index 0d1e578f2..4c30b407a 100644 --- a/nix/checks/effects.nix +++ b/nix/checks/effects.nix @@ -2,11 +2,13 @@ name = "effects"; nodes = { # `self` here is set by using specialArgs in `lib.nix` - node1 = { self, pkgs, ... }: { - environment.systemPackages = [ - (pkgs.python3.pkgs.callPackage ../../nix/buildbot-effects.nix { }) - ]; - }; + node1 = + { self, pkgs, ... }: + { + environment.systemPackages = [ + (pkgs.python3.pkgs.callPackage ../../nix/buildbot-effects.nix { }) + ]; + }; }; testScript = '' start_all() diff --git a/nix/master.nix b/nix/master.nix index 20d1d46f5..2d5690a46 100644 --- a/nix/master.nix +++ b/nix/master.nix @@ -26,6 +26,19 @@ let else builtins.toJSON value; + cleanUpRepoName = + name: + builtins.replaceStrings + [ + "/" + ":" + ] + [ + "_slash_" + "_colon_" + ] + name; + backendPort = if (cfg.accessMode ? "fullyPrivate") then cfg.accessMode.fullyPrivate.port @@ -492,6 +505,13 @@ in ''; default = 50; }; + + effects.perRepoSecretFiles = lib.mkOption { + type = lib.types.attrsOf lib.types.path; + description = "Per-repository secrets files for buildbot effects. The attribute name is of the form \"github:owner/repo\". The secrets themselves need to be valid JSON files."; + default = { }; + example = ''{ "github:nix-community/buildbot-nix" = config.agenix.secrets.buildbot-nix-effects-secrets.path; }''; + }; }; }; config = lib.mkMerge [ @@ -659,6 +679,10 @@ in post_build_steps = cfg.postBuildSteps; job_report_limit = cfg.jobReportLimit; http_basic_auth_password_file = cfg.httpBasicAuthPasswordFile; + effects_per_repo_secrets = lib.mapAttrs' (name: _path: { + inherit name; + value = "effects-secret__${cleanUpRepoName name}"; + }) cfg.effects.perRepoSecretFiles; } }").read_text())) ) @@ -726,7 +750,10 @@ in ++ lib.optionals cfg.gitea.enable [ "gitea-token:${cfg.gitea.tokenFile}" "gitea-webhook-secret:${cfg.gitea.webhookSecretFile}" - ]; + ] + ++ (lib.mapAttrsToList ( + repoName: path: "effects-secret__${cleanUpRepoName repoName}:${path}" + ) cfg.effects.perRepoSecretFiles); RuntimeDirectory = "buildbot-master"; }; }; From d197405368cca9d1a8955996251be8768bb5c6a5 Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Tue, 1 Oct 2024 18:40:33 -0400 Subject: [PATCH 11/23] Set TMPDIR to /tmp and clear all other temporary dir variables `nix develop` sets TMPDIR, TMP, TEMP, TEMPDIR to a directory that doesn't exist in the new tmpfs, which some programs don't take too kindly to. Instead, let's override the TMPDIR variable with a temporary dir that we know exists & is good. Just for hygiene reasons (and to keep others from tearing their hair out), just unset the other temporary directory variables. --- buildbot_effects/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/buildbot_effects/__init__.py b/buildbot_effects/__init__.py index af3d9e7cd..c0aa8a3ad 100644 --- a/buildbot_effects/__init__.py +++ b/buildbot_effects/__init__.py @@ -142,12 +142,15 @@ def parse_derivation(path: str) -> dict[str, Any]: return json.loads(proc.stdout) -def env_args(env: dict[str, str]) -> list[str]: +def env_args(env: dict[str, str], clear_env: set[str]) -> list[str]: result = [] for k, v in env.items(): result.append("--setenv") result.append(f"{k}") result.append(f"{v}") + for k in clear_env: + result.append("--unsetenv") + result.append(f"{k}") return result @@ -180,6 +183,11 @@ def run_effects( env["IN_HERCULES_CI_EFFECT"] = "true" env["HERCULES_CI_SECRETS_JSON"] = "/run/secrets.json" env["NIX_BUILD_TOP"] = "/build" + env["TMPDIR"] = "/tmp" # noqa: S108 + clear_env = set() + clear_env.add("TMP") + clear_env.add("TEMP") + clear_env.add("TEMPDIR") bwrap = shutil.which("bwrap") if bwrap is None: msg = "bwrap' executable not found" @@ -231,7 +239,7 @@ def run_effects( "/run/secrets.json", ], ) - bubblewrap_cmd.extend(env_args(env)) + bubblewrap_cmd.extend(env_args(env, clear_env)) bubblewrap_cmd.append("--") bubblewrap_cmd.extend(sandboxed_cmd) with pipe() as (r_file, w_file): From 0f3e8804eaf6aef93de639d8e71138ed2cc8a1bc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 3 Nov 2024 02:11:58 +0000 Subject: [PATCH 12/23] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/3d04084d54bedc3d6b8b736c70ef449225c361b1?narHash=sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0%3D' (2024-10-01) → 'github:hercules-ci/flake-parts/506278e768c2a08bec68eb62932193e341f55c90?narHash=sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS%2Bb4tfNFCwE%3D' (2024-11-01) • Updated input 'nixpkgs': 'github:Nixos/nixpkgs/ccd7e10e004a6e93486c1dd536f89e85d482d685?narHash=sha256-YaZZlvZj9isN2i8l24A4/7TT89fvpX11nVn2dusl2MM%3D' (2024-10-30) → 'github:Nixos/nixpkgs/cf3e5d3744dc26c3498aa5dadfa0e078c632cede?narHash=sha256-dYSnUmWkVwwEpXv0F/fZBBYT10Bgx%2BU2PJ2obdBTqo8%3D' (2024-11-02) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 856e6df14..bf20ec4b5 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1727826117, - "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=", + "lastModified": 1730504689, + "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1", + "rev": "506278e768c2a08bec68eb62932193e341f55c90", "type": "github" }, "original": { @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1730247135, - "narHash": "sha256-YaZZlvZj9isN2i8l24A4/7TT89fvpX11nVn2dusl2MM=", + "lastModified": 1730555274, + "narHash": "sha256-dYSnUmWkVwwEpXv0F/fZBBYT10Bgx+U2PJ2obdBTqo8=", "owner": "Nixos", "repo": "nixpkgs", - "rev": "ccd7e10e004a6e93486c1dd536f89e85d482d685", + "rev": "cf3e5d3744dc26c3498aa5dadfa0e078c632cede", "type": "github" }, "original": { From 3bd9819dfc06941fa20c7a944e643448cf3d981e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Nov 2024 02:03:27 +0000 Subject: [PATCH 13/23] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:Nixos/nixpkgs/cf3e5d3744dc26c3498aa5dadfa0e078c632cede?narHash=sha256-dYSnUmWkVwwEpXv0F/fZBBYT10Bgx%2BU2PJ2obdBTqo8%3D' (2024-11-02) → 'github:Nixos/nixpkgs/1c07b97d2d4302baca8c61fa2d0d4632427972a7?narHash=sha256-OrCMJZ8qZftRplhoB%2BBksvoPLBOZQpH8mnACgPKNuMc%3D' (2024-11-06) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index bf20ec4b5..8b3f407d5 100644 --- a/flake.lock +++ b/flake.lock @@ -22,11 +22,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1730555274, - "narHash": "sha256-dYSnUmWkVwwEpXv0F/fZBBYT10Bgx+U2PJ2obdBTqo8=", + "lastModified": 1730902633, + "narHash": "sha256-OrCMJZ8qZftRplhoB+BksvoPLBOZQpH8mnACgPKNuMc=", "owner": "Nixos", "repo": "nixpkgs", - "rev": "cf3e5d3744dc26c3498aa5dadfa0e078c632cede", + "rev": "1c07b97d2d4302baca8c61fa2d0d4632427972a7", "type": "github" }, "original": { From 26a49a127e83293282eada75638fb122750e58e8 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 9 Nov 2024 20:31:01 +0100 Subject: [PATCH 14/23] Fix some type errors I encountered Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 3d059b083..6da367604 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -29,7 +29,7 @@ from buildbot.process.properties import Properties from buildbot.process.results import ALL_RESULTS, statusToString, worst_status from buildbot.reporters.utils import getURLForBuildrequest -from buildbot.schedulers.base import BaseScheduler +from buildbot.schedulers.triggerable import Triggerable from buildbot.secrets.providers.file import SecretInAFile from buildbot.steps.trigger import Trigger from buildbot.www.authz import Authz @@ -217,7 +217,7 @@ def interrupt(self, reason: str | Failure) -> None: if self.build.conn is None and self.wait_for_finish_deferred is not None: self.wait_for_finish_deferred.cancel() - def get_scheduler_by_name(self, name: str) -> BaseScheduler: + def get_scheduler_by_name(self, name: str) -> Triggerable: schedulers = self.master.scheduler_manager.namedServices if name not in schedulers: message = f"unknown triggered scheduler: {name!r}" @@ -228,7 +228,7 @@ def get_scheduler_by_name(self, name: str) -> BaseScheduler: @staticmethod def set_common_properties( props: Properties, - project: Project, + project: GitProject, source: str, combine_builds: bool, job: NixEvalJob, @@ -314,7 +314,7 @@ async def schedule( scheduler_name: str, props: Properties, ) -> tuple[dict[int, int], defer.Deferred[list[int]]]: - scheduler: BaseScheduler = self.get_scheduler_by_name(scheduler_name) + scheduler: Triggerable = self.get_scheduler_by_name(scheduler_name) ids_deferred, results_deferred = scheduler.trigger( waited_for=True, @@ -438,7 +438,7 @@ async def run(self) -> None: self.running = True build_props = self.build.getProperties() ss_for_trigger = self.prepare_sourcestamp_list_for_trigger() - scheduler_log: Log = await self.addLog("scheduler") + scheduler_log: StreamLog = await self.addLog("scheduler") # inject failed buildsteps for any failed eval jobs we got overall_result = SUCCESS if not self.failed_jobs else util.FAILURE @@ -1663,7 +1663,7 @@ async def activate(self) -> None: await super().activate() -DB = None +DB: FailedBuildDB | None = None class NixConfigurator(ConfiguratorBase): From dff4ee94e6fa3fb7fa4f55801bd09a8589e7cbc8 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 9 Nov 2024 21:16:42 +0100 Subject: [PATCH 15/23] Actually enable `mypy` in `treefmt` Signed-off-by: magic_rb --- nix/treefmt/flake-module.nix | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/nix/treefmt/flake-module.nix b/nix/treefmt/flake-module.nix index 3750655de..d69377741 100644 --- a/nix/treefmt/flake-module.nix +++ b/nix/treefmt/flake-module.nix @@ -19,13 +19,25 @@ programs.mypy = { enable = pkgs.stdenv.buildPlatform.isLinux; - directories.".".extraPythonPackages = [ - pkgs.buildbot - pkgs.buildbot-worker - pkgs.python3.pkgs.twisted - ]; + package = pkgs.buildbot.python.pkgs.mypy; + directories."." = { + modules = [ + "buildbot_nix" + ]; + extraPythonPackages = [ + (pkgs.python3.pkgs.toPythonModule pkgs.buildbot) + pkgs.buildbot-worker + pkgs.python3.pkgs.twisted + pkgs.python3.pkgs.pydantic + pkgs.python3.pkgs.zope-interface + ]; + }; }; + # the mypy module adds `./buildbot_nix/**/*.py` which does not appear to work + # furthermore, saying `directories.""` will lead to `/buildbot_nix/**/*.py` which + # is obviously incorrect... + settings.formatter."mypy-.".includes = [ "buildbot_nix/**/*.py" ]; settings.formatter.ruff-check.priority = 1; settings.formatter.ruff-format.priority = 2; }; From 9697d0d907fceb8a80f1022b04aefa99435a2a98 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 9 Nov 2024 21:19:36 +0100 Subject: [PATCH 16/23] Disable `disallow_untyped_calls` in `pyproject.toml [mypy]` Signed-off-by: magic_rb --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0a9782512..cc64de5e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,6 @@ ignore-names = [ python_version = "3.11" pretty = true warn_redundant_casts = true -disallow_untyped_calls = true +disallow_untyped_calls = false disallow_untyped_defs = true no_implicit_optional = true From f3ff3ff7defd0f90209c471f0d64f23a1628d9bf Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 9 Nov 2024 21:58:29 +0100 Subject: [PATCH 17/23] Fix all type errors `mypy` complained about Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 4 ++-- buildbot_nix/github_projects.py | 8 ++++---- buildbot_nix/models.py | 2 +- buildbot_nix/nix_status_generator.py | 26 ++++++++++++-------------- buildbot_nix/oauth2_proxy_auth.py | 2 +- buildbot_nix/repo_config/__init__.py | 9 ++++++--- 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 6da367604..f8a594b21 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -1607,7 +1607,7 @@ async def match_BuildEndpoint_stop( # noqa: N802 epobject: Any, epdict: dict[str, Any], options: dict[str, Any], - ) -> Match: + ) -> Match | None: return await self.check_builder(epobject, epdict, "build") async def match_BuildRequestEndpoint_stop( # noqa: N802 @@ -1615,7 +1615,7 @@ async def match_BuildRequestEndpoint_stop( # noqa: N802 epobject: Any, epdict: dict[str, Any], options: dict[str, Any], - ) -> Match: + ) -> Match | None: return await self.check_builder(epobject, epdict, "buildrequest") diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 3d18a0eea..47838cf28 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -365,10 +365,10 @@ def create_reload_builder_steps( class GitHubLegacySecretService(SecretProviderBase): - name = "GitHubLegacySecretService" + name: str | None = "GitHubLegacySecretService" # type: ignore token: LegacyToken - def reconfigService(self, token: LegacyToken) -> None: + def reconfigService(self, token: LegacyToken) -> None: # type: ignore self.token = token def get(self, entry: str) -> str | None: @@ -458,11 +458,11 @@ def create_reload_builder_steps( class GitHubAppSecretService(SecretProviderBase): - name = "GitHubAppSecretService" + name: str | None = "GitHubAppSecretService" # type: ignore installation_tokens: dict[int, InstallationToken] jwt_token: JWTToken - def reconfigService( + def reconfigService( # type: ignore self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken ) -> None: self.installation_tokens = installation_tokens diff --git a/buildbot_nix/models.py b/buildbot_nix/models.py index 912f9ea5f..038815381 100644 --- a/buildbot_nix/models.py +++ b/buildbot_nix/models.py @@ -231,7 +231,7 @@ class NixEvalJobSuccess(BaseModel): NixEvalJob = NixEvalJobError | NixEvalJobSuccess -NixEvalJobModel = TypeAdapter(NixEvalJob) +NixEvalJobModel: TypeAdapter[NixEvalJob] = TypeAdapter(NixEvalJob) class NixDerivation(BaseModel): diff --git a/buildbot_nix/nix_status_generator.py b/buildbot_nix/nix_status_generator.py index 3b95a0295..fcf9ecd49 100644 --- a/buildbot_nix/nix_status_generator.py +++ b/buildbot_nix/nix_status_generator.py @@ -13,10 +13,10 @@ from buildbot.reporters import utils from buildbot.reporters.base import ReporterBase from buildbot.reporters.generators.utils import BuildStatusGeneratorMixin -from buildbot.reporters.message import MessageFormatterRenderable +from buildbot.reporters.message import MessageFormatterRenderable, MessageFormatterBase from buildbot.reporters.utils import getDetailsForBuild from twisted.logger import Logger -from zope.interface import implementer +from zope.interface import implementer # type: ignore[import] log = Logger() @@ -91,8 +91,8 @@ class BuildNixEvalStatusGenerator(BuildStatusGeneratorMixin): compare_attrs: ClassVar[list[str]] = ["start_formatter", "end_formatter"] - start_formatter: IRenderable - end_formatter: IRenderable + start_formatter: MessageFormatterBase + endj_formatter: MessageFormatterBase def __init__( self, @@ -102,23 +102,21 @@ def __init__( branches: None | list[str] = None, add_logs: bool | list[str] = False, add_patch: bool = False, - start_formatter: None | IRenderable = None, - end_formatter: None | IRenderable = None, + start_formatter: None | MessageFormatterBase = None, + end_formatter: None | MessageFormatterBase = None, ) -> None: super().__init__( "all", tags, builders, schedulers, branches, None, add_logs, add_patch ) - self.start_formatter = start_formatter - if self.start_formatter is None: - self.start_formatter = MessageFormatterRenderable("Build started.") - self.end_formatter = end_formatter - if self.end_formatter is None: - self.end_formatter = MessageFormatterRenderable("Build done.") + + self.start_formatter = start_formatter or MessageFormatterRenderable("Build started.") + self.end_formatter = end_formatter or MessageFormatterRenderable("Build done.") + # TODO: copy pasted from buildbot, make it static upstream and reuse @staticmethod async def partial_build_dict( - master: BuildMaster, buildrequest: BuildRequest + master: BuildMaster, buildrequest: dict[str, Any] ) -> dict[str, Any]: brdict: Any = await master.db.buildrequests.getBuildRequest( buildrequest["buildrequestid"] @@ -166,7 +164,7 @@ async def generate( master: BuildMaster, reporter: ReporterBase, key: tuple[str, None | Any, str], - data: Build | BuildRequest, + data: dict[str, Any], # TODO database types ) -> None | dict[str, Any]: what, _, event = key if what == "builds": diff --git a/buildbot_nix/oauth2_proxy_auth.py b/buildbot_nix/oauth2_proxy_auth.py index 2c9f32c6f..f4a42515b 100644 --- a/buildbot_nix/oauth2_proxy_auth.py +++ b/buildbot_nix/oauth2_proxy_auth.py @@ -17,7 +17,7 @@ class OAuth2ProxyAuth(AuthBase): header: ClassVar[bytes] = b"Authorization" prefix: ClassVar[bytes] = b"Basic " - user_info_provider: UserInfoProviderBase = None + user_info_provider: UserInfoProviderBase password: bytes def __init__(self, password: str, **kwargs: Any) -> None: diff --git a/buildbot_nix/repo_config/__init__.py b/buildbot_nix/repo_config/__init__.py index 2658baf77..fb9c3afb6 100644 --- a/buildbot_nix/repo_config/__init__.py +++ b/buildbot_nix/repo_config/__init__.py @@ -3,9 +3,12 @@ from typing import TYPE_CHECKING, Self from buildbot.process.buildstep import ShellMixin +from buildbot.process.buildstep import BuildStep if TYPE_CHECKING: - from buildbot.process.log import Log + from buildbot.process.log import StreamLog + class BuildStepShellMixin(BuildStep, ShellMixin): + pass from pydantic import BaseModel, ValidationError @@ -18,8 +21,8 @@ class BranchConfig(BaseModel): attribute: str = "checks" @classmethod - async def extract_during_step(cls, buildstep: ShellMixin) -> Self: - stdio: Log = await buildstep.addLog("stdio") + async def extract_during_step(cls, buildstep: BuildStepShellMixin) -> Self: + stdio: StreamLog = await buildstep.addLog("stdio") cmd = await buildstep.makeRemoteShellCommand( collectStdout=True, collectStderr=True, From 66c0085477f826ae9ea1369f1e8508d8175fb602 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sat, 9 Nov 2024 22:01:11 +0100 Subject: [PATCH 18/23] Satisfy `treefmt` Signed-off-by: magic_rb --- buildbot_nix/__init__.py | 2 +- buildbot_nix/github_projects.py | 8 ++++---- buildbot_nix/nix_status_generator.py | 13 +++++++------ buildbot_nix/repo_config/__init__.py | 11 ++++++----- nix/treefmt/flake-module.nix | 6 ++++-- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index f8a594b21..945df7cb6 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -36,7 +36,7 @@ from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match if TYPE_CHECKING: - from buildbot.process.log import Log, StreamLog + from buildbot.process.log import StreamLog from buildbot.www.auth import AuthBase from twisted.internet import defer diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 47838cf28..c5be58e44 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -365,10 +365,10 @@ def create_reload_builder_steps( class GitHubLegacySecretService(SecretProviderBase): - name: str | None = "GitHubLegacySecretService" # type: ignore + name: str | None = "GitHubLegacySecretService" # type: ignore[assignment] token: LegacyToken - def reconfigService(self, token: LegacyToken) -> None: # type: ignore + def reconfigService(self, token: LegacyToken) -> None: # type: ignore[override] self.token = token def get(self, entry: str) -> str | None: @@ -458,11 +458,11 @@ def create_reload_builder_steps( class GitHubAppSecretService(SecretProviderBase): - name: str | None = "GitHubAppSecretService" # type: ignore + name: str | None = "GitHubAppSecretService" # type: ignore[assignment] installation_tokens: dict[int, InstallationToken] jwt_token: JWTToken - def reconfigService( # type: ignore + def reconfigService( # type: ignore[override] self, installation_tokens: dict[int, InstallationToken], jwt_token: JWTToken ) -> None: self.installation_tokens = installation_tokens diff --git a/buildbot_nix/nix_status_generator.py b/buildbot_nix/nix_status_generator.py index fcf9ecd49..4818f71bc 100644 --- a/buildbot_nix/nix_status_generator.py +++ b/buildbot_nix/nix_status_generator.py @@ -4,7 +4,7 @@ from enum import Enum from typing import Any, ClassVar -from buildbot.interfaces import IRenderable, IReportGenerator +from buildbot.interfaces import IReportGenerator from buildbot.master import BuildMaster from buildbot.process.build import Build from buildbot.process.buildrequest import BuildRequest @@ -13,10 +13,10 @@ from buildbot.reporters import utils from buildbot.reporters.base import ReporterBase from buildbot.reporters.generators.utils import BuildStatusGeneratorMixin -from buildbot.reporters.message import MessageFormatterRenderable, MessageFormatterBase +from buildbot.reporters.message import MessageFormatterBase, MessageFormatterRenderable from buildbot.reporters.utils import getDetailsForBuild from twisted.logger import Logger -from zope.interface import implementer # type: ignore[import] +from zope.interface import implementer # type: ignore[import] log = Logger() @@ -109,10 +109,11 @@ def __init__( "all", tags, builders, schedulers, branches, None, add_logs, add_patch ) - self.start_formatter = start_formatter or MessageFormatterRenderable("Build started.") + self.start_formatter = start_formatter or MessageFormatterRenderable( + "Build started." + ) self.end_formatter = end_formatter or MessageFormatterRenderable("Build done.") - # TODO: copy pasted from buildbot, make it static upstream and reuse @staticmethod async def partial_build_dict( @@ -164,7 +165,7 @@ async def generate( master: BuildMaster, reporter: ReporterBase, key: tuple[str, None | Any, str], - data: dict[str, Any], # TODO database types + data: dict[str, Any], # TODO database types ) -> None | dict[str, Any]: what, _, event = key if what == "builds": diff --git a/buildbot_nix/repo_config/__init__.py b/buildbot_nix/repo_config/__init__.py index fb9c3afb6..9c39fb40f 100644 --- a/buildbot_nix/repo_config/__init__.py +++ b/buildbot_nix/repo_config/__init__.py @@ -2,14 +2,15 @@ from tomllib import TOMLDecodeError from typing import TYPE_CHECKING, Self -from buildbot.process.buildstep import ShellMixin -from buildbot.process.buildstep import BuildStep +from buildbot.process.buildstep import BuildStep, ShellMixin +from pydantic import BaseModel, ValidationError if TYPE_CHECKING: from buildbot.process.log import StreamLog - class BuildStepShellMixin(BuildStep, ShellMixin): - pass -from pydantic import BaseModel, ValidationError + + +class BuildStepShellMixin(BuildStep, ShellMixin): + pass class RepoConfig(BaseModel): diff --git a/nix/treefmt/flake-module.nix b/nix/treefmt/flake-module.nix index d69377741..64b02a215 100644 --- a/nix/treefmt/flake-module.nix +++ b/nix/treefmt/flake-module.nix @@ -2,7 +2,7 @@ { imports = [ inputs.treefmt-nix.flakeModule ]; perSystem = - { pkgs, ... }: + { pkgs, lib, ... }: { treefmt = { projectRootFile = "LICENSE.md"; @@ -37,7 +37,9 @@ # the mypy module adds `./buildbot_nix/**/*.py` which does not appear to work # furthermore, saying `directories.""` will lead to `/buildbot_nix/**/*.py` which # is obviously incorrect... - settings.formatter."mypy-.".includes = [ "buildbot_nix/**/*.py" ]; + settings.formatter."mypy-." = lib.mkIf pkgs.stdenv.buildPlatform.isLinux { + includes = [ "buildbot_nix/**/*.py" ]; + }; settings.formatter.ruff-check.priority = 1; settings.formatter.ruff-format.priority = 2; }; From 9ff96b4bec517312ec6b10956e2643516879f0f4 Mon Sep 17 00:00:00 2001 From: magic_rb Date: Sun, 10 Nov 2024 20:54:48 +0100 Subject: [PATCH 19/23] `.gitignore` also `result-*` Signed-off-by: magic_rb --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8b4a66117..d36e815db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ result +result-* # Byte-compiled / optimized / DLL files __pycache__/ From 6493c358937bc7d81b34dfd9013a1faa38816625 Mon Sep 17 00:00:00 2001 From: Mergify <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:32:43 +0000 Subject: [PATCH 20/23] ci(mergify): upgrade configuration to current format --- .mergify.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 14c508b2d..d347c9642 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,15 +1,14 @@ queue_rules: - name: default + queue_conditions: + - base=main + - label~=merge-queue|dependencies merge_conditions: - check-success=buildbot/nix-build -defaults: - actions: - queue: - merge_method: rebase + merge_method: rebase + pull_request_rules: - - name: merge using the merge queue - conditions: - - base=main - - label~=merge-queue|dependencies + - name: refactored queue action rule + conditions: [] actions: - queue: {} + queue: From 3dab9cd7c99ce4a6db2a27483cd204d9e23b9334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 25 Nov 2024 18:18:43 +0100 Subject: [PATCH 21/23] add hercules-ci effects to ci --- README.md | 5 +++++ flake.nix | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d95c2131a..02a0c3f75 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,11 @@ integrate run a systemd service as described in service watches for changes in the local buildbot-nix store and uploads the contents to the attic cache. +## (experimental) Hercules CI effects + +See [flake.nix](flake.nix) and +[https://docs.hercules-ci.com/hercules-ci/effects/] for documentation. + ## Real-World Deployments See Buildbot-nix in action in these deployments: diff --git a/flake.nix b/flake.nix index 57e717ce5..ca903bbd6 100644 --- a/flake.nix +++ b/flake.nix @@ -15,7 +15,12 @@ outputs = inputs@{ self, flake-parts, ... }: flake-parts.lib.mkFlake { inherit inputs; } ( - { lib, ... }: + { + lib, + config, + withSystem, + ... + }: { imports = [ ./nix/checks/flake-module.nix @@ -37,6 +42,21 @@ } ) ]; + + herculesCI = herculesCI: { + onPush.default.outputs.effects.deploy = withSystem config.defaultEffectSystem ( + { pkgs, hci-effects, ... }: + hci-effects.runIf (herculesCI.config.repo.branch == "main") ( + hci-effects.mkEffect { + effectScript = '' + echo "${builtins.toJSON { inherit (herculesCI.config.repo) branch tag rev; }}" + ${pkgs.hello}/bin/hello + ''; + } + ) + ); + }; + nixosModules.buildbot-worker.imports = [ ./nix/worker.nix ( From 124253dace946da51293bc6e701f49164cb2d855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 25 Nov 2024 22:18:54 +0100 Subject: [PATCH 22/23] fix mypy errors --- buildbot_nix/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 945df7cb6..4964e3adf 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -765,11 +765,10 @@ def __init__(self, project: GitProject, **kwargs: Any) -> None: self.observer = logobserver.BufferLogObserver() self.addLogObserver("stdio", self.observer) - @defer.inlineCallbacks - def run(self) -> Generator[Any, object, Any]: + async def run(self) -> int: # run nix-eval-jobs --flake .#checks to generate the dict of stages - cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() - yield self.runCommand(cmd) + cmd: remotecommand.RemoteCommand = await self.makeRemoteShellCommand() + await self.runCommand(cmd) # if the command passes extract the list of stages result = cmd.results() @@ -1351,7 +1350,7 @@ def nix_register_gcroot_config( def buildbot_effects_config( - project: Project, git_url: str, worker_names: list[str], secrets: str | None + project: GitProject, git_url: str, worker_names: list[str], secrets: str | None ) -> util.BuilderConfig: """Builds one nix flake attribute.""" factory = util.BuildFactory() From 7674b57533fa1e098ba95b4eabfc2744620827fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 25 Nov 2024 22:21:06 +0100 Subject: [PATCH 23/23] only include buildbot-effects on Linux --- flake.nix | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index ca903bbd6..3414b7aec 100644 --- a/flake.nix +++ b/flake.nix @@ -96,15 +96,20 @@ ... }: { - packages.default = pkgs.mkShell { - packages = [ - pkgs.bashInteractive - pkgs.mypy - pkgs.ruff - ]; - }; - packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { }; - packages.buildbot-effects = pkgs.python3.pkgs.callPackage ./nix/buildbot-effects.nix { }; + packages = + { + default = pkgs.mkShell { + packages = [ + pkgs.bashInteractive + pkgs.mypy + pkgs.ruff + ]; + }; + buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { }; + } + // lib.optionalAttrs pkgs.stdenv.isLinux { + buildbot-effects = pkgs.python3.pkgs.callPackage ./nix/buildbot-effects.nix { }; + }; checks = let nixosMachines = lib.mapAttrs' (