Integrate effects

.gitignore

@@ -1,4 +1,5 @@
 result
+result-*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.mergify.yml

@@ -1,15 +1,14 @@
 queue_rules:
   - name: default
+    queue_conditions:
+      - base=main
+      - label~=merge-queue|dependencies
     merge_conditions:
       - check-success=buildbot/nix-build
-defaults:
-  actions:
-    queue:
-      merge_method: rebase
+    merge_method: rebase
+
 pull_request_rules:
-  - name: merge using the merge queue
-    conditions:
-      - base=main
-      - label~=merge-queue|dependencies
+  - name: refactored queue action rule
+    conditions: []
     actions:
-      queue: {}
+      queue:

README.md

@@ -6,6 +6,21 @@ into the Nix ecosystem. This module is under active development, and while it's
 generally stable and widely used, please be aware that some APIs may change over
 time.
 
+## Features
+
+- Fast, Parallel evaluation using
+  [nix-eval-jobs](https://github.com/nix-community/nix-eval-jobs)
+- Github integration:
+  - Login with GitHub to control builds
+  - CI status notification in pull requests and on the default branch
+- All builds share the same nix store for speed
+- The last attribute of a build is protected from garbage collection
+- Build matrix based on `.#checks` attributes
+- No arbitrary code runs outside of the Nix sandbox
+- _experimental_
+  [hercules-ci effect](https://docs.hercules-ci.com/hercules-ci-effects/) to run
+  impure CI steps i.e. deploying NixOS
+
 ## Getting Started with Buildbot Setup
 
 To set up Buildbot using Buildbot-nix, you can start by exploring the provided
@@ -238,6 +253,11 @@ integrate run a systemd service as described in
 service watches for changes in the local buildbot-nix store and uploads the
 contents to the attic cache.
 
+## (experimental) Hercules CI effects
+
+See [flake.nix](flake.nix) and
+[https://docs.hercules-ci.com/hercules-ci/effects/] for documentation.
+
 ## Real-World Deployments
 
 See Buildbot-nix in action in these deployments:

buildbot_effects/__init__.py

@@ -96,25 +96,34 @@ def effect_function(opts: EffectsOptions) -> str:
     rev = args["rev"]
     escaped_args = json.dumps(json.dumps(args))
     url = json.dumps(f"git+file://{opts.path}?rev={rev}#")
-    return f"""(((builtins.getFlake {url}).outputs.herculesCI (builtins.fromJSON {escaped_args})).onPush.default.outputs.hci-effects)"""
+    return f"""
+      let
+        flake = builtins.getFlake {url};
+        effects = flake.outputs.herculesCI (builtins.fromJSON {escaped_args});
+      in
+        if flake.outputs ? herculesCI then
+          effects.onPush.default.outputs.effects or {{}}
+        else
+          {{}}
+    """
 
 
 def list_effects(opts: EffectsOptions) -> list[str]:
     cmd = nix_command(
         "eval",
         "--json",
         "--expr",
-        f"builtins.attrNames {effect_function(opts)}",
+        f"builtins.attrNames ({effect_function(opts)})",
     )
     proc = run(cmd, stdout=subprocess.PIPE)
     return json.loads(proc.stdout)
 
 
-def instantiate_effects(opts: EffectsOptions) -> str:
+def instantiate_effects(effect: str, opts: EffectsOptions) -> str:
     cmd = [
         "nix-instantiate",
         "--expr",
-        f"{effect_function(opts)}.deploy.run",
+        f"(({effect_function(opts)}).{effect}).run or []",
     ]
     proc = run(cmd, stdout=subprocess.PIPE)
     return proc.stdout.rstrip()
@@ -133,12 +142,15 @@ def parse_derivation(path: str) -> dict[str, Any]:
     return json.loads(proc.stdout)
 
 
-def env_args(env: dict[str, str]) -> list[str]:
+def env_args(env: dict[str, str], clear_env: set[str]) -> list[str]:
     result = []
     for k, v in env.items():
         result.append("--setenv")
         result.append(f"{k}")
         result.append(f"{v}")
+    for k in clear_env:
+        result.append("--unsetenv")
+        result.append(f"{k}")
     return result
 
 
@@ -171,6 +183,11 @@ def run_effects(
     env["IN_HERCULES_CI_EFFECT"] = "true"
     env["HERCULES_CI_SECRETS_JSON"] = "/run/secrets.json"
     env["NIX_BUILD_TOP"] = "/build"
+    env["TMPDIR"] = "/tmp"  # noqa: S108
+    clear_env = set()
+    clear_env.add("TMP")
+    clear_env.add("TEMP")
+    clear_env.add("TEMPDIR")
     bwrap = shutil.which("bwrap")
     if bwrap is None:
         msg = "bwrap' executable not found"
@@ -214,14 +231,15 @@ def run_effects(
         secrets = secrets.copy()
         secrets["hercules-ci"] = {"data": {"token": "dummy"}}
         tmp.write(json.dumps(secrets).encode())
+        tmp.flush()
         bubblewrap_cmd.extend(
             [
                 "--ro-bind",
                 tmp.name,
                 "/run/secrets.json",
             ],
         )
-        bubblewrap_cmd.extend(env_args(env))
+        bubblewrap_cmd.extend(env_args(env, clear_env))
         bubblewrap_cmd.append("--")
         bubblewrap_cmd.extend(sandboxed_cmd)
         with pipe() as (r_file, w_file):

buildbot_effects/cli.py

@@ -1,30 +1,34 @@
 import argparse
 import json
-from collections.abc import Callable
+import sys
 from pathlib import Path
 
 from . import instantiate_effects, list_effects, parse_derivation, run_effects
 from .options import EffectsOptions
 
 
-def list_command(options: EffectsOptions) -> None:
-    print(list_effects(options))
+def list_command(args: argparse.Namespace, options: EffectsOptions) -> None:
+    json.dump(list_effects(options), fp=sys.stdout, indent=2)
 
 
-def run_command(options: EffectsOptions) -> None:
-    drv_path = instantiate_effects(options)
+def run_command(args: argparse.Namespace, options: EffectsOptions) -> None:
+    effect = args.effect
+    drv_path = instantiate_effects(effect, options)
+    if drv_path == "":
+        print(f"Effect {effect} not found or not runnable for {options}")
+        return
     drvs = parse_derivation(drv_path)
     drv = next(iter(drvs.values()))
 
     secrets = json.loads(options.secrets.read_text()) if options.secrets else {}
     run_effects(drv_path, drv, secrets=secrets)
 
 
-def run_all_command(options: EffectsOptions) -> None:
+def run_all_command(args: argparse.Namespace, options: EffectsOptions) -> None:
     print("TODO")
 
 
-def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]:
+def parse_args() -> tuple[argparse.Namespace, EffectsOptions]:
     parser = argparse.ArgumentParser(description="Run effects from a hercules-ci flake")
     parser.add_argument(
         "--secrets",
@@ -48,7 +52,8 @@ def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]:
     )
     parser.add_argument(
         "--path",
-        type=str,
+        type=Path,
+        default=Path(),
         help="Path to the repository",
     )
     subparser = parser.add_subparsers(
@@ -77,9 +82,16 @@ def parse_args() -> tuple[Callable[[EffectsOptions], None], EffectsOptions]:
     run_all_parser.set_defaults(command=run_all_command)
 
     args = parser.parse_args()
-    return args.command, EffectsOptions(secrets=args.secrets)
+    opts = EffectsOptions(
+        secrets=args.secrets,
+        branch=args.branch,
+        rev=args.rev,
+        repo=args.repo,
+        path=args.path.resolve(),
+    )
+    return args, opts
 
 
 def main() -> None:
-    command, options = parse_args()
-    command(options)
+    args, options = parse_args()
+    args.command(args, options)