fix+feat: push to feedstocks and handle tmpdir properly (#3278)

* doc: update runner allocation * feat+fix: push to feedstocks and fix tmpdir handling
regro · Dec 4, 2024 · 74f1293 · 74f1293
1 parent 0c6bcdc
commit 74f1293
Show file tree

Hide file tree

Showing 3 changed files with 133 additions and 81 deletions.
diff --git a/conda_forge_tick/events/pr_events.py b/conda_forge_tick/events/pr_events.py
@@ -11,6 +11,7 @@
     lazy_json_override_backends,
     push_lazy_json_via_gh_api,
 )
+from conda_forge_tick.os_utils import pushd
 
 
 def _react_to_pr(uid: str, dry_run: bool = False) -> None:
@@ -19,45 +20,48 @@ def _react_to_pr(uid: str, dry_run: bool = False) -> None:
     with lazy_json_override_backends(["github"], use_file_cache=False):
         pr_data = copy.deepcopy(LazyJson(f"pr_json/{uid}.json").data)
 
-    with tempfile.TemporaryDirectory():
-        with lazy_json_override_backends(["file"]):
-            pr_json = LazyJson(f"pr_json/{uid}.json")
-            with pr_json:
-                pr_json.update(pr_data)
+    with (
+        tempfile.TemporaryDirectory() as tmpdir,
+        pushd(str(tmpdir)),
+        lazy_json_override_backends(["file"]),
+    ):
+        pr_json = LazyJson(f"pr_json/{uid}.json")
+        with pr_json:
+            pr_json.update(pr_data)
 
-            with pr_json:
-                pr_data = refresh_pr(pr_json, dry_run=dry_run)
-                if not dry_run and pr_data is not None and pr_data != pr_json.data:
-                    print("refreshed PR data", flush=True)
-                    updated_pr = True
-                    pr_json.update(pr_data)
+        with pr_json:
+            pr_data = refresh_pr(pr_json, dry_run=dry_run)
+            if not dry_run and pr_data is not None and pr_data != pr_json.data:
+                print("refreshed PR data", flush=True)
+                updated_pr = True
+                pr_json.update(pr_data)
 
-            with pr_json:
-                pr_data = close_out_labels(pr_json, dry_run=dry_run)
-                if not dry_run and pr_data is not None and pr_data != pr_json.data:
-                    print("closed PR due to bot-rerun label", flush=True)
-                    updated_pr = True
-                    pr_json.update(pr_data)
+        with pr_json:
+            pr_data = close_out_labels(pr_json, dry_run=dry_run)
+            if not dry_run and pr_data is not None and pr_data != pr_json.data:
+                print("closed PR due to bot-rerun label", flush=True)
+                updated_pr = True
+                pr_json.update(pr_data)
 
-            with pr_json:
-                pr_data = refresh_pr(pr_json, dry_run=dry_run)
-                if not dry_run and pr_data is not None and pr_data != pr_json.data:
-                    print("refreshed PR data", flush=True)
-                    updated_pr = True
-                    pr_json.update(pr_data)
+        with pr_json:
+            pr_data = refresh_pr(pr_json, dry_run=dry_run)
+            if not dry_run and pr_data is not None and pr_data != pr_json.data:
+                print("refreshed PR data", flush=True)
+                updated_pr = True
+                pr_json.update(pr_data)
 
-            with pr_json:
-                pr_data = close_out_dirty_prs(pr_json, dry_run=dry_run)
-                if not dry_run and pr_data is not None and pr_data != pr_json.data:
-                    print("closed PR due to merge conflicts", flush=True)
-                    updated_pr = True
-                    pr_json.update(pr_data)
+        with pr_json:
+            pr_data = close_out_dirty_prs(pr_json, dry_run=dry_run)
+            if not dry_run and pr_data is not None and pr_data != pr_json.data:
+                print("closed PR due to merge conflicts", flush=True)
+                updated_pr = True
+                pr_json.update(pr_data)
 
-            if not dry_run and updated_pr:
-                push_lazy_json_via_gh_api(pr_json)
-                print("pushed PR update", flush=True)
-            else:
-                print("no changes to push", flush=True)
+        if not dry_run and updated_pr:
+            push_lazy_json_via_gh_api(pr_json)
+            print("pushed PR update", flush=True)
+        else:
+            print("no changes to push", flush=True)
 
 
 def react_to_pr(uid: str, dry_run: bool = False) -> None:

diff --git a/conda_forge_tick/events/push_events.py b/conda_forge_tick/events/push_events.py
@@ -4,6 +4,7 @@
 import networkx as nx
 import requests
 
+from conda_forge_tick.git_utils import github_client
 from conda_forge_tick.lazy_json_backends import (
     LazyJson,
     lazy_json_override_backends,
@@ -14,20 +15,61 @@
     _migrate_schema,
     try_load_feedstock,
 )
+from conda_forge_tick.os_utils import pushd
 
 
-def _get_archived_feedstocks():
+def _get_feedstocks():
     r = requests.get(
         "https://raw.githubusercontent.com/regro/cf-graph-countyfair/refs/heads/master/all_feedstocks.json"
     )
     r.raise_for_status()
-    return r.json()["archived"]
+    return r.json()
+
+
+def _get_archived_feedstocks():
+    return _get_feedstocks()["archived"]
+
+
+def _update_feedstocks(name: str) -> None:
+    feedstocks = copy.deepcopy(_get_feedstocks())
+
+    gh = github_client()
+    repo = gh.get_repo(f"conda-forge/{name}-feedstock")
+    changed = False
+    if repo.archived:
+        if name in feedstocks["active"]:
+            feedstocks["active"].remove(name)
+            changed = True
+        if name not in feedstocks["archived"]:
+            feedstocks["archived"].append(name)
+            changed = True
+    elif not repo.archived:
+        if name in feedstocks["archived"]:
+            feedstocks["archived"].remove(name)
+            changed = True
+        if name not in feedstocks["active"]:
+            feedstocks["active"].append(name)
+            changed = True
+
+    if changed:
+        with (
+            tempfile.TemporaryDirectory() as tmpdir,
+            pushd(str(tmpdir)),
+            lazy_json_override_backends(["file"]),
+        ):
+            with LazyJson("all_feedstocks.json") as f:
+                f.update(feedstocks)
+            push_lazy_json_via_gh_api(f)
 
 
 def _react_to_push(name: str, dry_run: bool = False) -> None:
     updated_node = False
     fname = f"node_attrs/{name}.json"
 
+    # first update the feedstocks
+    _update_feedstocks(name)
+
+    # now pull down the data
     with lazy_json_override_backends(["github"], use_file_cache=False):
         try:
             attrs_data = copy.deepcopy(LazyJson(fname).data)
@@ -39,49 +81,53 @@ def _react_to_push(name: str, dry_run: bool = False) -> None:
         graph_data = copy.deepcopy(LazyJson("graph.json").data)
         gx = nx.node_link_graph(graph_data, edges="links")
 
-    with tempfile.TemporaryDirectory():
-        with lazy_json_override_backends(["file"]):
-            attrs = LazyJson(fname)
-            with attrs:
-                attrs.update(attrs_data)
-
-            with attrs:
-                data_before = copy.deepcopy(attrs.data)
-
-                if not dry_run:
-                    try_load_feedstock(name, attrs, mark_not_archived=True)
-                else:
-                    print("dry run - loading feedstock", flush=True)
-
-                if not dry_run:
-                    _add_run_exports_per_node(
-                        attrs,
-                        gx.graph["outputs_lut"],
-                        gx.graph["strong_exports"],
-                    )
-                else:
-                    print("dry run - adding run exports", flush=True)
-
-                if not dry_run:
-                    _migrate_schema(name, attrs)
-                else:
-                    print("dry run - migrating schema", flush=True)
-
-                if not dry_run:
-                    archived_names = _get_archived_feedstocks()
-                    if name in archived_names:
-                        attrs["archived"] = True
-                else:
-                    print("dry run - checking archived", flush=True)
-
-                if not dry_run and data_before != attrs.data:
-                    updated_node = True
-
-            if not dry_run and updated_node:
-                push_lazy_json_via_gh_api(attrs)
-                print("pushed node update", flush=True)
+    # now update the node
+    with (
+        tempfile.TemporaryDirectory() as tmpdir,
+        pushd(str(tmpdir)),
+        lazy_json_override_backends(["file"]),
+    ):
+        attrs = LazyJson(fname)
+        with attrs:
+            attrs.update(attrs_data)
+
+        with attrs:
+            data_before = copy.deepcopy(attrs.data)
+
+            if not dry_run:
+                try_load_feedstock(name, attrs, mark_not_archived=True)
+            else:
+                print("dry run - loading feedstock", flush=True)
+
+            if not dry_run:
+                _add_run_exports_per_node(
+                    attrs,
+                    gx.graph["outputs_lut"],
+                    gx.graph["strong_exports"],
+                )
+            else:
+                print("dry run - adding run exports", flush=True)
+
+            if not dry_run:
+                _migrate_schema(name, attrs)
+            else:
+                print("dry run - migrating schema", flush=True)
+
+            if not dry_run:
+                archived_names = _get_archived_feedstocks()
+                if name in archived_names:
+                    attrs["archived"] = True
             else:
-                print("no changes to push", flush=True)
+                print("dry run - checking archived", flush=True)
+
+            if not dry_run and data_before != attrs.data:
+                updated_node = True
+
+        if not dry_run and updated_node:
+            push_lazy_json_via_gh_api(attrs)
+            print("pushed node update", flush=True)
+        else:
+            print("no changes to push", flush=True)
 
 
 def react_to_push(uid: str, dry_run: bool = False) -> None:

diff --git a/docs/runner_allocation.md b/docs/runner_allocation.md
@@ -1,19 +1,21 @@
 # GitHub Runner Allocation
 
-Last Updated: 2024-07-29
+Last Updated: 2024-12-03
 
 We have a
 [concurrency of 20 runners available](https://docs.github.com/en/actions/administering-github-actions/usage-limits-billing-and-administration#usage-limits) in the entire `regro` account.
 
 These are split across our workflows as follows:
 - `bot-bot` - 1 runner
 - `bot-cache` - currently disabled
+- `bot-events` (on demand, responds to GitHub webhook events) - any available runner
 - `bot-feedstocks` (daily for ~30 minutes) - 1 runner
-- `bot-make-graph` - 3 runners
+- `bot-make-graph` - 1 runner
 - `bot-make-migrators` - 1 runner
-- `bot-prs` - 4 runners
+- `bot-prs` (daily for ~30 minutes) - 4 runners
 - `bot-pypi-mapping` (hourly for ~5 minutes) - 1 runner
 - `bot-update-status-page` - 1 runner
+- `bot-update-nodes` (4x daily for ~30 minutes) - 6 runners
 - `bot-versions` - 6 runners
 - `docker` (on demand) - 1 runner
 - `keepalive` (hourly for ~5 minutes) - 1 runner