diff --git a/conda_forge_tick/events/pr_events.py b/conda_forge_tick/events/pr_events.py index d500ab37a..f4588697a 100644 --- a/conda_forge_tick/events/pr_events.py +++ b/conda_forge_tick/events/pr_events.py @@ -11,6 +11,7 @@ lazy_json_override_backends, push_lazy_json_via_gh_api, ) +from conda_forge_tick.os_utils import pushd def _react_to_pr(uid: str, dry_run: bool = False) -> None: @@ -19,45 +20,48 @@ def _react_to_pr(uid: str, dry_run: bool = False) -> None: with lazy_json_override_backends(["github"], use_file_cache=False): pr_data = copy.deepcopy(LazyJson(f"pr_json/{uid}.json").data) - with tempfile.TemporaryDirectory(): - with lazy_json_override_backends(["file"]): - pr_json = LazyJson(f"pr_json/{uid}.json") - with pr_json: - pr_json.update(pr_data) + with ( + tempfile.TemporaryDirectory() as tmpdir, + pushd(str(tmpdir)), + lazy_json_override_backends(["file"]), + ): + pr_json = LazyJson(f"pr_json/{uid}.json") + with pr_json: + pr_json.update(pr_data) - with pr_json: - pr_data = refresh_pr(pr_json, dry_run=dry_run) - if not dry_run and pr_data is not None and pr_data != pr_json.data: - print("refreshed PR data", flush=True) - updated_pr = True - pr_json.update(pr_data) + with pr_json: + pr_data = refresh_pr(pr_json, dry_run=dry_run) + if not dry_run and pr_data is not None and pr_data != pr_json.data: + print("refreshed PR data", flush=True) + updated_pr = True + pr_json.update(pr_data) - with pr_json: - pr_data = close_out_labels(pr_json, dry_run=dry_run) - if not dry_run and pr_data is not None and pr_data != pr_json.data: - print("closed PR due to bot-rerun label", flush=True) - updated_pr = True - pr_json.update(pr_data) + with pr_json: + pr_data = close_out_labels(pr_json, dry_run=dry_run) + if not dry_run and pr_data is not None and pr_data != pr_json.data: + print("closed PR due to bot-rerun label", flush=True) + updated_pr = True + pr_json.update(pr_data) - with pr_json: - pr_data = refresh_pr(pr_json, dry_run=dry_run) - if not dry_run and pr_data is not None and pr_data != pr_json.data: - print("refreshed PR data", flush=True) - updated_pr = True - pr_json.update(pr_data) + with pr_json: + pr_data = refresh_pr(pr_json, dry_run=dry_run) + if not dry_run and pr_data is not None and pr_data != pr_json.data: + print("refreshed PR data", flush=True) + updated_pr = True + pr_json.update(pr_data) - with pr_json: - pr_data = close_out_dirty_prs(pr_json, dry_run=dry_run) - if not dry_run and pr_data is not None and pr_data != pr_json.data: - print("closed PR due to merge conflicts", flush=True) - updated_pr = True - pr_json.update(pr_data) + with pr_json: + pr_data = close_out_dirty_prs(pr_json, dry_run=dry_run) + if not dry_run and pr_data is not None and pr_data != pr_json.data: + print("closed PR due to merge conflicts", flush=True) + updated_pr = True + pr_json.update(pr_data) - if not dry_run and updated_pr: - push_lazy_json_via_gh_api(pr_json) - print("pushed PR update", flush=True) - else: - print("no changes to push", flush=True) + if not dry_run and updated_pr: + push_lazy_json_via_gh_api(pr_json) + print("pushed PR update", flush=True) + else: + print("no changes to push", flush=True) def react_to_pr(uid: str, dry_run: bool = False) -> None: diff --git a/conda_forge_tick/events/push_events.py b/conda_forge_tick/events/push_events.py index e42df0c6e..8e6d60036 100644 --- a/conda_forge_tick/events/push_events.py +++ b/conda_forge_tick/events/push_events.py @@ -4,6 +4,7 @@ import networkx as nx import requests +from conda_forge_tick.git_utils import github_client from conda_forge_tick.lazy_json_backends import ( LazyJson, lazy_json_override_backends, @@ -14,20 +15,61 @@ _migrate_schema, try_load_feedstock, ) +from conda_forge_tick.os_utils import pushd -def _get_archived_feedstocks(): +def _get_feedstocks(): r = requests.get( "https://raw.githubusercontent.com/regro/cf-graph-countyfair/refs/heads/master/all_feedstocks.json" ) r.raise_for_status() - return r.json()["archived"] + return r.json() + + +def _get_archived_feedstocks(): + return _get_feedstocks()["archived"] + + +def _update_feedstocks(name: str) -> None: + feedstocks = copy.deepcopy(_get_feedstocks()) + + gh = github_client() + repo = gh.get_repo(f"conda-forge/{name}-feedstock") + changed = False + if repo.archived: + if name in feedstocks["active"]: + feedstocks["active"].remove(name) + changed = True + if name not in feedstocks["archived"]: + feedstocks["archived"].append(name) + changed = True + elif not repo.archived: + if name in feedstocks["archived"]: + feedstocks["archived"].remove(name) + changed = True + if name not in feedstocks["active"]: + feedstocks["active"].append(name) + changed = True + + if changed: + with ( + tempfile.TemporaryDirectory() as tmpdir, + pushd(str(tmpdir)), + lazy_json_override_backends(["file"]), + ): + with LazyJson("all_feedstocks.json") as f: + f.update(feedstocks) + push_lazy_json_via_gh_api(f) def _react_to_push(name: str, dry_run: bool = False) -> None: updated_node = False fname = f"node_attrs/{name}.json" + # first update the feedstocks + _update_feedstocks(name) + + # now pull down the data with lazy_json_override_backends(["github"], use_file_cache=False): try: attrs_data = copy.deepcopy(LazyJson(fname).data) @@ -39,49 +81,53 @@ def _react_to_push(name: str, dry_run: bool = False) -> None: graph_data = copy.deepcopy(LazyJson("graph.json").data) gx = nx.node_link_graph(graph_data, edges="links") - with tempfile.TemporaryDirectory(): - with lazy_json_override_backends(["file"]): - attrs = LazyJson(fname) - with attrs: - attrs.update(attrs_data) - - with attrs: - data_before = copy.deepcopy(attrs.data) - - if not dry_run: - try_load_feedstock(name, attrs, mark_not_archived=True) - else: - print("dry run - loading feedstock", flush=True) - - if not dry_run: - _add_run_exports_per_node( - attrs, - gx.graph["outputs_lut"], - gx.graph["strong_exports"], - ) - else: - print("dry run - adding run exports", flush=True) - - if not dry_run: - _migrate_schema(name, attrs) - else: - print("dry run - migrating schema", flush=True) - - if not dry_run: - archived_names = _get_archived_feedstocks() - if name in archived_names: - attrs["archived"] = True - else: - print("dry run - checking archived", flush=True) - - if not dry_run and data_before != attrs.data: - updated_node = True - - if not dry_run and updated_node: - push_lazy_json_via_gh_api(attrs) - print("pushed node update", flush=True) + # now update the node + with ( + tempfile.TemporaryDirectory() as tmpdir, + pushd(str(tmpdir)), + lazy_json_override_backends(["file"]), + ): + attrs = LazyJson(fname) + with attrs: + attrs.update(attrs_data) + + with attrs: + data_before = copy.deepcopy(attrs.data) + + if not dry_run: + try_load_feedstock(name, attrs, mark_not_archived=True) + else: + print("dry run - loading feedstock", flush=True) + + if not dry_run: + _add_run_exports_per_node( + attrs, + gx.graph["outputs_lut"], + gx.graph["strong_exports"], + ) + else: + print("dry run - adding run exports", flush=True) + + if not dry_run: + _migrate_schema(name, attrs) + else: + print("dry run - migrating schema", flush=True) + + if not dry_run: + archived_names = _get_archived_feedstocks() + if name in archived_names: + attrs["archived"] = True else: - print("no changes to push", flush=True) + print("dry run - checking archived", flush=True) + + if not dry_run and data_before != attrs.data: + updated_node = True + + if not dry_run and updated_node: + push_lazy_json_via_gh_api(attrs) + print("pushed node update", flush=True) + else: + print("no changes to push", flush=True) def react_to_push(uid: str, dry_run: bool = False) -> None: diff --git a/docs/runner_allocation.md b/docs/runner_allocation.md index ef91ff9de..e8b476f7e 100644 --- a/docs/runner_allocation.md +++ b/docs/runner_allocation.md @@ -1,6 +1,6 @@ # GitHub Runner Allocation -Last Updated: 2024-07-29 +Last Updated: 2024-12-03 We have a [concurrency of 20 runners available](https://docs.github.com/en/actions/administering-github-actions/usage-limits-billing-and-administration#usage-limits) in the entire `regro` account. @@ -8,12 +8,14 @@ We have a These are split across our workflows as follows: - `bot-bot` - 1 runner - `bot-cache` - currently disabled +- `bot-events` (on demand, responds to GitHub webhook events) - any available runner - `bot-feedstocks` (daily for ~30 minutes) - 1 runner -- `bot-make-graph` - 3 runners +- `bot-make-graph` - 1 runner - `bot-make-migrators` - 1 runner -- `bot-prs` - 4 runners +- `bot-prs` (daily for ~30 minutes) - 4 runners - `bot-pypi-mapping` (hourly for ~5 minutes) - 1 runner - `bot-update-status-page` - 1 runner +- `bot-update-nodes` (4x daily for ~30 minutes) - 6 runners - `bot-versions` - 6 runners - `docker` (on demand) - 1 runner - `keepalive` (hourly for ~5 minutes) - 1 runner