From fea8ae9f9ffdabb15d81cbca5636f7ab82fa419c Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 25 Nov 2022 13:36:04 +0000 Subject: [PATCH 1/2] Enable managing RTD redirects in-tree This is designed as a script and a data file (in YAML format), and meant to manage the RTD redirects with a version controlled file. This makes it possible for pull requests to this repository to update the redirects for this project's documentation (eg: for better error urls) and for this evolution to be tracked as a part of version control history. --- .pre-commit-config.yaml | 1 + .readthedocs-custom-redirects.yml | 15 +++ MANIFEST.in | 1 + tools/update-rtd-redirects.py | 155 ++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 .readthedocs-custom-redirects.yml create mode 100644 tools/update-rtd-redirects.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20a85438c5a..a49016eed3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,7 @@ repos: 'types-setuptools==57.4.14', 'types-freezegun==1.1.9', 'types-six==1.16.15', + 'types-pyyaml==6.0.12.2', ] - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/.readthedocs-custom-redirects.yml b/.readthedocs-custom-redirects.yml new file mode 100644 index 00000000000..46c1f819c88 --- /dev/null +++ b/.readthedocs-custom-redirects.yml @@ -0,0 +1,15 @@ +# This file is read by tools/update-rtd-redirects.py. +# It is related to Read the Docs, but is not a file processed by the platform. + +/dev/news-entry-failure: >- + https://pip.pypa.io/en/stable/development/contributing/#news-entries +/errors/resolution-impossible: >- + https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts +/surveys/backtracking: >- + https://forms.gle/LkZP95S4CfqBAU1N6 +/warnings/backtracking: >- + https://pip.pypa.io/en/stable/topics/dependency-resolution/#possible-ways-to-reduce-backtracking +/warnings/enable-long-paths: >- + https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later +/warnings/venv: >- + https://docs.python.org/3/tutorial/venv.html diff --git a/MANIFEST.in b/MANIFEST.in index ff3825f65c2..e0fba8222af 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -18,6 +18,7 @@ exclude .mailmap exclude .appveyor.yml exclude .readthedocs.yml exclude .pre-commit-config.yaml +exclude .readthedocs-custom-redirects.yml exclude tox.ini exclude noxfile.py diff --git a/tools/update-rtd-redirects.py b/tools/update-rtd-redirects.py new file mode 100644 index 00000000000..8515c026cb7 --- /dev/null +++ b/tools/update-rtd-redirects.py @@ -0,0 +1,155 @@ +"""Update the 'exact' redirects on Read the Docs to match an in-tree file's contents. + +Relevant API reference: https://docs.readthedocs.io/en/stable/api/v3.html#redirects +""" +import operator +import os +import sys +from pathlib import Path + +import httpx +import rich +import yaml + +try: + _TOKEN = os.environ["RTD_API_TOKEN"] +except KeyError: + rich.print( + "[bold]error[/]: [red]No API token provided. Please set `RTD_API_TOKEN`.[/]", + file=sys.stderr, + ) + sys.exit(1) + +RTD_API_HEADERS = {"Authorization": f"token {_TOKEN}"} +RTD_API_BASE_URL = "https://readthedocs.org/api/v3/projects/pip/" +REPO_ROOT = Path(__file__).resolve().parent.parent + + +# -------------------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------------------- +def next_step(msg: str) -> None: + rich.print(f"> [blue]{msg}[/]") + + +def log_response(response: httpx.Response) -> None: + request = response.request + rich.print(f"[bold magenta]{request.method}[/] {request.url} -> {response}") + + +def get_rtd_api() -> httpx.Client: + return httpx.Client( + headers=RTD_API_HEADERS, + base_url=RTD_API_BASE_URL, + event_hooks={"response": [log_response]}, + ) + + +# -------------------------------------------------------------------------------------- +# Actual logic +# -------------------------------------------------------------------------------------- +next_step("Loading local redirects from the yaml file.") + +with open(REPO_ROOT / ".readthedocs-custom-redirects.yml") as f: + local_redirects = yaml.safe_load(f) + +rich.print("Loaded local redirects!") +for src, dst in sorted(local_redirects.items()): + rich.print(f" [yellow]{src}[/] --> {dst}") +rich.print(f"{len(local_redirects)} entries.") + + +next_step("Fetch redirects configured on RTD.") + +with get_rtd_api() as rtd_api: + response = rtd_api.get("redirects/") + response.raise_for_status() + + rtd_redirects = response.json() + +for redirect in sorted( + rtd_redirects["results"], key=operator.itemgetter("type", "from_url", "to_url") +): + if redirect["type"] != "exact": + rich.print(f" [magenta]{redirect['type']}[/]") + continue + + pk = redirect["pk"] + src = redirect["from_url"] + dst = redirect["to_url"] + rich.print(f" [yellow]{src}[/] -({pk:^5})-> {dst}") + +rich.print(f"{rtd_redirects['count']} entries.") + + +next_step("Compare and determine modifications.") + +redirects_to_remove: list[int] = [] +redirects_to_add: dict[str, str] = {} + +for redirect in rtd_redirects["results"]: + if redirect["type"] != "exact": + continue + + rtd_src = redirect["from_url"] + rtd_dst = redirect["to_url"] + redirect_id = redirect["pk"] + + if rtd_src not in local_redirects: + redirects_to_remove.append(redirect_id) + continue + + local_dst = local_redirects[rtd_src] + if local_dst != rtd_dst: + redirects_to_remove.append(redirect_id) + redirects_to_add[rtd_src] = local_dst + + del local_redirects[rtd_src] + +for src, dst in sorted(local_redirects.items()): + redirects_to_add[src] = dst + del local_redirects[src] + +assert not local_redirects + +if not redirects_to_remove: + rich.print("Nothing to remove.") +else: + rich.print(f"To remove: ({len(redirects_to_remove)} entries)") + for redirect_id in redirects_to_remove: + rich.print(" ", redirect_id) + +if not redirects_to_add: + rich.print("Nothing to add.") +else: + rich.print(f"To add: ({len(redirects_to_add)} entries)") + for src, dst in redirects_to_add.items(): + rich.print(f" {src} --> {dst}") + + +next_step("Update the RTD redirects.") + +if not (redirects_to_add or redirects_to_remove): + rich.print("[green]Nothing to do![/]") + sys.exit(0) + +exit_code = 0 +with get_rtd_api() as rtd_api: + for redirect_id in redirects_to_remove: + response = rtd_api.delete(f"redirects/{redirect_id}/") + response.raise_for_status() + if response.status_code != 204: + rich.print("[red]This might not have been removed correctly.[/]") + exit_code = 1 + + for src, dst in redirects_to_add.items(): + response = rtd_api.post( + "redirects/", + json={"from_url": src, "to_url": dst, "type": "exact"}, + ) + response.raise_for_status() + if response.status_code != 201: + rich.print("[red]This might not have been added correctly.[/]") + exit_code = 1 + +sys.exit(exit_code) From 8328135d934d9136756639606632ab70625242ae Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 25 Nov 2022 13:47:05 +0000 Subject: [PATCH 2/2] Add GitHub action for RTD redirect updates This makes it possible for pip's documentation's redirects to be automatically synchronised with the `main` branch. --- .github/workflows/update-rtd-redirects.yml | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/update-rtd-redirects.yml diff --git a/.github/workflows/update-rtd-redirects.yml b/.github/workflows/update-rtd-redirects.yml new file mode 100644 index 00000000000..5ac9c63130e --- /dev/null +++ b/.github/workflows/update-rtd-redirects.yml @@ -0,0 +1,27 @@ +name: Update documentation redirects + +on: + push: + branches: [main] + schedule: + - cron: 0 0 * * MON # Run every Monday at 00:00 UTC + +env: + FORCE_COLOR: "1" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + update-rtd-redirects: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - run: pip install httpx requests pyyaml + - run: python tools/update-rtd-redirects.py + env: + RTD_API_TOKEN: ${{ secrets.RTD_API_TOKEN }}