From c01236baaac27d294a95f0dde9c44c3f8f3c3006 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 24 Jan 2025 10:52:41 +0100 Subject: [PATCH] Add `read-file` action (#203) * Add read-file * Use YYYY.MM cache hash * Correct inputs & outputs * Cleanup * Add default value for when file doesn't exist * Convert positionals to options * Invalidate cache based on workflows * Correct README * Improve caching * Use Python >=3.8 * Initialize test suite * Install all dependencies * Upload coverage * Add badges * Update README.md * Update test_combine_durations.py * Add test_read_file.py * Correct nested double quotes * Use Python >=3.9 * Stringify * Don't specify testpaths, allow pytest to autodetect * Add comment regarding hashing * Test GHA * Debug * Correct remote URL * Fix quoting * Setup python * Standardize Python * Enable dependabot updates for pip * Add analyze step with automated error reports * Minor cleanup * Update existing failures * Apply suggestions from code review * Typing * Update README.md * Apply suggestions from code review * Update .github/workflows/tests.yml --------- Co-authored-by: Jannis Leidel --- .github/workflows/tests.yml | 39 ++++++++- pyproject.toml | 7 +- read-file/README.md | 52 +++++++++++ read-file/action.yml | 59 +++++++++++++ read-file/data/json.json | 1 + read-file/data/yaml.yaml | 1 + read-file/read_file.py | 97 +++++++++++++++++++++ read-file/requirements.txt | 2 + read-file/test_read_file.py | 168 ++++++++++++++++++++++++++++++++++++ 9 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 read-file/README.md create mode 100644 read-file/action.yml create mode 100644 read-file/data/json.json create mode 100644 read-file/data/yaml.yaml create mode 100644 read-file/read_file.py create mode 100644 read-file/requirements.txt create mode 100644 read-file/test_read_file.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d6779d3..ebdee11 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,9 +47,46 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + read-file: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Read Remote JSON + id: json + uses: ./read-file + env: + # `full_name` is the org/repo slug, extracting from `head` in case PR was opened via a fork + FULL_NAME: ${{ github.event.pull_request.head.repo.full_name || github.event.repository.full_name }} + # `head_ref` is the pull_request branch + # `ref_name` is the push/workflow_dispatch/schedule branch + REF: ${{ github.head_ref || github.ref_name }} + with: + path: https://raw.githubusercontent.com/${{ env.FULL_NAME }}/refs/heads/${{ env.REF }}/read-file/data/json.json + parser: json + + - name: Read Local YAML + id: yaml + uses: ./read-file + with: + path: ./read-file/data/yaml.yaml + parser: yaml + + - name: Setup Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: '>=3.9' + + - name: Run Tests + shell: python + run: | + assert '''${{ steps.json.outputs.content }}''' == '''${{ steps.yaml.outputs.content }}''' + assert '''${{ fromJSON(steps.json.outputs.content)['foo'] }}''' == '''${{ fromJSON(steps.yaml.outputs.content)['foo'] }}''' + # required check analyze: - needs: [pytest] + needs: [pytest, read-file] if: '!cancelled()' runs-on: ubuntu-latest steps: diff --git a/pyproject.toml b/pyproject.toml index 44406e4..1328b87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,10 @@ skip = '.git' [tool.coverage.report] -exclude_also = ["if TYPE_CHECKING:"] +exclude_also = [ + "if TYPE_CHECKING:", + 'if __name__ == "__main__":', +] [tool.coverage.run] omit = [ @@ -15,6 +18,7 @@ omit = [ addopts = [ "--color=yes", "--cov=combine-durations", + "--cov=read-file", "--cov=template-files", "--cov-append", "--cov-branch", @@ -46,5 +50,6 @@ select = [ [tool.ruff.lint.isort] known-first-party = [ "combine_durations", + "read_file", "template_files", ] diff --git a/read-file/README.md b/read-file/README.md new file mode 100644 index 0000000..8fbaac7 --- /dev/null +++ b/read-file/README.md @@ -0,0 +1,52 @@ +# Read File + +A composite GitHub Action to read a local file or remote URL with an optional JSON/YAML parser. + +## Action Inputs + +| Name | Description | Default | +|------|-------------|---------| +| `path` | Local path or remote URL to the file to read. | **required** | +| `parser` | Parser to use for the file. Choose json, yaml, or null (to leave it as plain text). | **optional** | +| `default` | File contents to use if the file is not found. | **optional** | + +## Action Outputs + +| Name | Description | +|------|-------------| +| `content` | File contents as a JSON object (if a parser is specified) or the raw text. | + +## Sample Workflows + +```yaml +name: Read File + +on: + pull_request: + +jobs: + read: + steps: + - id: read_json + uses: conda/actions/read-file + with: + path: https://raw.githubusercontent.com/owner/repo/ref/path/to/json.json + default: '{}' + parser: json + + - id: read_yaml + uses: conda/actions/read-file + with: + path: https://raw.githubusercontent.com/owner/repo/ref/path/to/yaml.yaml + default: '{}' + parser: yaml + + - id: read_text + uses: conda/actions/read-file + with: + path: path/to/text.text + + - run: echo "${{ fromJSON(steps.read_json.outputs.content)['key'] }}" + - run: echo "${{ fromJSON(steps.read_yaml.outputs.content)['key'] }}" + - run: echo "${{ steps.read_file.outputs.content }}" +``` diff --git a/read-file/action.yml b/read-file/action.yml new file mode 100644 index 0000000..0df13af --- /dev/null +++ b/read-file/action.yml @@ -0,0 +1,59 @@ +name: Read File +description: Read a local file or remote URL. +author: Anaconda Inc. +branding: + icon: book-open + color: green + +inputs: + path: + description: Local path or remote URL to the file to read. + required: true + parser: + description: Parser to use for the file. Choose json, yaml, or null (to leave it as plain text). + default: + description: File contents to use if the file is not found. +outputs: + content: + description: File contents as a JSON object (if a parser is specified) or the raw text. + value: ${{ steps.read.outputs.content }} + +runs: + using: composite + steps: + # `hashFiles` only works on files within the working directory, since `requirements.txt` + # is not in the working directory we need to manually compute the SHA256 hash + - name: Compute Hash + id: hash + shell: bash + run: echo hash=$(sha256sum ${{ github.action_path }}/requirements.txt | awk '{print $1}') >> $GITHUB_OUTPUT + + - name: Pip Cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ~/.cache/pip + key: ${{ github.workflow }}-read-file-${{ steps.hash.outputs.hash }} + + - name: Setup Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: '>=3.9' + + - name: Pip Install + shell: bash + run: pip install --quiet -r ${{ github.action_path }}/requirements.txt + + - name: Pip List + shell: bash + run: pip list + + - name: Read JSON + id: read + shell: bash + run: > + python ${{ github.action_path }}/read_file.py + ${{ inputs.path }} + ${{ inputs.parser && format('"--parser={0}"', inputs.parser) || '' }} + ${{ inputs.default && format('"--default={0}"', inputs.default) || '' }} + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/read-file/data/json.json b/read-file/data/json.json new file mode 100644 index 0000000..6d95903 --- /dev/null +++ b/read-file/data/json.json @@ -0,0 +1 @@ +{"foo": "bar"} diff --git a/read-file/data/yaml.yaml b/read-file/data/yaml.yaml new file mode 100644 index 0000000..20e9ff3 --- /dev/null +++ b/read-file/data/yaml.yaml @@ -0,0 +1 @@ +foo: bar diff --git a/read-file/read_file.py b/read-file/read_file.py new file mode 100644 index 0000000..9ec993b --- /dev/null +++ b/read-file/read_file.py @@ -0,0 +1,97 @@ +"""Read a local file or remote URL.""" + +from __future__ import annotations + +import json +import os +from argparse import ArgumentParser +from pathlib import Path +from typing import TYPE_CHECKING + +import requests +import yaml +from requests.exceptions import HTTPError, MissingSchema + +if TYPE_CHECKING: + from argparse import Namespace + from collections.abc import Sequence + from typing import Literal + + +def parse_args(argv: Sequence[str] | None = None) -> Namespace: + # parse CLI for inputs + parser = ArgumentParser() + parser.add_argument( + "file", + type=str, + help="Local path or remote URL to the file to read.", + ) + parser.add_argument( + "--parser", + choices=["json", "yaml"], + help=( + "Parser to use for the file. " + "If not specified, the file content is returned as is." + ), + ) + parser.add_argument( + "--default", + type=str, + help=( + "Default value to use if the file is not found. " + "If not specified, an error is raised." + ), + ) + return parser.parse_args(argv) + + +def read_file(file: str | os.PathLike[str] | Path, default: str | None) -> str: + try: + response = requests.get(file) + response.raise_for_status() + except (HTTPError, MissingSchema): + # HTTPError: if the response status code is not ok + # MissingSchema: if the URL is not valid + try: + return Path(file).read_text() + except FileNotFoundError: + if default is None: + raise + return default + else: + return response.text + + +def parse_content(content: str, parser: Literal["json", "yaml"]) -> str: + # if a parser is defined we parse the content and dump it as JSON + if parser == "json": + content = json.loads(content) + return json.dumps(content) + elif parser == "yaml": + content = yaml.safe_load(content) + return json.dumps(content) + else: + raise ValueError("Parser not supported.") + + +def get_output(content: str) -> str: + return f"content< None: + if output := os.getenv("GITHUB_OUTPUT"): + with Path(output).open("a") as fh: + fh.write(get_output(content)) + + +def main() -> None: + args = parse_args() + + content = read_file(args.file, args.default) + if args.parser: + content = parse_content(content, args.parser) + dump_output(content) + + +if __name__ == "__main__": + main() diff --git a/read-file/requirements.txt b/read-file/requirements.txt new file mode 100644 index 0000000..48cade3 --- /dev/null +++ b/read-file/requirements.txt @@ -0,0 +1,2 @@ +pyyaml==6.0.2 +requests==2.32.3 diff --git a/read-file/test_read_file.py b/read-file/test_read_file.py new file mode 100644 index 0000000..0147dfe --- /dev/null +++ b/read-file/test_read_file.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from argparse import Namespace +from contextlib import nullcontext, suppress +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from json.decoder import JSONDecodeError +from pathlib import Path +from queue import Queue +from socket import IPPROTO_IPV6, IPV6_V6ONLY +from threading import Thread +from typing import TYPE_CHECKING + +import pytest + +from read_file import dump_output, get_output, parse_args, parse_content, read_file + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Final, Literal + + from pytest import MonkeyPatch + + +DATA: Final = Path(__file__).parent / "data" + + +@pytest.fixture(scope="session") +def test_server() -> Iterator[ThreadingHTTPServer]: + """ + Run a test server on a random port. Inspect returned server to get port, + shutdown etc. + + See https://github.com/conda/conda/blob/52b6393d6331e8aa36b2e23ab65766a980f381d2/tests/http_test_server.py + """ + + class DualStackServer(ThreadingHTTPServer): + daemon_threads = False # These are per-request threads + allow_reuse_address = True # Good for tests + request_queue_size = 64 # Should be more than the number of test packages + + def server_bind(self): + # suppress exception when protocol is IPv4 + with suppress(Exception): + self.socket.setsockopt(IPPROTO_IPV6, IPV6_V6ONLY, 0) + return super().server_bind() + + def finish_request(self, request, client_address): + self.RequestHandlerClass(request, client_address, self, directory=DATA) + + def __str__(self) -> str: + host, port = self.socket.getsockname()[:2] + url_host = f"[{host}]" if ":" in host else host + return f"http://{url_host}:{port}/" + + def start_server(queue: Queue): + with DualStackServer(("localhost", 0), SimpleHTTPRequestHandler) as httpd: + queue.put(httpd) + print(f"Serving ({httpd}) ...") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + + started: Queue[ThreadingHTTPServer] = Queue() + Thread(target=start_server, args=(started,), daemon=True).start() + yield (server := started.get(timeout=1)) + server.shutdown() + + +def test_parse_args_file() -> None: + # file is required + with pytest.raises(SystemExit): + assert parse_args([]) + with pytest.raises(SystemExit): + assert parse_args(["--parser=json"]) + with pytest.raises(SystemExit): + assert parse_args(["--default=text"]) + assert parse_args(["file"]) + + +@pytest.mark.parametrize( + "parser,default", + [ + (None, None), + ("json", None), + ("yaml", None), + (None, "text"), + ("json", "text"), + ("yaml", "text"), + ], +) +def test_parse_args_optional(parser: str | None, default: str | None) -> None: + # other args are optional + assert parse_args( + [ + "file", + *([f"--parser={parser}"] if parser else []), + *([f"--default={default}"] if default else []), + ] + ) == Namespace(file="file", parser=parser, default=default) + + +@pytest.mark.parametrize("source", ["local", "test_server"]) +@pytest.mark.parametrize( + "path,default,raises", + [ + ("json.json", None, False), + ("json.json", "default", False), + ("yaml.yaml", None, False), + ("yaml.yaml", "default", False), + ("missing", None, True), + ("missing", "default", False), + ], +) +def test_read_file( + test_server: ThreadingHTTPServer, + source: Literal["local", "test_server"], + path: str, + default: str | None, + raises: bool, +) -> None: + content = default + with suppress(FileNotFoundError): + content = (DATA / path).read_text() + + with pytest.raises(FileNotFoundError) if raises else nullcontext(): + uri = f"{DATA if source == 'local' else test_server}/{path}" + assert read_file(uri, default) == content + + +@pytest.mark.parametrize( + "path,parser,raises", + [ + ("json.json", "json", False), + ("json.json", "yaml", False), + ("json.json", "unknown", ValueError), + ("yaml.yaml", "json", JSONDecodeError), + ("yaml.yaml", "yaml", False), + ("yaml.yaml", "unknown", ValueError), + ], +) +def test_parse_content( + path: str, parser: Literal["json", "yaml"], raises: bool +) -> None: + content = (DATA / path).read_text() + expected = (DATA / "json.json").read_text().strip() + with pytest.raises(raises) if raises else nullcontext(): + assert parse_content(content, parser) == expected + + +def test_get_output() -> None: + assert get_output("content") == ( + "content< None: + # no-op if GITHUB_OUTPUT is not set + dump_output("noop") + + # set GITHUB_OUTPUT and check for content + monkeypatch.setenv("GITHUB_OUTPUT", str(output := tmp_path / "output")) + + dump_output("content") + assert output.read_text() == (content := get_output("content")) + + dump_output("more") + assert output.read_text() == content + get_output("more")