Refactor testing

.coveragerc

@@ -2,22 +2,11 @@
 source = custom_components
 
 omit =
-    # omit pieces of code that rely on external API calls
-    # These should be mocked one day....
-    custom_components/hacs/helpers/get_defaults.py
-    custom_components/hacs/operational/setup_actions/load_hacs_repository.py
-    custom_components/hacs/operational/factory.py
-    custom_components/hacs/operational/relaod.py
-    custom_components/hacs/operational/remove.py
-    custom_components/hacs/operational/setup.py
-    custom_components/hacs/config_flow.py
-    custom_components/hacs/__init__.py
-
     # omit tests
     tests/*
 
     # omit scripts
-    scripts/*
+    scripts/update/*
 
 [report]
 exclude_lines =

.github/workflows/pytest.yml

@@ -13,45 +13,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run:
-    name: With pytest for Python ${{ matrix.python-version }}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11"]
-    steps:
-      - name: 📥 Checkout the repository
-        uses: actions/[email protected]
-
-      - name: 🛠️ Set up Python
-        uses: actions/[email protected]
-        id: python
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: |
-            requirements_base.txt
-            requirements_test.txt
-
-      - name: 📦 Install requirements
-        run: |
-          scripts/install/frontend
-          scripts/install/pip_packages --requirement requirements_test.txt
-
-      - name: 🏃 Run tests
-        env:
-          PYTEST: true
-        run: |
-          python3 -m pytest
-
-      - name: 📤 Upload coverage to Codecov
-        if: ${{ matrix.python-version == '3.11' }}
-        run: |
-          scripts/coverage
-          curl -sfSL https://codecov.io/bash | bash -
-
   legacy:
-    name: With pytest with Home Assistant 2023.6.0
+    name: With pytest with Home Assistant (min. supported version)
     runs-on: ubuntu-latest
     steps:
       - name: 📥 Checkout the repository
@@ -78,17 +41,23 @@ jobs:
         run: |
           python3 -m pytest
 
-  core_dev:
-    name: With pytest with Home Assistant dev
+  dev:
+    name: With pytest with Home Assistant (${{ matrix.homeassistant-version }}) & Python (${{ matrix.python-version }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        homeassistant-version:
+          - "dev"
+        python-version:
+          - "3.12"
     steps:
       - name: 📥 Checkout the repository
         uses: actions/[email protected]
 
-      - name: 🛠️ Set up Python 3.12
+      - name: 🛠️ Set up Python ${{ matrix.python-version }}
         uses: actions/[email protected]
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
           cache: 'pip'
           cache-dependency-path: |
             requirements_base.txt
@@ -105,3 +74,9 @@ jobs:
           PYTEST: true
         run: |
           python3 -m pytest
+
+      - name: 📤 Upload coverage to Codecov
+        if: ${{ matrix.python-version == '3.12' }}
+        run: |
+          scripts/coverage
+          curl -sfSL https://codecov.io/bash | bash -

.github/workflows/validate.yml

@@ -44,6 +44,10 @@ jobs:
       - name: Checkout the repository
         uses: actions/[email protected]
 
+      # Test files conflics with running hassfest
+      - name: Remove tests
+        run: rm -rf tests
+
       - name: Hassfest validation
         uses: "home-assistant/actions/hassfest@master"
 

action/action.py

@@ -168,5 +168,5 @@ async def validate_repository(hacs, repository, category, ref=None):
     except HacsException as exception:
         error(exception)
 
-
-asyncio.run(preflight())
+if __name__ == "__main__":
+    asyncio.run(preflight())

pyproject.toml

@@ -19,7 +19,8 @@ combine_as_imports = true
 
 
 [tool.pytest.ini_options]
-addopts = "-rxf -x -v -l --cov=./ --cov-report=xml"
+asyncio_mode="auto"
+addopts = "-rf -v -l --cov=./ --cov-report=xml"
 filterwarnings = [
     "ignore::DeprecationWarning",
 ]

requirements_base.txt

@@ -4,4 +4,4 @@ aiohttp_cors==0.7.0
 async-timeout>=4.0.3
 asynctest==0.13.0
 colorlog==6.7.0
-
+setuptools==68.0.0

requirements_test.txt

@@ -1,8 +1,15 @@
 -r requirements_base.txt
-aresponses==2.1.6
 asynctest==0.13.0
+fnv-hash-fast==0.5.0
+home-assistant-frontend
 homeassistant
+janus==1.0.0
+Pillow==10.1.0
+psutil-home-assistant==0.0.1
 pytest==7.4.3
 pytest-asyncio==0.21.1
 pytest-cov==4.1.0
+pytest-snapshot==0.9.0
 pytest-socket==0.6.0
+RestrictedPython==7.0
+SQLAlchemy==2.0.23

scripts/snapshot-update

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+python3 -m \
+    pytest \
+    tests \
+    --snapshot-update
+

tests/__init__.py

@@ -0,0 +1,28 @@
+import json
+
+from awesomeversion import AwesomeVersion
+from homeassistant import const, core, loader
+
+_async_suggest_report_issue_mock_call_tracker = []
+
+try:
+    _orig_async_suggest_report_issue = loader.async_suggest_report_issue
+except AttributeError:
+    if AwesomeVersion(const.__version__) >= "2023.11.0":
+        raise RuntimeError("loader.async_suggest_report_issue does not exist")
+
+    @core.callback
+    def _fallback_async_suggest_report_issue(*args, **kwargs):
+        return json.dumps(kwargs)
+
+    _orig_async_suggest_report_issue = _fallback_async_suggest_report_issue
+
+
+@core.callback
+def async_suggest_report_issue_mock(*args, **kwargs):
+    result = _orig_async_suggest_report_issue(*args, **kwargs)
+    _async_suggest_report_issue_mock_call_tracker.append(result)
+    return result
+
+
+loader.async_suggest_report_issue = async_suggest_report_issue_mock

tests/action/test_hacs_action_integration.py

@@ -0,0 +1,73 @@
+"""Tests for the HACS action."""
+import base64
+import json
+import os
+from unittest import mock
+
+import pytest
+
+from tests.common import TOKEN, MockedResponse, ResponseMocker, current_function_name
+from tests.conftest import SnapshotFixture
+
+
+@pytest.mark.parametrize(
+    "key,manifest,succeed",
+    (
+        ("bad_documentation", {"documentation": None}, False),
+        ("bad_issue_tracker", {"issue_tracker": None}, False),
+        ("valid_manifest1", {}, True),
+    ),
+)
+async def test_hacs_action_integration(
+    key: str,
+    manifest: dict[str, str | bool | None],
+    succeed: bool,
+    caplog: pytest.LogCaptureFixture,
+    response_mocker: ResponseMocker,
+    snapshots: SnapshotFixture,
+):
+    """Test the action."""
+    basemanifest = {
+        "domain": "example",
+        "version": "1.0.0",
+        "documentation": "https://example.com",
+        "codeowners": ["hacs-test-org"],
+        "issue_tracker": "https://example.com",
+        "name": "Example",
+    }
+    envpatch = {
+        "INPUT_GITHUB_TOKEN": TOKEN,
+        "INPUT_REPOSITORY": "hacs-test-org/integration-basic",
+        "INPUT_CATEGORY": "integration",
+    }
+
+    response_mocker.add(
+        "https://brands.home-assistant.io/domains.json",
+        response=MockedResponse(status=200, content={"custom": ["example"]}),
+    )
+    response_mocker.add(
+        f"https://api.github.com/repos/hacs-test-org/integration-basic/contents/custom_components/example/manifest.json",
+        response=MockedResponse(
+            status=200,
+            content={
+                "content": base64.b64encode(
+                    json.dumps({**basemanifest, **manifest}).encode("ascii")
+                ).decode("ascii")
+            },
+            keep=True,
+        ),
+    )
+
+    with mock.patch.dict(os.environ, envpatch), mock.patch("builtins.exit", mock.MagicMock()):
+        from action.action import preflight
+
+        await preflight()
+
+    assert ("All (8) checks passed" if succeed else "1/8 checks failed") in caplog.text
+
+    splitlines = [f"<{l.rsplit(' <')[1]}" for l in caplog.text.split("\n") if " <" in l]
+
+    snapshots.assert_match(
+        "\n".join(splitlines[0:2] + sorted(splitlines[2:-2]) + splitlines[-2:]),
+        f"action/{current_function_name()}/{key}.log",
+    )