Merge pull request #98 from mozilla-services/test/integration_layer_c…

…onsvc-2059

[CONSVC-2059] test: create integration directory and test infrastructure

.circleci/config.yml

@@ -15,6 +15,8 @@ workflows:
               ignore: main
       - unit-tests:
           <<: *pr-filters
+      - integration-tests:
+          <<: *pr-filters
       - contract-tests:
           <<: *pr-filters
           requires:
@@ -30,6 +32,8 @@ workflows:
               only: main
       - unit-tests:
           <<: *main-filters
+      - integration-tests:
+          <<: *main-filters
       - contract-tests:
           <<: *main-filters
           requires:
@@ -44,17 +48,19 @@ workflows:
           <<: *main-filters
           requires:
             - checks
-            - contract-tests
             - unit-tests
+            - integration-tests
+            - contract-tests
       # The following job will require manual approval in the CircleCI web application.
       # Once provided, and when all the requirements are fullfilled (e.g. tests)
       - unhold-to-deploy-to-prod:
           <<: *main-filters
           type: approval
           requires:
             - checks
-            - contract-tests
             - unit-tests
+            - integration-tests
+            - contract-tests
       # On approval of the `unhold-to-deploy-to-prod` job, any successive job that requires it
       # will run. In this case, it's manually triggering deployment to production.
       - docker-image-publish-prod:
@@ -77,8 +83,16 @@ jobs:
     steps:
       - checkout
       - run:
-          name: Testing
-          command: make test
+          name: Unit tests
+          command: make unit-tests
+  integration-tests:
+    docker:
+      - image: cimg/python:3.10
+    steps:
+      - checkout
+      - run:
+          name: Integration tests
+          command: make integration-tests
   contract-tests:
     machine:
       docker_layer_caching: true

Makefile

@@ -1,6 +1,7 @@
 APP_DIR := merino
 TEST_DIR := tests
 UNIT_TEST_DIR := $(TEST_DIR)/unit
+INTEGRATION_TEST_DIR := $(TEST_DIR)/integration
 CONTRACT_TEST_DIR := $(TEST_DIR)/contract
 APP_AND_TEST_DIRS := $(APP_DIR) $(TEST_DIR)
 INSTALL_STAMP := .install.stamp
@@ -23,17 +24,13 @@ lint: $(INSTALL_STAMP)  ##  Run various linters
 	$(POETRY) run flake8 $(APP_AND_TEST_DIRS)
 	$(POETRY) run bandit --quiet -r $(APP_AND_TEST_DIRS) -c "pyproject.toml"
 	$(POETRY) run pydocstyle $(APP_DIR) --config="pyproject.toml"
-	$(POETRY) run mypy $(APP_DIR) $(CONTRACT_TEST_DIR) --config-file="pyproject.toml"
+	$(POETRY) run mypy $(APP_DIR) $(INTEGRATION_TEST_DIR) $(CONTRACT_TEST_DIR) --config-file="pyproject.toml"
 
 .PHONY: format
 format: $(INSTALL_STAMP)  ##  Sort imports and reformat code
 	$(POETRY) run isort $(APP_AND_TEST_DIRS)
 	$(POETRY) run black $(APP_AND_TEST_DIRS)
 
-.PHONY: test
-test: $(INSTALL_STAMP)  ##  Run unit tests
-	MERINO_ENV=testing $(POETRY) run pytest -v $(UNIT_TEST_DIR) --cov $(APP_DIR) -r s
-
 .PHONY: dev
 dev: $(INSTALL_STAMP)  ##  Run merino locally and reload automatically
 	$(POETRY) run uvicorn $(APP_DIR).main:app --reload
@@ -42,6 +39,26 @@ dev: $(INSTALL_STAMP)  ##  Run merino locally and reload automatically
 run: $(INSTALL_STAMP)  ##  Run merino locally
 	$(POETRY) run uvicorn $(APP_DIR).main:app
 
+.PHONY: test
+test: $(INSTALL_STAMP)  ##  Run unit and integration tests
+	MERINO_ENV=testing $(POETRY) run pytest $(UNIT_TEST_DIR) $(INTEGRATION_TEST_DIR) --cov $(APP_DIR)
+
+.PHONY: unit-tests
+unit-tests: $(INSTALL_STAMP)  ##  Run unit tests
+	MERINO_ENV=testing $(POETRY) run pytest $(UNIT_TEST_DIR) --cov $(APP_DIR)
+
+.PHONY: unit-test-fixtures
+unit-test-fixtures: $(INSTALL_STAMP)  ##  List fixtures in use per unit test
+	MERINO_ENV=testing $(POETRY) run pytest $(UNIT_TEST_DIR) --fixtures-per-test
+
+.PHONY: integration-tests
+integration-tests: $(INSTALL_STAMP)  ##  Run integration tests
+	MERINO_ENV=testing $(POETRY) run pytest $(INTEGRATION_TEST_DIR) --cov $(APP_DIR)
+
+.PHONY: integration-test-fixtures
+integration-test-fixtures: $(INSTALL_STAMP)  ##  List fixtures in use per integration test
+	MERINO_ENV=testing $(POETRY) run pytest $(INTEGRATION_TEST_DIR) --fixtures-per-test
+
 .PHONY: contract-tests
 contract-tests:  ##  Run contract tests using docker compose
 	docker-compose \

docs/dev/index.md

@@ -41,6 +41,12 @@ $ make dev
 # Run merino-py without the auto code reloading
 $ make run
 
+# Run unit tests
+$ make unit-tests
+
+# Run integration tests
+$ make integration-tests
+
 # Run contract tests
 $ make contract-tests
 

docs/dev/testing.md

@@ -1,16 +1,134 @@
 # Testing strategies
 
-There are three major testing strategies used in this repository: unit tests,
-Python contract tests, and Python load tests.
+Merino is tested using a four tier strategy composed of unit, integration, contract
+and load testing.
 
 Test code resides in the `tests` directory.
 
 ## Unit Tests
 
-Unit tests, located in the `tests/unit` directory, should appear close to the
-code they are testing, using the pytest unit test library. This is suitable for
-testing complex behavior at a small scale, with fine grained control over the
-inputs.
+The unit layer is suitable for testing complex behavior at a small scale, with
+fine-grained control over the inputs. Due to their narrow scope, unit tests are
+fundamental to thorough test coverage.
+
+To execute unit tests, use: `make unit-tests`
+
+Unit tests are written and executed with pytest and are located in the `tests/unit`
+directory, using the same organizational structure as the source code of the merino
+service. Type aliases dedicated for test should be stored in the `types.py` module.
+The `conftest.py` modules contain common utilities in fixtures.
+
+For a breakdown of fixtures in use per test, use: `make unit-test-fixtures`
+
+Available fixtures include:
+
+#### FilterCaplogFixture
+Useful when verifying log messages, this fixture filters log records captured with
+pytest's caplog by a given `logger_name`.
+
+_**Usage:**_
+```python
+def test_with_filter_caplog(
+    caplog: LogCaptureFixture, filter_caplog: FilterCaplogFixture
+) -> None:
+    records: list[LogRecord] = filter_caplog(caplog.records, "merino.providers.adm")
+```
+Note: This fixture is shared with integration tests.
+
+#### SuggestionRequestFixture
+For use when querying providers, this fixture creates a SuggestionRequest object with
+a given `query`
+
+_**Usage:**_
+```python
+def test_with_suggestion_request(srequest: SuggestionRequestFixture) -> None:
+    request: SuggestionRequest = srequest("example")
+    result: list[BaseSuggestion] = await provider.query(request)
+```
+
+
+## Integration Tests
+
+The integration layer of testing allows for verification of interactions between
+service components, with lower development, maintenance and execution costs compared
+with higher level tests, such as contract tests.
+
+To execute integration tests, use: `make integration-tests`
+
+Integration tests are located in the `tests/integration` directory. They use pytest and
+the FastAPI `TestClient` to send requests to specific merino endpoints and verify
+responses as well as other outputs, such as logs. Tests are organized according to the
+API path under test. Type aliases dedicated for test should be stored in the `types.py`
+module. Fake providers created for test should be stored in the `fake_providers.py`
+module. The `conftest.py` modules contain common utilities in fixtures.
+
+For a breakdown of fixtures in use per test, use: `make integration-test-fixtures`
+
+Available fixtures include:
+
+#### FilterCaplogFixture
+
+[Details](#FilterCaplogFixture) available in Unit Tests section
+
+#### TestClientFixture
+This fixture creates an instance of the TestClient to be used in testing API calls.
+
+_**Usage:**_
+```python
+def test_with_test_client(client: TestClient):
+    response: Response = client.get("/api/v1/endpoint")
+```
+
+#### TestClientWithEventsFixture
+This fixture creates an instance of the TestClient, that will trigger event handlers
+(i.e. `startup` and `shutdown`) to be used in testing API calls.
+
+_**Usage:**_
+```python
+def test_with_test_client_with_event(client_with_events: TestClient):
+    response: Response = client_with_events.get("/api/v1/endpoint")
+```
+
+#### InjectProvidersFixture & ProvidersFixture
+These fixture will setup and teardown given providers.
+
+_**Usage:**_
+
+If specifying providers for a module:
+```python
+@pytest.fixture(name="providers")
+def fixture_providers() -> Providers:
+    return {"test-provider": TestProvider()}
+```
+
+If specifying providers for a test:
+```python
+@pytest.mark.parametrize("providers", [{"test-provider": TestProvider()}])
+def test_with_provider() -> None:
+    pass
+```
+
+#### SetupProvidersFixture
+This fixture sets application provider dependency overrides.
+
+_**Usage:**_
+```python
+def test_with_setup_providers(setup_providers: SetupProvidersFixture):
+    providers: dict[str, BaseProvider] = {"test-provider": TestProvider()}
+    setup_providers(providers)
+```
+
+#### TeardownProvidersFixture
+This fixture resets application provider dependency overrides and is often used in
+teardown fixtures.
+
+_**Usage:**_
+```python
+@pytest.fixture(autouse=True)
+def teardown(teardown_providers: TeardownProvidersFixture):
+    yield
+    teardown_providers()
+```
 
 ## Contract tests
 

merino/configs/testing.toml

@@ -37,4 +37,4 @@ enabled_by_default = true
 enabled_by_default = true
 score = 0.25
 query_char_limit = 4
-top_picks_file_path = "tests/unit/providers/top_picks.json"
+top_picks_file_path = "tests/data/top_picks.json"

pyproject.toml

@@ -43,6 +43,7 @@ add-ignore = ["D105","D107","D203", "D205", "D400"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "-v -r s"
 
 [tool.poetry]
 name = "merino-py"

tests/conftest.py

@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from logging import LogRecord
+
+import pytest
+
+from tests.types import FilterCaplogFixture
+
+
+@pytest.fixture(scope="session", name="filter_caplog")
+def fixture_filter_caplog() -> FilterCaplogFixture:
+    """
+    Return a function that will filter pytest captured log records for a given logger
+    name
+    """
+
+    def filter_caplog(records: list[LogRecord], logger_name: str) -> list[LogRecord]:
+        """Filter pytest captured log records for a given logger name"""
+        return [record for record in records if record.name == logger_name]
+
+    return filter_caplog

tests/contract/README.md

@@ -21,7 +21,7 @@ The following sequence diagram depicts container interactions during the
 
 The `client` container consists of a Python-based test framework that executes the
 contract tests. The HTTP client used in the framework can be instructed to prepare
-remote settings data through requests to kinto and can verify merino functionality
+Remote Settings data through requests to kinto and can verify merino functionality
 through requests to the merino service.
 
 For more details see the client [README][client_readme]
@@ -36,7 +36,7 @@ For more details, see the merino [README][merino_readme] or project
 ### kinto-setup
 
 The `kinto-setup` container consists of a Python-based program responsible for
-defining the remote settings bucket, "main", and collection, "quicksuggest", prior
+defining the Remote Settings bucket, "main", and collection, "quicksuggest", prior
 to the `merino` container startup, a pre-requisite.
 
 For more details see the kinto-setup [README][kinto_setup_readme]
@@ -46,7 +46,7 @@ For more details see the kinto-setup [README][kinto_setup_readme]
 The `kinto` container holds a minimalist storage service with synchronisation and
 sharing abilities. It uses the `kinto-attachments` container to store data locally.
 
-For more details see the kinto [documentation][kinto_docs]
+For more details see the Remote Settings [documentation][kinto_docs]
 
 ## Local Execution
 
@@ -64,7 +64,7 @@ make contract-tests-clean
 ```
 
 [client_readme]: ./client/README.md
-[kinto_docs]: https://pypi.org/project/kinto/
+[kinto_docs]: https://remote-settings.readthedocs.io/en/latest/
 [kinto_setup_readme]: ./kinto-setup/README.md
 [merino_docs]: ../../docs/SUMMARY.md
 [merino_readme]: ../../README.md

tests/contract/client/README.md

@@ -19,7 +19,7 @@ and steps.
 
 #### Name
 
-A test name should identify the use case under test . Names are written in snake case
+A test name should identify the use case under test. Names are written in snake case
 with double-underscores `__` for scenario and behavior delineation.
 
 Example:
@@ -113,9 +113,9 @@ Example:
 
 ## Local Execution
 
-To execute the test scenarios outside the client Docker container, create a python
+To execute the test scenarios outside the client Docker container, create a Python
 virtual environment, set environment variables, expose the Merino and Kinto API ports
-in the docker-compose.yml and use a pytest command. It is recommended to execute the
+in the `docker-compose.yml` and use a pytest command. It is recommended to execute the
 tests within a Python virtual environment to prevent dependency cross contamination.
 
 1. Create a Virtual Environment
@@ -183,7 +183,7 @@ tests within a Python virtual environment to prevent dependency cross contaminat
 
       Example executing the `remote_settings__refresh` scenario:
       ```shell
-      pytest tests/contract/client/tests/test_contile.py -vv -k remote_settings__refresh
+      pytest tests/contract/client/tests/test_merino.py -vv -k remote_settings__refresh
       ```
 
 [contract_tests_readme]: ../README.md

tests/contract/kinto-setup/README.md

@@ -3,17 +3,17 @@
 ## Overview
 
 This directory contains source code for setting up Kinto for contract tests.
-Specifically, it is responsible for the creation of the remote settings bucket and
+Specifically, it is responsible for the creation of the Remote Settings bucket and
 collection, a pre-requisite for the Merino service.
 
 For more details on contract test design, refer to the contract-tests
 [README][contract_tests_readme].
 
 ## Local Execution
 
-To execute the kinto-setup outside the Docker container, create a python virtual
+To execute the kinto-setup outside the Docker container, create a Python virtual
 environment, set environment variables, expose the Kinto API port in the
-docker-compose.yml and use a python command. It is recommended to execute the setup
+`docker-compose.yml` and use a Python command. It is recommended to execute the setup
 within a Python virtual environment to prevent dependency cross contamination.
 
 1. Create a Virtual Environment

tests/unit/providers/top_picks.json → tests/data/top_picks.json

@@ -47,4 +47,4 @@
             ]
         }
     ]
-}
+}