From 93ba8f85e10f2594011b9b2c8c5099f9ce38ab73 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Tue, 21 Feb 2023 15:16:37 -0500 Subject: [PATCH 1/7] Combine conftest.py files --- tests/conftest.py | 718 +++++++++++++++++- tests/ctl/cli/test_cli_utils.py | 2 +- tests/ctl/conftest.py | 387 ---------- tests/ctl/core/test_dataset.py | 6 +- tests/{ops => }/fixtures/__init__.py | 0 .../fixtures/application_fixtures.py | 0 tests/{ops => }/fixtures/bigquery_fixtures.py | 0 tests/{ops => }/fixtures/email_fixtures.py | 0 .../fides_connector_example_fixtures.py | 0 .../fixtures/integration_fixtures.py | 0 tests/{ops => }/fixtures/manual_fixtures.py | 0 .../fixtures/manual_webhook_fixtures.py | 0 tests/{ops => }/fixtures/mariadb_fixtures.py | 0 tests/{ops => }/fixtures/mongodb_fixtures.py | 0 tests/{ops => }/fixtures/mssql_fixtures.py | 0 tests/{ops => }/fixtures/mysql_fixtures.py | 0 tests/{ops => }/fixtures/postgres_fixtures.py | 0 .../bad_test_config.json | 0 .../privacy_center_config/test_config.json | 0 tests/{ops => }/fixtures/redshift_fixtures.py | 0 tests/{ops => }/fixtures/saas/__init__.py | 0 .../fixtures/saas/adobe_campaign_fixtures.py | 0 .../{ops => }/fixtures/saas/auth0_fixtures.py | 0 .../fixtures/saas/braintree_fixtures.py | 2 +- .../{ops => }/fixtures/saas/braze_fixtures.py | 0 .../saas/connection_template_fixtures.py | 0 .../fixtures/saas/datadog_fixtures.py | 0 .../{ops => }/fixtures/saas/domo_fixtures.py | 0 .../fixtures/saas/doordash_fixtures.py | 2 +- .../saas/external_datasets/braintree.sql | 0 .../saas/external_datasets/doordash.sql | 0 .../saas/external_datasets/friendbuy.sql | 0 .../saas/external_datasets/fullstory.sql | 0 .../saas/external_datasets/twilio.sql | 0 .../fixtures/saas/firebase_auth_fixtures.py | 0 .../fixtures/saas/friendbuy_fixtures.py | 2 +- .../saas/friendbuy_nextgen_fixtures.py | 0 .../fixtures/saas/fullstory_fixtures.py | 2 +- .../saas/google_analytics_fixtures.py | 0 .../fixtures/saas/hubspot_fixtures.py | 0 .../fixtures/saas/mailchimp_fixtures.py | 0 .../saas/mailchimp_override_fixtures.py | 2 +- .../saas/mailchimp_transactional_fixtures.py | 0 .../fixtures/saas/outreach_fixtures.py | 0 .../fixtures/saas/recharge_fixtures.py | 0 .../fixtures/saas/rollbar_fixtures.py | 0 .../fixtures/saas/salesforce_fixtures.py | 0 .../fixtures/saas/segment_fixtures.py | 0 .../fixtures/saas/sendgrid_fixtures.py | 0 .../fixtures/saas/sentry_fixtures.py | 0 .../fixtures/saas/shopify_fixtures.py | 0 .../saas/slack_enterprise_fixtures.py | 0 .../fixtures/saas/square_fixtures.py | 0 .../fixtures/saas/stripe_fixtures.py | 0 .../saas/twilio_conversations_fixtures.py | 2 +- .../saas/universal_analytics_fixtures.py | 0 .../fixtures/saas/wunderkind_fixtures.py | 0 .../fixtures/saas/zendesk_fixtures.py | 0 .../fixtures/saas_example_fixtures.py | 2 +- .../{ops => }/fixtures/snowflake_fixtures.py | 0 .../{ops => }/fixtures/timescale_fixtures.py | 0 tests/lib/conftest.py | 154 ---- tests/lib/test_client_model.py | 12 +- .../test_connection_config_endpoints.py | 2 +- .../api/v1/endpoints/test_user_endpoints.py | 2 +- .../test_user_permission_endpoints.py | 2 +- tests/ops/conftest.py | 333 -------- tests/ops/graph/graph_test_util.py | 3 +- .../saas/test_fullstory_task.py | 2 +- .../saas/test_hubspot_task.py | 2 +- .../saas/test_sendgrid_task.py | 2 +- tests/ops/integration_tests/test_execution.py | 2 +- .../test_consent_email_batch_send.py | 2 +- tests/ops/util/test_cache.py | 3 +- 74 files changed, 742 insertions(+), 906 deletions(-) delete mode 100644 tests/ctl/conftest.py rename tests/{ops => }/fixtures/__init__.py (100%) rename tests/{ops => }/fixtures/application_fixtures.py (100%) rename tests/{ops => }/fixtures/bigquery_fixtures.py (100%) rename tests/{ops => }/fixtures/email_fixtures.py (100%) rename tests/{ops => }/fixtures/fides_connector_example_fixtures.py (100%) rename tests/{ops => }/fixtures/integration_fixtures.py (100%) rename tests/{ops => }/fixtures/manual_fixtures.py (100%) rename tests/{ops => }/fixtures/manual_webhook_fixtures.py (100%) rename tests/{ops => }/fixtures/mariadb_fixtures.py (100%) rename tests/{ops => }/fixtures/mongodb_fixtures.py (100%) rename tests/{ops => }/fixtures/mssql_fixtures.py (100%) rename tests/{ops => }/fixtures/mysql_fixtures.py (100%) rename tests/{ops => }/fixtures/postgres_fixtures.py (100%) rename tests/{ops => }/fixtures/privacy_center_config/bad_test_config.json (100%) rename tests/{ops => }/fixtures/privacy_center_config/test_config.json (100%) rename tests/{ops => }/fixtures/redshift_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/__init__.py (100%) rename tests/{ops => }/fixtures/saas/adobe_campaign_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/auth0_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/braintree_fixtures.py (99%) rename tests/{ops => }/fixtures/saas/braze_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/connection_template_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/datadog_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/domo_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/doordash_fixtures.py (98%) rename tests/{ops => }/fixtures/saas/external_datasets/braintree.sql (100%) rename tests/{ops => }/fixtures/saas/external_datasets/doordash.sql (100%) rename tests/{ops => }/fixtures/saas/external_datasets/friendbuy.sql (100%) rename tests/{ops => }/fixtures/saas/external_datasets/fullstory.sql (100%) rename tests/{ops => }/fixtures/saas/external_datasets/twilio.sql (100%) rename tests/{ops => }/fixtures/saas/firebase_auth_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/friendbuy_fixtures.py (99%) rename tests/{ops => }/fixtures/saas/friendbuy_nextgen_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/fullstory_fixtures.py (99%) rename tests/{ops => }/fixtures/saas/google_analytics_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/hubspot_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/mailchimp_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/mailchimp_override_fixtures.py (98%) rename tests/{ops => }/fixtures/saas/mailchimp_transactional_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/outreach_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/recharge_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/rollbar_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/salesforce_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/segment_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/sendgrid_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/sentry_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/shopify_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/slack_enterprise_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/square_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/stripe_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/twilio_conversations_fixtures.py (99%) rename tests/{ops => }/fixtures/saas/universal_analytics_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/wunderkind_fixtures.py (100%) rename tests/{ops => }/fixtures/saas/zendesk_fixtures.py (100%) rename tests/{ops => }/fixtures/saas_example_fixtures.py (99%) rename tests/{ops => }/fixtures/snowflake_fixtures.py (100%) rename tests/{ops => }/fixtures/timescale_fixtures.py (100%) delete mode 100644 tests/lib/conftest.py delete mode 100644 tests/ops/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py index 944e7a3a78..74c33b2f16 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,241 @@ +import asyncio +import json +import os +from datetime import datetime +from pathlib import Path +from uuid import uuid4 + import pytest +import requests +import yaml +from fastapi.testclient import TestClient +from fideslang import models +from httpx import AsyncClient from loguru import logger +from pytest import MonkeyPatch from sqlalchemy.engine.base import Engine +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker +from toml import load as load_toml +from fides.api.ctl.database.session import sync_engine, sync_session +from fides.api.main import app +from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY +from fides.api.ops.db.base import Base +from fides.api.ops.models.privacy_request import generate_request_callback_jwe +from fides.api.ops.schemas.messaging.messaging import MessagingServiceType +from fides.api.ops.tasks.scheduled.scheduler import scheduler +from fides.api.ops.util.cache import get_cache +from fides.core import api from fides.core.config import get_config +from fides.lib.cryptography.schemas.jwt import ( + JWE_ISSUED_AT, + JWE_PAYLOAD_CLIENT_ID, + JWE_PAYLOAD_SCOPES, +) +from fides.lib.db.session import get_db_engine, get_db_session +from fides.lib.models.client import ClientDetail +from fides.lib.models.fides_user import FidesUser +from fides.lib.models.fides_user_permissions import FidesUserPermissions +from fides.lib.oauth.jwt import generate_jwe +from fides.lib.oauth.scopes import PRIVACY_REQUEST_READ +from tests.fixtures.application_fixtures import * +from tests.fixtures.bigquery_fixtures import * +from tests.fixtures.email_fixtures import * +from tests.fixtures.fides_connector_example_fixtures import * +from tests.fixtures.integration_fixtures import * +from tests.fixtures.manual_fixtures import * +from tests.fixtures.manual_webhook_fixtures import * +from tests.fixtures.mariadb_fixtures import * +from tests.fixtures.mongodb_fixtures import * +from tests.fixtures.mssql_fixtures import * +from tests.fixtures.mysql_fixtures import * +from tests.fixtures.postgres_fixtures import * +from tests.fixtures.redshift_fixtures import * +from tests.fixtures.saas import * +from tests.fixtures.saas_example_fixtures import * +from tests.fixtures.snowflake_fixtures import * +from tests.fixtures.timescale_fixtures import * + +ROOT_PATH = Path().absolute() +CONFIG = get_config() +TEST_CONFIG_PATH = "tests/ctl/test_config.toml" +TEST_INVALID_CONFIG_PATH = "tests/ctl/test_invalid_config.toml" +TEST_DEPRECATED_CONFIG_PATH = "tests/ctl/test_deprecated_config.toml" + +orig_requests_get = requests.get +orig_requests_post = requests.post +orig_requests_put = requests.put +orig_requests_patch = requests.patch +orig_requests_delete = requests.delete + + +@pytest.fixture(scope="session") +def monkeysession(): + """monkeypatch fixture at the session level instead of the function level""" + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + +@pytest.fixture(autouse=True, scope="session") +def monkeypatch_requests(test_client, monkeysession) -> None: + """The requests library makes requests against the running webserver + which talks to the application db. This monkeypatching operation + makes `requests` calls from src/fides/core/api.py in a test + context talk to the test db instead""" + monkeysession.setattr(requests, "get", test_client.get) + monkeysession.setattr(requests, "post", test_client.post) + monkeysession.setattr(requests, "put", test_client.put) + monkeysession.setattr(requests, "patch", test_client.patch) + monkeysession.setattr(requests, "delete", test_client.delete) + + +@pytest.fixture(scope="session") +def test_client(): + """Starlette test client fixture. Easier to use mocks with when testing out API calls""" + with TestClient(app) as test_client: + yield test_client + + +@pytest.fixture(scope="session") +@pytest.mark.asyncio +async def async_session(test_client): + assert CONFIG.test_mode + assert requests.post == test_client.post + + create_citext_extension(sync_engine) + + async_engine = create_async_engine( + CONFIG.database.async_database_uri, + echo=False, + ) + + session_maker = sessionmaker( + async_engine, class_=AsyncSession, expire_on_commit=False + ) + + async with session_maker() as session: + yield session + session.close() + async_engine.dispose() + + +@pytest.fixture(scope="session", autouse=True) +def setup_ctl_db(test_config, test_client, monkeypatch_requests): + "Sets up the database for testing." + assert CONFIG.test_mode + assert ( + requests.post == test_client.post + ) # Sanity check to make sure monkeypatch_requests fixture has run + yield api.db_action( + server_url=test_config.cli.server_url, + headers=CONFIG.user.auth_header, + action="reset", + ) + + +@pytest.fixture +def ctl_db(): + create_citext_extension(sync_engine) + + session = sync_session() + + yield session + session.close() + + +@pytest.fixture(scope="session", autouse=True) +def setup_db(api_client): + """Apply migrations at beginning and end of testing session""" + assert CONFIG.test_mode + assert requests.post != api_client.post + yield api_client.post(url=f"{CONFIG.cli.server_url}/v1/admin/db/reset") + + +@pytest.fixture(scope="session") +def db(api_client): + """Return a connection to the test DB""" + # Create the test DB engine + assert CONFIG.test_mode + assert requests.post != api_client.post + engine = get_db_engine( + database_uri=CONFIG.database.sqlalchemy_test_database_uri, + ) + + create_citext_extension(engine) + + if not scheduler.running: + scheduler.start() + SessionLocal = get_db_session(CONFIG, engine=engine) + the_session = SessionLocal() + # Setup above... + + yield the_session + # Teardown below... + the_session.close() + engine.dispose() + + +@pytest.fixture(scope="session") +def api_client(): + """Return a client used to make API requests""" + with TestClient(app) as c: + yield c + + +@pytest.fixture(scope="session") +async def async_api_client(): + """Return an async client used to make API requests""" + async with AsyncClient( + app=app, base_url="http://0.0.0.0:8080", follow_redirects=True + ) as client: + yield client + + +@pytest.fixture(scope="session", autouse=True) +def event_loop(): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +@pytest.fixture(autouse=True) +def clear_db_tables(db): + """Clear data from tables between tests. + + If relationships are not set to cascade on delete they will fail with an + IntegrityError if there are relationsips present. This function stores tables + that fail with this error then recursively deletes until no more IntegrityErrors + are present. + """ + yield + + def delete_data(tables): + redo = [] + for table in tables: + try: + db.execute(table.delete()) + except IntegrityError: + redo.append(table) + finally: + db.commit() + + if redo: + delete_data(redo) + + db.commit() # make sure all transactions are closed before starting deletes + delete_data(Base.metadata.sorted_tables) @pytest.fixture(scope="session") def config(): - config = get_config() - config.is_test_mode = True - yield config + CONFIG.is_test_mode = True + yield CONFIG @pytest.fixture @@ -22,3 +248,489 @@ def loguru_caplog(caplog): def create_citext_extension(engine: Engine) -> None: with engine.connect() as con: con.execute("CREATE EXTENSION IF NOT EXISTS citext;") + + +@pytest.fixture +def fides_toml_path(): + yield ROOT_PATH / ".fides" / "fides.toml" + + +@pytest.fixture +def oauth_client(db): + """Return a client for authentication purposes.""" + client = ClientDetail( + hashed_secret="thisisatest", + salt="thisisstillatest", + scopes=SCOPE_REGISTRY, + ) + db.add(client) + db.commit() + db.refresh(client) + yield client + + +@pytest.fixture +def oauth_root_client(db): + """Return the configured root client (never persisted)""" + return ClientDetail.get( + db, + object_id=CONFIG.security.oauth_root_client_id, + config=CONFIG, + scopes=SCOPE_REGISTRY, + ) + + +@pytest.fixture +def application_user(db, oauth_client): + unique_username = f"user-{uuid4()}" + user = FidesUser.create( + db=db, + data={ + "username": unique_username, + "password": "test_password", + "first_name": "Test", + "last_name": "User", + }, + ) + oauth_client.user_id = user.id + oauth_client.save(db=db) + yield user + + +@pytest.fixture +def user(db): + user = FidesUser.create( + db=db, + data={ + "username": "test_fidesops_user", + "password": "TESTdcnG@wzJeu0&%3Qe2fGo7", + }, + ) + client = ClientDetail( + hashed_secret="thisisatest", + salt="thisisstillatest", + scopes=SCOPE_REGISTRY, + user_id=user.id, + ) + + FidesUserPermissions.create( + db=db, data={"user_id": user.id, "scopes": [PRIVACY_REQUEST_READ]} + ) + + db.add(client) + db.commit() + db.refresh(client) + yield user + + +@pytest.fixture +def auth_header(request, oauth_client, config): + client_id = oauth_client.id + + payload = { + JWE_PAYLOAD_SCOPES: request.param, + JWE_PAYLOAD_CLIENT_ID: client_id, + JWE_ISSUED_AT: datetime.now().isoformat(), + } + jwe = generate_jwe(json.dumps(payload), config.security.app_encryption_key) + + return {"Authorization": "Bearer " + jwe} + + +@pytest.fixture(autouse=True) +def clear_get_config_cache() -> None: + get_config.cache_clear() + + +@pytest.fixture(scope="session") +def test_config_path(): + yield TEST_CONFIG_PATH + + +@pytest.fixture(scope="session") +def test_deprecated_config_path(): + yield TEST_DEPRECATED_CONFIG_PATH + + +@pytest.fixture(scope="session") +def test_invalid_config_path(): + """ + This config file contains url/connection strings that are invalid. + + This ensures that the CLI isn't calling out to those resources + directly during certain tests. + """ + yield TEST_INVALID_CONFIG_PATH + + +@pytest.fixture(scope="session") +def test_config(test_config_path: str): + yield get_config(test_config_path) + + +@pytest.fixture +def test_config_dev_mode_disabled(): + original_value = CONFIG.dev_mode + CONFIG.dev_mode = False + yield CONFIG + CONFIG.dev_mode = original_value + + +@pytest.fixture +def resources_dict(): + """ + Yields a resource containing sample representations of different + Fides resources. + """ + resources_dict = { + "data_category": models.DataCategory( + organization_fides_key=1, + fides_key="user.custom", + parent_key="user", + name="Custom Data Category", + description="Custom Data Category", + ), + "data_qualifier": models.DataQualifier( + organization_fides_key=1, + fides_key="custom_data_qualifier", + name="Custom Data Qualifier", + description="Custom Data Qualifier", + ), + "dataset": models.Dataset( + organization_fides_key=1, + fides_key="test_sample_db_dataset", + name="Sample DB Dataset", + description="This is a Sample Database Dataset", + collections=[ + models.DatasetCollection( + name="user", + fields=[ + models.DatasetField( + name="Food_Preference", + description="User's favorite food", + path="some.path", + ), + models.DatasetField( + name="First_Name", + description="A First Name Field", + path="another.path", + data_categories=["user.name"], + data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + ), + models.DatasetField( + name="Email", + description="User's Email", + path="another.another.path", + data_categories=["user.contact.email"], + data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + ), + ], + ) + ], + ), + "data_subject": models.DataSubject( + organization_fides_key=1, + fides_key="custom_subject", + name="Custom Data Subject", + description="Custom Data Subject", + ), + "data_use": models.DataUse( + organization_fides_key=1, + fides_key="custom_data_use", + name="Custom Data Use", + description="Custom Data Use", + ), + "evaluation": models.Evaluation( + fides_key="test_evaluation", status="PASS", details=["foo"], message="bar" + ), + "organization": models.Organization( + fides_key="test_organization", + name="Test Organization", + description="Test Organization", + ), + "policy": models.Policy( + organization_fides_key=1, + fides_key="test_policy", + name="Test Policy", + version="1.3", + description="Test Policy", + rules=[], + ), + "policy_rule": models.PolicyRule( + name="Test Policy", + data_categories=models.PrivacyRule(matches="NONE", values=[]), + data_uses=models.PrivacyRule(matches="NONE", values=["provide.service"]), + data_subjects=models.PrivacyRule(matches="ANY", values=[]), + data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized", + ), + "registry": models.Registry( + organization_fides_key=1, + fides_key="test_registry", + name="Test Registry", + description="Test Regsitry", + systems=[], + ), + "system": models.System( + organization_fides_key=1, + registryId=1, + fides_key="test_system", + system_type="SYSTEM", + name="Test System", + description="Test Policy", + privacy_declarations=[ + models.PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="provide", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ) + ], + system_dependencies=[], + ), + } + yield resources_dict + + +@pytest.fixture +def test_manifests(): + test_manifests = { + "manifest_1": { + "dataset": [ + { + "name": "Test Dataset 1", + "organization_fides_key": 1, + "datasetType": {}, + "datasetLocation": "somedb:3306", + "description": "Test Dataset 1", + "fides_key": "some_dataset", + "datasetTables": [], + } + ], + "system": [ + { + "name": "Test System 1", + "organization_fides_key": 1, + "systemType": "mysql", + "description": "Test System 1", + "fides_key": "some_system", + } + ], + }, + "manifest_2": { + "dataset": [ + { + "name": "Test Dataset 2", + "description": "Test Dataset 2", + "organization_fides_key": 1, + "datasetType": {}, + "datasetLocation": "somedb:3306", + "fides_key": "another_dataset", + "datasetTables": [], + } + ], + "system": [ + { + "name": "Test System 2", + "organization_fides_key": 1, + "systemType": "mysql", + "description": "Test System 2", + "fides_key": "another_system", + } + ], + }, + } + yield test_manifests + + +@pytest.fixture +def populated_manifest_dir(test_manifests, tmp_path): + manifest_dir = f"{tmp_path}/populated_manifest" + os.mkdir(manifest_dir) + for manifest in test_manifests.keys(): + with open(f"{manifest_dir}/{manifest}.yml", "w") as manifest_file: + yaml.dump(test_manifests[manifest], manifest_file) + return manifest_dir + + +@pytest.fixture +def populated_nested_manifest_dir(test_manifests, tmp_path): + manifest_dir = f"{tmp_path}/populated_nested_manifest" + os.mkdir(manifest_dir) + for manifest in test_manifests.keys(): + nested_manifest_dir = f"{manifest_dir}/{manifest}" + os.mkdir(nested_manifest_dir) + with open(f"{nested_manifest_dir}/{manifest}.yml", "w") as manifest_file: + yaml.dump(test_manifests[manifest], manifest_file) + return manifest_dir + + +@pytest.fixture(scope="session") +def cache(): + yield get_cache() + + +@pytest.fixture +def root_auth_header(oauth_root_client): + """Return an auth header for the root client""" + payload = { + JWE_PAYLOAD_SCOPES: oauth_root_client.scopes, + JWE_PAYLOAD_CLIENT_ID: oauth_root_client.id, + JWE_ISSUED_AT: datetime.now().isoformat(), + } + jwe = generate_jwe(json.dumps(payload), CONFIG.security.app_encryption_key) + return {"Authorization": "Bearer " + jwe} + + +def generate_auth_header_for_user(user, scopes): + payload = { + JWE_PAYLOAD_SCOPES: scopes, + JWE_PAYLOAD_CLIENT_ID: user.client.id, + JWE_ISSUED_AT: datetime.now().isoformat(), + } + jwe = generate_jwe(json.dumps(payload), CONFIG.security.app_encryption_key) + return {"Authorization": "Bearer " + jwe} + + +@pytest.fixture +def generate_auth_header(oauth_client): + return _generate_auth_header(oauth_client, CONFIG.security.app_encryption_key) + + +@pytest.fixture +def generate_auth_header_ctl_config(oauth_client): + return _generate_auth_header(oauth_client, CONFIG.security.app_encryption_key) + + +def _generate_auth_header(oauth_client, app_encryption_key): + client_id = oauth_client.id + + def _build_jwt(scopes): + payload = { + JWE_PAYLOAD_SCOPES: scopes, + JWE_PAYLOAD_CLIENT_ID: client_id, + JWE_ISSUED_AT: datetime.now().isoformat(), + } + jwe = generate_jwe(json.dumps(payload), app_encryption_key) + return {"Authorization": "Bearer " + jwe} + + return _build_jwt + + +@pytest.fixture +def generate_webhook_auth_header(): + def _build_jwt(webhook): + jwe = generate_request_callback_jwe(webhook) + return {"Authorization": "Bearer " + jwe} + + return _build_jwt + + +@pytest.fixture(scope="session") +def integration_config(): + yield load_toml("tests/ops/integration_test_config.toml") + + +@pytest.fixture(scope="session") +def celery_config(): + return {"task_always_eager": False} + + +@pytest.fixture(autouse=True, scope="session") +def celery_enable_logging(): + """Turns on celery output logs.""" + return True + + +@pytest.fixture(autouse=True, scope="session") +def celery_use_virtual_worker(celery_session_worker): + """ + This is a catch-all fixture that forces all of our + tests to use a virtual celery worker if a registered + task is executed within the scope of the test. + """ + yield celery_session_worker + + +@pytest.fixture(scope="session") +def run_privacy_request_task(celery_session_app): + """ + This fixture is the version of the run_privacy_request task that is + registered to the `celery_app` fixture which uses the virtualised `celery_worker` + """ + yield celery_session_app.tasks[ + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request" + ] + + +@pytest.fixture(autouse=True, scope="session") +def analytics_opt_out(): + """Disable sending analytics when running tests.""" + original_value = CONFIG.user.analytics_opt_out + CONFIG.user.analytics_opt_out = True + yield + CONFIG.user.analytics_opt_out = original_value + + +@pytest.fixture +def require_manual_request_approval(): + """Require manual request approval""" + original_value = CONFIG.execution.require_manual_request_approval + CONFIG.execution.require_manual_request_approval = True + yield + CONFIG.execution.require_manual_request_approval = original_value + + +@pytest.fixture +def subject_identity_verification_required(): + """Enable identity verification.""" + original_value = CONFIG.execution.subject_identity_verification_required + CONFIG.execution.subject_identity_verification_required = True + yield + CONFIG.execution.subject_identity_verification_required = original_value + + +@pytest.fixture(autouse=True, scope="function") +def subject_identity_verification_not_required(): + """Disable identity verification for most tests unless overridden""" + original_value = CONFIG.execution.subject_identity_verification_required + CONFIG.execution.subject_identity_verification_required = False + yield + CONFIG.execution.subject_identity_verification_required = original_value + + +@pytest.fixture(autouse=True, scope="function") +def privacy_request_complete_email_notification_disabled(): + """Disable request completion email for most tests unless overridden""" + original_value = CONFIG.notifications.send_request_completion_notification + CONFIG.notifications.send_request_completion_notification = False + yield + CONFIG.notifications.send_request_completion_notification = original_value + + +@pytest.fixture(autouse=True, scope="function") +def privacy_request_receipt_notification_disabled(): + """Disable request receipt notification for most tests unless overridden""" + original_value = CONFIG.notifications.send_request_receipt_notification + CONFIG.notifications.send_request_receipt_notification = False + yield + CONFIG.notifications.send_request_receipt_notification = original_value + + +@pytest.fixture(autouse=True, scope="function") +def privacy_request_review_notification_disabled(): + """Disable request review notification for most tests unless overridden""" + original_value = CONFIG.notifications.send_request_review_notification + CONFIG.notifications.send_request_review_notification = False + yield + CONFIG.notifications.send_request_review_notification = original_value + + +@pytest.fixture(scope="function", autouse=True) +def set_notification_service_type_mailgun(): + """Set default notification service type""" + original_value = CONFIG.notifications.notification_service_type + CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value + yield + CONFIG.notifications.notification_service_type = original_value diff --git a/tests/ctl/cli/test_cli_utils.py b/tests/ctl/cli/test_cli_utils.py index 95455aeb19..50989dc5cc 100644 --- a/tests/ctl/cli/test_cli_utils.py +++ b/tests/ctl/cli/test_cli_utils.py @@ -6,7 +6,7 @@ import fides.cli.utils as utils from fides.api.ctl.routes.util import API_PREFIX from fides.core.config import FidesConfig -from tests.ctl.conftest import orig_requests_get +from tests.conftest import orig_requests_get @pytest.mark.unit diff --git a/tests/ctl/conftest.py b/tests/ctl/conftest.py deleted file mode 100644 index 3c604c3c3a..0000000000 --- a/tests/ctl/conftest.py +++ /dev/null @@ -1,387 +0,0 @@ -# pylint: disable=invalid-name, missing-docstring, redefined-outer-name - -"""Common fixtures to be used across tests.""" - -from __future__ import annotations - -import asyncio -import json -import os -from datetime import datetime -from typing import Dict, Generator, Union - -import pytest -import requests -import yaml -from fideslang import models -from pytest import MonkeyPatch -from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine -from sqlalchemy.orm import sessionmaker -from starlette.testclient import TestClient - -from fides.api import main -from fides.api.ctl.database.session import sync_engine, sync_session -from fides.api.ctl.sql_models import FidesUser -from fides.core import api -from fides.core.config import FidesConfig, get_config -from fides.core.user import login_command -from fides.lib.cryptography.schemas.jwt import ( - JWE_ISSUED_AT, - JWE_PAYLOAD_CLIENT_ID, - JWE_PAYLOAD_SCOPES, -) -from fides.lib.oauth.jwt import generate_jwe -from tests.conftest import create_citext_extension - -TEST_CONFIG_PATH = "tests/ctl/test_config.toml" -TEST_INVALID_CONFIG_PATH = "tests/ctl/test_invalid_config.toml" -TEST_DEPRECATED_CONFIG_PATH = "tests/ctl/test_deprecated_config.toml" -CONFIG = get_config() - - -orig_requests_get = requests.get -orig_requests_post = requests.post -orig_requests_put = requests.put -orig_requests_patch = requests.patch -orig_requests_delete = requests.delete - - -@pytest.fixture(scope="session") -def monkeysession(): - """monkeypatch fixture at the session level instead of the function level""" - mpatch = MonkeyPatch() - yield mpatch - mpatch.undo() - - -@pytest.fixture(autouse=True, scope="session") -def monkeypatch_requests(test_client, monkeysession) -> None: - """The requests library makes requests against the running webserver - which talks to the application db. This monkeypatching operation - makes `requests` calls from src/fides/core/api.py in a test - context talk to the test db instead""" - monkeysession.setattr(requests, "get", test_client.get) - monkeysession.setattr(requests, "post", test_client.post) - monkeysession.setattr(requests, "put", test_client.put) - monkeysession.setattr(requests, "patch", test_client.patch) - monkeysession.setattr(requests, "delete", test_client.delete) - - -@pytest.fixture(scope="session") -def test_config_path() -> Generator: - yield TEST_CONFIG_PATH - - -@pytest.fixture(scope="session") -def test_deprecated_config_path() -> Generator: - yield TEST_DEPRECATED_CONFIG_PATH - - -@pytest.fixture(scope="session") -def test_invalid_config_path() -> Generator: - """ - This config file contains url/connection strings that are invalid. - - This ensures that the CLI isn't calling out to those resources - directly during certain tests. - """ - yield TEST_INVALID_CONFIG_PATH - - -@pytest.fixture(scope="session") -def test_config(test_config_path: str) -> Generator: - yield get_config(test_config_path) - - -@pytest.fixture(scope="function") -def test_config_dev_mode_disabled() -> Generator: - original_value = CONFIG.dev_mode - CONFIG.dev_mode = False - yield CONFIG - CONFIG.dev_mode = original_value - - -@pytest.fixture(scope="session") -def test_client() -> Generator: - """Starlette test client fixture. Easier to use mocks with when testing out API calls""" - with TestClient(main.app) as test_client: - yield test_client - - -@pytest.fixture(scope="session", autouse=True) -def setup_db(test_config: FidesConfig, test_client, monkeypatch_requests) -> Generator: - "Sets up the database for testing." - assert CONFIG.test_mode - assert ( - requests.post == test_client.post - ) # Sanity check to make sure monkeypatch_requests fixture has run - yield api.db_action( - server_url=test_config.cli.server_url, - headers=CONFIG.user.auth_header, - action="reset", - ) - - -@pytest.fixture(scope="function") -def resources_dict() -> Generator: - """ - Yields a resource containing sample representations of different - Fides resources. - """ - resources_dict: Dict[ - str, - Union[ - models.DataCategory, - models.DataQualifier, - models.Dataset, - models.DataSubject, - models.DataUse, - models.Evaluation, - models.Organization, - models.Policy, - models.PolicyRule, - models.Registry, - models.System, - ], - ] = { - "data_category": models.DataCategory( - organization_fides_key=1, - fides_key="user.custom", - parent_key="user", - name="Custom Data Category", - description="Custom Data Category", - ), - "data_qualifier": models.DataQualifier( - organization_fides_key=1, - fides_key="custom_data_qualifier", - name="Custom Data Qualifier", - description="Custom Data Qualifier", - ), - "dataset": models.Dataset( - organization_fides_key=1, - fides_key="test_sample_db_dataset", - name="Sample DB Dataset", - description="This is a Sample Database Dataset", - collections=[ - models.DatasetCollection( - name="user", - fields=[ - models.DatasetField( - name="Food_Preference", - description="User's favorite food", - path="some.path", - ), - models.DatasetField( - name="First_Name", - description="A First Name Field", - path="another.path", - data_categories=["user.name"], - data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", - ), - models.DatasetField( - name="Email", - description="User's Email", - path="another.another.path", - data_categories=["user.contact.email"], - data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", - ), - ], - ) - ], - ), - "data_subject": models.DataSubject( - organization_fides_key=1, - fides_key="custom_subject", - name="Custom Data Subject", - description="Custom Data Subject", - ), - "data_use": models.DataUse( - organization_fides_key=1, - fides_key="custom_data_use", - name="Custom Data Use", - description="Custom Data Use", - ), - "evaluation": models.Evaluation( - fides_key="test_evaluation", status="PASS", details=["foo"], message="bar" - ), - "organization": models.Organization( - fides_key="test_organization", - name="Test Organization", - description="Test Organization", - ), - "policy": models.Policy( - organization_fides_key=1, - fides_key="test_policy", - name="Test Policy", - version="1.3", - description="Test Policy", - rules=[], - ), - "policy_rule": models.PolicyRule( - name="Test Policy", - data_categories=models.PrivacyRule(matches="NONE", values=[]), - data_uses=models.PrivacyRule(matches="NONE", values=["provide.service"]), - data_subjects=models.PrivacyRule(matches="ANY", values=[]), - data_qualifier="aggregated.anonymized.unlinked_pseudonymized.pseudonymized", - ), - "registry": models.Registry( - organization_fides_key=1, - fides_key="test_registry", - name="Test Registry", - description="Test Regsitry", - systems=[], - ), - "system": models.System( - organization_fides_key=1, - registryId=1, - fides_key="test_system", - system_type="SYSTEM", - name="Test System", - description="Test Policy", - privacy_declarations=[ - models.PrivacyDeclaration( - name="declaration-name", - data_categories=[], - data_use="provide", - data_subjects=[], - data_qualifier="aggregated_data", - dataset_references=[], - ) - ], - system_dependencies=[], - ), - } - yield resources_dict - - -@pytest.fixture() -def test_manifests() -> Generator: - test_manifests = { - "manifest_1": { - "dataset": [ - { - "name": "Test Dataset 1", - "organization_fides_key": 1, - "datasetType": {}, - "datasetLocation": "somedb:3306", - "description": "Test Dataset 1", - "fides_key": "some_dataset", - "datasetTables": [], - } - ], - "system": [ - { - "name": "Test System 1", - "organization_fides_key": 1, - "systemType": "mysql", - "description": "Test System 1", - "fides_key": "some_system", - } - ], - }, - "manifest_2": { - "dataset": [ - { - "name": "Test Dataset 2", - "description": "Test Dataset 2", - "organization_fides_key": 1, - "datasetType": {}, - "datasetLocation": "somedb:3306", - "fides_key": "another_dataset", - "datasetTables": [], - } - ], - "system": [ - { - "name": "Test System 2", - "organization_fides_key": 1, - "systemType": "mysql", - "description": "Test System 2", - "fides_key": "another_system", - } - ], - }, - } - yield test_manifests - - -@pytest.fixture() -def populated_manifest_dir(test_manifests: Dict, tmp_path: str) -> str: - manifest_dir = f"{tmp_path}/populated_manifest" - os.mkdir(manifest_dir) - for manifest in test_manifests.keys(): - with open(f"{manifest_dir}/{manifest}.yml", "w") as manifest_file: - yaml.dump(test_manifests[manifest], manifest_file) - return manifest_dir - - -@pytest.fixture() -def populated_nested_manifest_dir(test_manifests: Dict, tmp_path: str) -> str: - manifest_dir = f"{tmp_path}/populated_nested_manifest" - os.mkdir(manifest_dir) - for manifest in test_manifests.keys(): - nested_manifest_dir = f"{manifest_dir}/{manifest}" - os.mkdir(nested_manifest_dir) - with open(f"{nested_manifest_dir}/{manifest}.yml", "w") as manifest_file: - yaml.dump(test_manifests[manifest], manifest_file) - return manifest_dir - - -@pytest.fixture -def db() -> Generator: - create_citext_extension(sync_engine) - - session = sync_session() - - yield session - session.close() - - -@pytest.fixture(scope="session") -@pytest.mark.asyncio -async def async_session(test_client) -> AsyncSession: - assert CONFIG.test_mode - assert requests.post == test_client.post - - create_citext_extension(sync_engine) - - async_engine = create_async_engine( - CONFIG.database.async_database_uri, - echo=False, - ) - - session_maker = sessionmaker( - async_engine, class_=AsyncSession, expire_on_commit=False - ) - - async with session_maker() as session: - yield session - session.close() - async_engine.dispose() - - -def generate_auth_header_for_user( - user: FidesUser, scopes: list[str], test_config: FidesConfig -) -> Dict[str, str]: - payload = { - JWE_PAYLOAD_SCOPES: scopes, - JWE_PAYLOAD_CLIENT_ID: user.client.id, - JWE_ISSUED_AT: datetime.now().isoformat(), - } - jwe = generate_jwe(json.dumps(payload), test_config.security.app_encryption_key) - return {"Authorization": "Bearer " + jwe} - - -@pytest.fixture(autouse=True) -def clear_get_config_cache() -> None: - get_config.cache_clear() - - -@pytest.fixture(scope="session", autouse=True) -def event_loop() -> Generator: - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() diff --git a/tests/ctl/core/test_dataset.py b/tests/ctl/core/test_dataset.py index 7021b99107..a857bb3701 100644 --- a/tests/ctl/core/test_dataset.py +++ b/tests/ctl/core/test_dataset.py @@ -163,7 +163,7 @@ def connection_config( @pytest.mark.unit async def test_upsert_db_datasets( - test_config: FidesConfig, db: Session, connection_config, async_session + test_config: FidesConfig, ctl_db: Session, connection_config, async_session ) -> None: """ Upsert a CTL Dataset, link this to a DatasetConfig and then upsert that CTL Dataset again. @@ -227,7 +227,7 @@ async def test_upsert_db_datasets( # Create a DatasetConfig that links to the created CTL Dataset dataset_config = DatasetConfig.create( - db=db, + db=ctl_db, data={ "connection_config_id": connection_config.id, "fides_key": "new_fides_key", @@ -250,7 +250,7 @@ async def test_upsert_db_datasets( assert resp.json()["inserted"] == 0 assert resp.json()["updated"] == 1 - db.refresh(dataset_config) + ctl_db.refresh(dataset_config) assert dataset_config.ctl_dataset.name == "new name" assert dataset_config.ctl_dataset.id == ctl_dataset_id, "Id unchanged with upsert" diff --git a/tests/ops/fixtures/__init__.py b/tests/fixtures/__init__.py similarity index 100% rename from tests/ops/fixtures/__init__.py rename to tests/fixtures/__init__.py diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py similarity index 100% rename from tests/ops/fixtures/application_fixtures.py rename to tests/fixtures/application_fixtures.py diff --git a/tests/ops/fixtures/bigquery_fixtures.py b/tests/fixtures/bigquery_fixtures.py similarity index 100% rename from tests/ops/fixtures/bigquery_fixtures.py rename to tests/fixtures/bigquery_fixtures.py diff --git a/tests/ops/fixtures/email_fixtures.py b/tests/fixtures/email_fixtures.py similarity index 100% rename from tests/ops/fixtures/email_fixtures.py rename to tests/fixtures/email_fixtures.py diff --git a/tests/ops/fixtures/fides_connector_example_fixtures.py b/tests/fixtures/fides_connector_example_fixtures.py similarity index 100% rename from tests/ops/fixtures/fides_connector_example_fixtures.py rename to tests/fixtures/fides_connector_example_fixtures.py diff --git a/tests/ops/fixtures/integration_fixtures.py b/tests/fixtures/integration_fixtures.py similarity index 100% rename from tests/ops/fixtures/integration_fixtures.py rename to tests/fixtures/integration_fixtures.py diff --git a/tests/ops/fixtures/manual_fixtures.py b/tests/fixtures/manual_fixtures.py similarity index 100% rename from tests/ops/fixtures/manual_fixtures.py rename to tests/fixtures/manual_fixtures.py diff --git a/tests/ops/fixtures/manual_webhook_fixtures.py b/tests/fixtures/manual_webhook_fixtures.py similarity index 100% rename from tests/ops/fixtures/manual_webhook_fixtures.py rename to tests/fixtures/manual_webhook_fixtures.py diff --git a/tests/ops/fixtures/mariadb_fixtures.py b/tests/fixtures/mariadb_fixtures.py similarity index 100% rename from tests/ops/fixtures/mariadb_fixtures.py rename to tests/fixtures/mariadb_fixtures.py diff --git a/tests/ops/fixtures/mongodb_fixtures.py b/tests/fixtures/mongodb_fixtures.py similarity index 100% rename from tests/ops/fixtures/mongodb_fixtures.py rename to tests/fixtures/mongodb_fixtures.py diff --git a/tests/ops/fixtures/mssql_fixtures.py b/tests/fixtures/mssql_fixtures.py similarity index 100% rename from tests/ops/fixtures/mssql_fixtures.py rename to tests/fixtures/mssql_fixtures.py diff --git a/tests/ops/fixtures/mysql_fixtures.py b/tests/fixtures/mysql_fixtures.py similarity index 100% rename from tests/ops/fixtures/mysql_fixtures.py rename to tests/fixtures/mysql_fixtures.py diff --git a/tests/ops/fixtures/postgres_fixtures.py b/tests/fixtures/postgres_fixtures.py similarity index 100% rename from tests/ops/fixtures/postgres_fixtures.py rename to tests/fixtures/postgres_fixtures.py diff --git a/tests/ops/fixtures/privacy_center_config/bad_test_config.json b/tests/fixtures/privacy_center_config/bad_test_config.json similarity index 100% rename from tests/ops/fixtures/privacy_center_config/bad_test_config.json rename to tests/fixtures/privacy_center_config/bad_test_config.json diff --git a/tests/ops/fixtures/privacy_center_config/test_config.json b/tests/fixtures/privacy_center_config/test_config.json similarity index 100% rename from tests/ops/fixtures/privacy_center_config/test_config.json rename to tests/fixtures/privacy_center_config/test_config.json diff --git a/tests/ops/fixtures/redshift_fixtures.py b/tests/fixtures/redshift_fixtures.py similarity index 100% rename from tests/ops/fixtures/redshift_fixtures.py rename to tests/fixtures/redshift_fixtures.py diff --git a/tests/ops/fixtures/saas/__init__.py b/tests/fixtures/saas/__init__.py similarity index 100% rename from tests/ops/fixtures/saas/__init__.py rename to tests/fixtures/saas/__init__.py diff --git a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py b/tests/fixtures/saas/adobe_campaign_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/adobe_campaign_fixtures.py rename to tests/fixtures/saas/adobe_campaign_fixtures.py diff --git a/tests/ops/fixtures/saas/auth0_fixtures.py b/tests/fixtures/saas/auth0_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/auth0_fixtures.py rename to tests/fixtures/saas/auth0_fixtures.py diff --git a/tests/ops/fixtures/saas/braintree_fixtures.py b/tests/fixtures/saas/braintree_fixtures.py similarity index 99% rename from tests/ops/fixtures/saas/braintree_fixtures.py rename to tests/fixtures/saas/braintree_fixtures.py index 0010e9d341..cdc4a579dc 100644 --- a/tests/ops/fixtures/saas/braintree_fixtures.py +++ b/tests/fixtures/saas/braintree_fixtures.py @@ -182,7 +182,7 @@ def braintree_postgres_dataset_config( def braintree_postgres_db(postgres_integration_session): postgres_integration_session = seed_postgres_data( postgres_integration_session, - "./tests/ops/fixtures/saas/external_datasets/braintree.sql", + "./tests/fixtures/saas/external_datasets/braintree.sql", ) yield postgres_integration_session drop_database(postgres_integration_session.bind.url) diff --git a/tests/ops/fixtures/saas/braze_fixtures.py b/tests/fixtures/saas/braze_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/braze_fixtures.py rename to tests/fixtures/saas/braze_fixtures.py diff --git a/tests/ops/fixtures/saas/connection_template_fixtures.py b/tests/fixtures/saas/connection_template_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/connection_template_fixtures.py rename to tests/fixtures/saas/connection_template_fixtures.py diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/fixtures/saas/datadog_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/datadog_fixtures.py rename to tests/fixtures/saas/datadog_fixtures.py diff --git a/tests/ops/fixtures/saas/domo_fixtures.py b/tests/fixtures/saas/domo_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/domo_fixtures.py rename to tests/fixtures/saas/domo_fixtures.py diff --git a/tests/ops/fixtures/saas/doordash_fixtures.py b/tests/fixtures/saas/doordash_fixtures.py similarity index 98% rename from tests/ops/fixtures/saas/doordash_fixtures.py rename to tests/fixtures/saas/doordash_fixtures.py index 3499d1c3c5..81a0b5be7a 100644 --- a/tests/ops/fixtures/saas/doordash_fixtures.py +++ b/tests/fixtures/saas/doordash_fixtures.py @@ -173,7 +173,7 @@ def doordash_postgres_dataset_config( def doordash_postgres_db(postgres_integration_session): postgres_integration_session = seed_postgres_data( postgres_integration_session, - "./tests/ops/fixtures/saas/external_datasets/doordash.sql", + "./tests/fixtures/saas/external_datasets/doordash.sql", ) yield postgres_integration_session drop_database(postgres_integration_session.bind.url) diff --git a/tests/ops/fixtures/saas/external_datasets/braintree.sql b/tests/fixtures/saas/external_datasets/braintree.sql similarity index 100% rename from tests/ops/fixtures/saas/external_datasets/braintree.sql rename to tests/fixtures/saas/external_datasets/braintree.sql diff --git a/tests/ops/fixtures/saas/external_datasets/doordash.sql b/tests/fixtures/saas/external_datasets/doordash.sql similarity index 100% rename from tests/ops/fixtures/saas/external_datasets/doordash.sql rename to tests/fixtures/saas/external_datasets/doordash.sql diff --git a/tests/ops/fixtures/saas/external_datasets/friendbuy.sql b/tests/fixtures/saas/external_datasets/friendbuy.sql similarity index 100% rename from tests/ops/fixtures/saas/external_datasets/friendbuy.sql rename to tests/fixtures/saas/external_datasets/friendbuy.sql diff --git a/tests/ops/fixtures/saas/external_datasets/fullstory.sql b/tests/fixtures/saas/external_datasets/fullstory.sql similarity index 100% rename from tests/ops/fixtures/saas/external_datasets/fullstory.sql rename to tests/fixtures/saas/external_datasets/fullstory.sql diff --git a/tests/ops/fixtures/saas/external_datasets/twilio.sql b/tests/fixtures/saas/external_datasets/twilio.sql similarity index 100% rename from tests/ops/fixtures/saas/external_datasets/twilio.sql rename to tests/fixtures/saas/external_datasets/twilio.sql diff --git a/tests/ops/fixtures/saas/firebase_auth_fixtures.py b/tests/fixtures/saas/firebase_auth_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/firebase_auth_fixtures.py rename to tests/fixtures/saas/firebase_auth_fixtures.py diff --git a/tests/ops/fixtures/saas/friendbuy_fixtures.py b/tests/fixtures/saas/friendbuy_fixtures.py similarity index 99% rename from tests/ops/fixtures/saas/friendbuy_fixtures.py rename to tests/fixtures/saas/friendbuy_fixtures.py index e21da1f563..67fd9af26e 100644 --- a/tests/ops/fixtures/saas/friendbuy_fixtures.py +++ b/tests/fixtures/saas/friendbuy_fixtures.py @@ -176,7 +176,7 @@ def friendbuy_postgres_dataset_config( def friendbuy_postgres_db(postgres_integration_session): postgres_integration_session = seed_postgres_data( postgres_integration_session, - "./tests/ops/fixtures/saas/external_datasets/friendbuy.sql", + "./tests/fixtures/saas/external_datasets/friendbuy.sql", ) yield postgres_integration_session drop_database(postgres_integration_session.bind.url) diff --git a/tests/ops/fixtures/saas/friendbuy_nextgen_fixtures.py b/tests/fixtures/saas/friendbuy_nextgen_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/friendbuy_nextgen_fixtures.py rename to tests/fixtures/saas/friendbuy_nextgen_fixtures.py diff --git a/tests/ops/fixtures/saas/fullstory_fixtures.py b/tests/fixtures/saas/fullstory_fixtures.py similarity index 99% rename from tests/ops/fixtures/saas/fullstory_fixtures.py rename to tests/fixtures/saas/fullstory_fixtures.py index 9879849c01..f0d384dd56 100644 --- a/tests/ops/fixtures/saas/fullstory_fixtures.py +++ b/tests/fixtures/saas/fullstory_fixtures.py @@ -179,7 +179,7 @@ def fullstory_postgres_dataset_config( def fullstory_postgres_db(postgres_integration_session): postgres_integration_session = seed_postgres_data( postgres_integration_session, - "./tests/ops/fixtures/saas/external_datasets/fullstory.sql", + "./tests/fixtures/saas/external_datasets/fullstory.sql", ) yield postgres_integration_session drop_database(postgres_integration_session.bind.url) diff --git a/tests/ops/fixtures/saas/google_analytics_fixtures.py b/tests/fixtures/saas/google_analytics_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/google_analytics_fixtures.py rename to tests/fixtures/saas/google_analytics_fixtures.py diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/hubspot_fixtures.py rename to tests/fixtures/saas/hubspot_fixtures.py diff --git a/tests/ops/fixtures/saas/mailchimp_fixtures.py b/tests/fixtures/saas/mailchimp_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/mailchimp_fixtures.py rename to tests/fixtures/saas/mailchimp_fixtures.py diff --git a/tests/ops/fixtures/saas/mailchimp_override_fixtures.py b/tests/fixtures/saas/mailchimp_override_fixtures.py similarity index 98% rename from tests/ops/fixtures/saas/mailchimp_override_fixtures.py rename to tests/fixtures/saas/mailchimp_override_fixtures.py index 057d17b981..fb53a5304f 100644 --- a/tests/ops/fixtures/saas/mailchimp_override_fixtures.py +++ b/tests/fixtures/saas/mailchimp_override_fixtures.py @@ -16,7 +16,7 @@ from fides.api.ops.service.connectors.saas_connector import SaaSConnector from fides.api.ops.util.saas_util import load_config from fides.lib.db import session -from tests.ops.fixtures.application_fixtures import load_dataset +from tests.fixtures.application_fixtures import load_dataset @pytest.fixture diff --git a/tests/ops/fixtures/saas/mailchimp_transactional_fixtures.py b/tests/fixtures/saas/mailchimp_transactional_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/mailchimp_transactional_fixtures.py rename to tests/fixtures/saas/mailchimp_transactional_fixtures.py diff --git a/tests/ops/fixtures/saas/outreach_fixtures.py b/tests/fixtures/saas/outreach_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/outreach_fixtures.py rename to tests/fixtures/saas/outreach_fixtures.py diff --git a/tests/ops/fixtures/saas/recharge_fixtures.py b/tests/fixtures/saas/recharge_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/recharge_fixtures.py rename to tests/fixtures/saas/recharge_fixtures.py diff --git a/tests/ops/fixtures/saas/rollbar_fixtures.py b/tests/fixtures/saas/rollbar_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/rollbar_fixtures.py rename to tests/fixtures/saas/rollbar_fixtures.py diff --git a/tests/ops/fixtures/saas/salesforce_fixtures.py b/tests/fixtures/saas/salesforce_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/salesforce_fixtures.py rename to tests/fixtures/saas/salesforce_fixtures.py diff --git a/tests/ops/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/segment_fixtures.py rename to tests/fixtures/saas/segment_fixtures.py diff --git a/tests/ops/fixtures/saas/sendgrid_fixtures.py b/tests/fixtures/saas/sendgrid_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/sendgrid_fixtures.py rename to tests/fixtures/saas/sendgrid_fixtures.py diff --git a/tests/ops/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/sentry_fixtures.py rename to tests/fixtures/saas/sentry_fixtures.py diff --git a/tests/ops/fixtures/saas/shopify_fixtures.py b/tests/fixtures/saas/shopify_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/shopify_fixtures.py rename to tests/fixtures/saas/shopify_fixtures.py diff --git a/tests/ops/fixtures/saas/slack_enterprise_fixtures.py b/tests/fixtures/saas/slack_enterprise_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/slack_enterprise_fixtures.py rename to tests/fixtures/saas/slack_enterprise_fixtures.py diff --git a/tests/ops/fixtures/saas/square_fixtures.py b/tests/fixtures/saas/square_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/square_fixtures.py rename to tests/fixtures/saas/square_fixtures.py diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/stripe_fixtures.py rename to tests/fixtures/saas/stripe_fixtures.py diff --git a/tests/ops/fixtures/saas/twilio_conversations_fixtures.py b/tests/fixtures/saas/twilio_conversations_fixtures.py similarity index 99% rename from tests/ops/fixtures/saas/twilio_conversations_fixtures.py rename to tests/fixtures/saas/twilio_conversations_fixtures.py index f55015bb5f..aa15eaf508 100644 --- a/tests/ops/fixtures/saas/twilio_conversations_fixtures.py +++ b/tests/fixtures/saas/twilio_conversations_fixtures.py @@ -180,7 +180,7 @@ def twilio_postgres_dataset_config( def twilio_postgres_db(postgres_integration_session): postgres_integration_session = seed_postgres_data( postgres_integration_session, - "./tests/ops/fixtures/saas/external_datasets/twilio.sql", + "./tests/fixtures/saas/external_datasets/twilio.sql", ) yield postgres_integration_session drop_database(postgres_integration_session.bind.url) diff --git a/tests/ops/fixtures/saas/universal_analytics_fixtures.py b/tests/fixtures/saas/universal_analytics_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/universal_analytics_fixtures.py rename to tests/fixtures/saas/universal_analytics_fixtures.py diff --git a/tests/ops/fixtures/saas/wunderkind_fixtures.py b/tests/fixtures/saas/wunderkind_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/wunderkind_fixtures.py rename to tests/fixtures/saas/wunderkind_fixtures.py diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/fixtures/saas/zendesk_fixtures.py similarity index 100% rename from tests/ops/fixtures/saas/zendesk_fixtures.py rename to tests/fixtures/saas/zendesk_fixtures.py diff --git a/tests/ops/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py similarity index 99% rename from tests/ops/fixtures/saas_example_fixtures.py rename to tests/fixtures/saas_example_fixtures.py index 1a51982456..9cabb01a7d 100644 --- a/tests/ops/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -30,7 +30,7 @@ from fides.api.ops.util.data_category import DataCategory from fides.api.ops.util.saas_util import load_config from fides.lib.models.client import ClientDetail -from tests.ops.fixtures.application_fixtures import load_dataset +from tests.fixtures.application_fixtures import load_dataset @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/snowflake_fixtures.py b/tests/fixtures/snowflake_fixtures.py similarity index 100% rename from tests/ops/fixtures/snowflake_fixtures.py rename to tests/fixtures/snowflake_fixtures.py diff --git a/tests/ops/fixtures/timescale_fixtures.py b/tests/fixtures/timescale_fixtures.py similarity index 100% rename from tests/ops/fixtures/timescale_fixtures.py rename to tests/fixtures/timescale_fixtures.py diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py deleted file mode 100644 index 1e6fa5c6db..0000000000 --- a/tests/lib/conftest.py +++ /dev/null @@ -1,154 +0,0 @@ -# pylint: disable=missing-function-docstring, redefined-outer-name - -import json -import os -from datetime import datetime -from pathlib import Path -from uuid import uuid4 - -import pytest -from fastapi import FastAPI -from sqlalchemy_utils.functions import create_database, database_exists -from starlette.testclient import TestClient - -from fides.core.config import get_config -from fides.lib.cryptography.schemas.jwt import ( - JWE_ISSUED_AT, - JWE_PAYLOAD_CLIENT_ID, - JWE_PAYLOAD_SCOPES, -) -from fides.lib.db.base import Base # type: ignore[attr-defined] -from fides.lib.db.session import get_db_engine, get_db_session -from fides.lib.models.client import ClientDetail -from fides.lib.models.fides_user import FidesUser -from fides.lib.models.fides_user_permissions import FidesUserPermissions -from fides.lib.oauth.api.routes.user_endpoints import router -from fides.lib.oauth.jwt import generate_jwe -from fides.lib.oauth.scopes import PRIVACY_REQUEST_READ, SCOPES -from tests.conftest import create_citext_extension - -ROOT_PATH = Path().absolute() - - -@pytest.fixture -def db(config): - """Yield a connection to the test DB.""" - # Included so that `AccessManualWebhook` can be located when - # `ConnectionConfig` is instantiated. - from fides.api.ops.models.manual_webhook import ( # pylint: disable=unused-import - AccessManualWebhook, - ) - - # Create the test DB engine - assert config.is_test_mode - engine = get_db_engine( - database_uri=config.database.sqlalchemy_database_uri, - ) - - create_citext_extension(engine) - - if not database_exists(engine.url): - create_database(engine.url) - # Create the database tables - Base.metadata.create_all(engine) - - SessionLocal = get_db_session(config, engine=engine) - session = SessionLocal() - yield session - - session.close() - - engine.dispose() - - -@pytest.fixture(autouse=True, scope="session") -def env_vars(): - os.environ["TESTING"] = "True" - - -@pytest.fixture(scope="session") -def client(): - """Starlette test client to use in testing API routes.""" - app = FastAPI() - app.include_router(router) - with TestClient(app) as client: - yield client - - -@pytest.fixture -def fides_toml_path(): - yield ROOT_PATH / ".fides" / "fides.toml" - - -@pytest.fixture -def oauth_client(db): - """Return a client for authentication purposes.""" - client = ClientDetail( - hashed_secret="thisisatest", - salt="thisisstillatest", - scopes=SCOPES, - ) - db.add(client) - db.commit() - db.refresh(client) - yield client - client.delete(db) - - -@pytest.fixture -def auth_header(request, oauth_client, config): - client_id = oauth_client.id - - payload = { - JWE_PAYLOAD_SCOPES: request.param, - JWE_PAYLOAD_CLIENT_ID: client_id, - JWE_ISSUED_AT: datetime.now().isoformat(), - } - jwe = generate_jwe(json.dumps(payload), config.security.app_encryption_key) - - return {"Authorization": "Bearer " + jwe} - - -@pytest.fixture -def user(db): - user = FidesUser.create( - db=db, - data={ - "username": "test_fidesops_user", - "password": "TESTdcnG@wzJeu0&%3Qe2fGo7", - }, - ) - client = ClientDetail( - hashed_secret="thisisatest", - salt="thisisstillatest", - scopes=SCOPES, - user_id=user.id, - ) - - FidesUserPermissions.create( - db=db, data={"user_id": user.id, "scopes": [PRIVACY_REQUEST_READ]} - ) - - db.add(client) - db.commit() - db.refresh(client) - yield user - user.delete(db) - - -@pytest.fixture(scope="function") -def application_user(db, oauth_client): - unique_username = f"user-{uuid4()}" - user = FidesUser.create( - db=db, - data={ - "username": unique_username, - "password": "test_password", - "first_name": "Test", - "last_name": "User", - }, - ) - oauth_client.user_id = user.id - oauth_client.save(db=db) - yield user - user.delete(db) diff --git a/tests/lib/test_client_model.py b/tests/lib/test_client_model.py index 9cc4e25d67..9c56681dc9 100644 --- a/tests/lib/test_client_model.py +++ b/tests/lib/test_client_model.py @@ -4,9 +4,9 @@ import pytest +from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY from fides.lib.cryptography.cryptographic_util import hash_with_salt from fides.lib.models.client import ClientDetail, _get_root_client_detail -from fides.lib.oauth.scopes import SCOPES def test_create_client_and_secret(db, config): @@ -30,7 +30,7 @@ def test_get_client(db, oauth_client, config): client = ClientDetail.get(db, object_id=oauth_client.id, config=config) assert client assert client.id == oauth_client.id - assert client.scopes == SCOPES + assert client.scopes == SCOPE_REGISTRY assert oauth_client.hashed_secret == "thisisatest" @@ -39,11 +39,11 @@ def test_get_client_root_client(db, config): db, object_id="fidesadmin", config=config, - scopes=SCOPES, + scopes=SCOPE_REGISTRY, ) assert client assert client.id == config.security.oauth_root_client_id - assert client.scopes == SCOPES + assert client.scopes == SCOPE_REGISTRY def test_get_client_root_client_no_scopes(db, config): @@ -57,7 +57,7 @@ def test_credentials_valid(db, config): db, config.security.oauth_client_id_length_bytes, config.security.oauth_client_secret_length_bytes, - scopes=SCOPES, + scopes=SCOPE_REGISTRY, ) assert new_client.credentials_valid("this-is-not-the-right-secret") is False @@ -68,4 +68,4 @@ def test_get_root_client_detail_no_root_client_hash(config): test_config = deepcopy(config) test_config.security.oauth_root_client_secret_hash = None with pytest.raises(ValueError): - _get_root_client_detail(test_config, SCOPES) + _get_root_client_detail(test_config, SCOPE_REGISTRY) diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 74b206316d..c0589a1ae8 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -33,7 +33,7 @@ from fides.api.ops.schemas.messaging.messaging import MessagingActionType from fides.api.ops.schemas.redis_cache import Identity from fides.lib.models.client import ClientDetail -from tests.ops.fixtures.application_fixtures import integration_secrets +from tests.fixtures.application_fixtures import integration_secrets page_size = Params().size diff --git a/tests/ops/api/v1/endpoints/test_user_endpoints.py b/tests/ops/api/v1/endpoints/test_user_endpoints.py index f23ef4cb95..3500034687 100644 --- a/tests/ops/api/v1/endpoints/test_user_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_endpoints.py @@ -44,7 +44,7 @@ from fides.lib.models.fides_user_permissions import FidesUserPermissions from fides.lib.oauth.jwt import generate_jwe from fides.lib.oauth.oauth_util import extract_payload -from tests.ops.conftest import generate_auth_header_for_user +from tests.conftest import generate_auth_header_for_user CONFIG = get_config() page_size = Params().size diff --git a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py index 16058db17e..dcdb29ef53 100644 --- a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py @@ -23,7 +23,7 @@ from fides.lib.models.client import ClientDetail from fides.lib.models.fides_user import FidesUser from fides.lib.models.fides_user_permissions import FidesUserPermissions -from tests.ops.conftest import generate_auth_header_for_user +from tests.conftest import generate_auth_header_for_user CONFIG = get_config() diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py deleted file mode 100644 index 1782bd6900..0000000000 --- a/tests/ops/conftest.py +++ /dev/null @@ -1,333 +0,0 @@ -# pylint: disable=unused-wildcard-import, wildcard-import - -import asyncio -import json -from typing import Any, Callable, Dict, Generator, List - -import pytest -import requests -from fastapi.testclient import TestClient -from httpx import AsyncClient -from sqlalchemy.exc import IntegrityError -from toml import load as load_toml - -from fides.api.main import app -from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY -from fides.api.ops.db.base import Base -from fides.api.ops.models.privacy_request import generate_request_callback_jwe -from fides.api.ops.tasks.scheduled.scheduler import scheduler -from fides.api.ops.util.cache import get_cache -from fides.core.api import db_action -from fides.core.config import get_config -from fides.lib.cryptography.schemas.jwt import ( - JWE_ISSUED_AT, - JWE_PAYLOAD_CLIENT_ID, - JWE_PAYLOAD_SCOPES, -) -from fides.lib.db.session import Session, get_db_engine, get_db_session -from fides.lib.models.client import ClientDetail -from fides.lib.oauth.jwt import generate_jwe -from tests.conftest import create_citext_extension - -from .fixtures.application_fixtures import * -from .fixtures.bigquery_fixtures import * -from .fixtures.email_fixtures import * -from .fixtures.fides_connector_example_fixtures import * -from .fixtures.integration_fixtures import * -from .fixtures.manual_fixtures import * -from .fixtures.manual_webhook_fixtures import * -from .fixtures.mariadb_fixtures import * -from .fixtures.mongodb_fixtures import * -from .fixtures.mssql_fixtures import * -from .fixtures.mysql_fixtures import * -from .fixtures.postgres_fixtures import * -from .fixtures.redshift_fixtures import * -from .fixtures.saas import * -from .fixtures.saas_example_fixtures import * -from .fixtures.snowflake_fixtures import * -from .fixtures.timescale_fixtures import * - -CONFIG = get_config() - - -@pytest.fixture(scope="session", autouse=True) -def setup_db(api_client): - """Apply migrations at beginning and end of testing session""" - assert CONFIG.test_mode - assert requests.post != api_client.post - yield api_client.post(url=f"{CONFIG.cli.server_url}/v1/admin/db/reset") - - -@pytest.fixture(scope="session") -def db(api_client) -> Generator: - """Return a connection to the test DB""" - # Create the test DB engine - assert CONFIG.test_mode - assert requests.post != api_client.post - engine = get_db_engine( - database_uri=CONFIG.database.sqlalchemy_test_database_uri, - ) - - create_citext_extension(engine) - - if not scheduler.running: - scheduler.start() - SessionLocal = get_db_session(CONFIG, engine=engine) - the_session = SessionLocal() - # Setup above... - - yield the_session - # Teardown below... - the_session.close() - engine.dispose() - - -@pytest.fixture(autouse=True) -def clear_db_tables(db): - """Clear data from tables between tests. - - If relationships are not set to cascade on delete they will fail with an - IntegrityError if there are relationsips present. This function stores tables - that fail with this error then recursively deletes until no more IntegrityErrors - are present. - """ - yield - - def delete_data(tables): - redo = [] - for table in tables: - try: - db.execute(table.delete()) - except IntegrityError: - redo.append(table) - finally: - db.commit() - - if redo: - delete_data(redo) - - db.commit() # make sure all transactions are closed before starting deletes - delete_data(Base.metadata.sorted_tables) - - -@pytest.fixture(scope="session") -def cache() -> Generator: - yield get_cache() - - -@pytest.fixture(scope="session") -def api_client() -> Generator: - """Return a client used to make API requests""" - with TestClient(app) as c: - yield c - - -@pytest.fixture(scope="session") -async def async_api_client() -> Generator: - """Return an async client used to make API requests""" - async with AsyncClient( - app=app, base_url="http://0.0.0.0:8080", follow_redirects=True - ) as client: - yield client - - -@pytest.fixture(scope="session", autouse=True) -def event_loop() -> Generator: - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() - - -@pytest.fixture(scope="function") -def oauth_client(db: Session) -> Generator: - """Return a client for authentication purposes""" - client = ClientDetail( - hashed_secret="thisisatest", - salt="thisisstillatest", - scopes=SCOPE_REGISTRY, - ) - db.add(client) - db.commit() - db.refresh(client) - yield client - - -@pytest.fixture(scope="function") -def oauth_root_client(db: Session) -> ClientDetail: - """Return the configured root client (never persisted)""" - return ClientDetail.get( - db, - object_id=CONFIG.security.oauth_root_client_id, - config=CONFIG, - scopes=SCOPE_REGISTRY, - ) - - -@pytest.fixture(scope="function") -def root_auth_header(oauth_root_client: ClientDetail) -> Dict[str, str]: - """Return an auth header for the root client""" - payload = { - JWE_PAYLOAD_SCOPES: oauth_root_client.scopes, - JWE_PAYLOAD_CLIENT_ID: oauth_root_client.id, - JWE_ISSUED_AT: datetime.now().isoformat(), - } - jwe = generate_jwe(json.dumps(payload), CONFIG.security.app_encryption_key) - return {"Authorization": "Bearer " + jwe} - - -def generate_auth_header_for_user(user, scopes) -> Dict[str, str]: - payload = { - JWE_PAYLOAD_SCOPES: scopes, - JWE_PAYLOAD_CLIENT_ID: user.client.id, - JWE_ISSUED_AT: datetime.now().isoformat(), - } - jwe = generate_jwe(json.dumps(payload), CONFIG.security.app_encryption_key) - return {"Authorization": "Bearer " + jwe} - - -@pytest.fixture(scope="function") -def generate_auth_header(oauth_client) -> Callable[[Any], Dict[str, str]]: - return _generate_auth_header(oauth_client, CONFIG.security.app_encryption_key) - - -@pytest.fixture -def generate_auth_header_ctl_config(oauth_client) -> Callable[[Any], Dict[str, str]]: - return _generate_auth_header(oauth_client, CONFIG.security.app_encryption_key) - - -def _generate_auth_header( - oauth_client, app_encryption_key -) -> Callable[[Any], Dict[str, str]]: - client_id = oauth_client.id - - def _build_jwt(scopes: List[str]) -> Dict[str, str]: - payload = { - JWE_PAYLOAD_SCOPES: scopes, - JWE_PAYLOAD_CLIENT_ID: client_id, - JWE_ISSUED_AT: datetime.now().isoformat(), - } - jwe = generate_jwe(json.dumps(payload), app_encryption_key) - return {"Authorization": "Bearer " + jwe} - - return _build_jwt - - -@pytest.fixture(scope="function") -def generate_webhook_auth_header() -> Callable[[Any], Dict[str, str]]: - def _build_jwt(webhook: PolicyPreWebhook) -> Dict[str, str]: - jwe = generate_request_callback_jwe(webhook) - return {"Authorization": "Bearer " + jwe} - - return _build_jwt - - -@pytest.fixture(scope="session") -def integration_config(): - yield load_toml("tests/ops/integration_test_config.toml") - - -@pytest.fixture(scope="session") -def celery_config(): - return {"task_always_eager": False} - - -@pytest.fixture(autouse=True, scope="session") -def celery_enable_logging(): - """Turns on celery output logs.""" - return True - - -@pytest.fixture(autouse=True, scope="session") -def celery_use_virtual_worker(celery_session_worker): - """ - This is a catch-all fixture that forces all of our - tests to use a virtual celery worker if a registered - task is executed within the scope of the test. - """ - yield celery_session_worker - - -@pytest.fixture(scope="session") -def run_privacy_request_task(celery_session_app): - """ - This fixture is the version of the run_privacy_request task that is - registered to the `celery_app` fixture which uses the virtualised `celery_worker` - """ - yield celery_session_app.tasks[ - "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request" - ] - - -@pytest.fixture(autouse=True, scope="session") -def analytics_opt_out(): - """Disable sending analytics when running tests.""" - original_value = CONFIG.user.analytics_opt_out - CONFIG.user.analytics_opt_out = True - yield - CONFIG.user.analytics_opt_out = original_value - - -@pytest.fixture(scope="function") -def require_manual_request_approval(): - """Require manual request approval""" - original_value = CONFIG.execution.require_manual_request_approval - CONFIG.execution.require_manual_request_approval = True - yield - CONFIG.execution.require_manual_request_approval = original_value - - -@pytest.fixture(scope="function") -def subject_identity_verification_required(): - """Enable identity verification.""" - original_value = CONFIG.execution.subject_identity_verification_required - CONFIG.execution.subject_identity_verification_required = True - yield - CONFIG.execution.subject_identity_verification_required = original_value - - -@pytest.fixture(autouse=True, scope="function") -def subject_identity_verification_not_required(): - """Disable identity verification for most tests unless overridden""" - original_value = CONFIG.execution.subject_identity_verification_required - CONFIG.execution.subject_identity_verification_required = False - yield - CONFIG.execution.subject_identity_verification_required = original_value - - -@pytest.fixture(autouse=True, scope="function") -def privacy_request_complete_email_notification_disabled(): - """Disable request completion email for most tests unless overridden""" - original_value = CONFIG.notifications.send_request_completion_notification - CONFIG.notifications.send_request_completion_notification = False - yield - CONFIG.notifications.send_request_completion_notification = original_value - - -@pytest.fixture(autouse=True, scope="function") -def privacy_request_receipt_notification_disabled(): - """Disable request receipt notification for most tests unless overridden""" - original_value = CONFIG.notifications.send_request_receipt_notification - CONFIG.notifications.send_request_receipt_notification = False - yield - CONFIG.notifications.send_request_receipt_notification = original_value - - -@pytest.fixture(autouse=True, scope="function") -def privacy_request_review_notification_disabled(): - """Disable request review notification for most tests unless overridden""" - original_value = CONFIG.notifications.send_request_review_notification - CONFIG.notifications.send_request_review_notification = False - yield - CONFIG.notifications.send_request_review_notification = original_value - - -@pytest.fixture(scope="function", autouse=True) -def set_notification_service_type_mailgun(): - """Set default notification service type""" - original_value = CONFIG.notifications.notification_service_type - CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value - yield - CONFIG.notifications.notification_service_type = original_value diff --git a/tests/ops/graph/graph_test_util.py b/tests/ops/graph/graph_test_util.py index 8b5181b971..ebad36d762 100644 --- a/tests/ops/graph/graph_test_util.py +++ b/tests/ops/graph/graph_test_util.py @@ -18,8 +18,7 @@ from fides.api.ops.task.task_resources import TaskResources from fides.api.ops.util.collection_util import Row from fides.lib.db.base_class import FidesBase - -from ..fixtures.application_fixtures import faker +from tests.fixtures.application_fixtures import faker class MockResources(TaskResources): diff --git a/tests/ops/integration_tests/saas/test_fullstory_task.py b/tests/ops/integration_tests/saas/test_fullstory_task.py index 6e2a928a1f..12920c2eaa 100644 --- a/tests/ops/integration_tests/saas/test_fullstory_task.py +++ b/tests/ops/integration_tests/saas/test_fullstory_task.py @@ -9,7 +9,7 @@ from fides.api.ops.task import graph_task from fides.api.ops.task.graph_task import get_cached_data_for_erasures from fides.core.config import get_config -from tests.ops.fixtures.saas.fullstory_fixtures import FullstoryTestClient, user_updated +from tests.fixtures.saas.fullstory_fixtures import FullstoryTestClient, user_updated from tests.ops.graph.graph_test_util import assert_rows_match from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index f038901dcd..909f0f2e82 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -10,7 +10,7 @@ from fides.api.ops.task.filter_results import filter_data_categories from fides.api.ops.task.graph_task import get_cached_data_for_erasures from fides.core.config import get_config -from tests.ops.fixtures.saas.hubspot_fixtures import HubspotTestClient, user_exists +from tests.fixtures.saas.hubspot_fixtures import HubspotTestClient, user_exists from tests.ops.graph.graph_test_util import assert_rows_match from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/integration_tests/saas/test_sendgrid_task.py b/tests/ops/integration_tests/saas/test_sendgrid_task.py index da946a13a8..3b83f52af7 100644 --- a/tests/ops/integration_tests/saas/test_sendgrid_task.py +++ b/tests/ops/integration_tests/saas/test_sendgrid_task.py @@ -9,7 +9,7 @@ from fides.api.ops.task import graph_task from fides.api.ops.task.graph_task import get_cached_data_for_erasures from fides.core.config import get_config -from tests.ops.fixtures.saas.sendgrid_fixtures import contact_exists +from tests.fixtures.saas.sendgrid_fixtures import contact_exists from tests.ops.graph.graph_test_util import assert_rows_match from tests.ops.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/ops/integration_tests/test_execution.py b/tests/ops/integration_tests/test_execution.py index f1831ef7f7..5846d9a586 100644 --- a/tests/ops/integration_tests/test_execution.py +++ b/tests/ops/integration_tests/test_execution.py @@ -24,8 +24,8 @@ from fides.api.ops.task.graph_task import get_cached_data_for_erasures from fides.core.config import get_config from fides.lib.db.session import get_db_session +from tests.fixtures.application_fixtures import integration_secrets -from ..fixtures.application_fixtures import integration_secrets from ..service.privacy_request.request_runner_service_test import ( get_privacy_request_results, ) diff --git a/tests/ops/service/privacy_request/test_consent_email_batch_send.py b/tests/ops/service/privacy_request/test_consent_email_batch_send.py index 45fcc73e20..66072a5bdc 100644 --- a/tests/ops/service/privacy_request/test_consent_email_batch_send.py +++ b/tests/ops/service/privacy_request/test_consent_email_batch_send.py @@ -30,7 +30,7 @@ ) from fides.core.config import get_config from fides.lib.models.audit_log import AuditLog, AuditLogAction -from tests.ops.fixtures.application_fixtures import _create_privacy_request_for_policy +from tests.fixtures.application_fixtures import _create_privacy_request_for_policy CONFIG = get_config() diff --git a/tests/ops/util/test_cache.py b/tests/ops/util/test_cache.py index 32a3244b4e..9842e55243 100644 --- a/tests/ops/util/test_cache.py +++ b/tests/ops/util/test_cache.py @@ -3,8 +3,7 @@ from fides.api.ops.util.cache import FidesopsRedis from fides.core.config import get_config - -from ..fixtures.application_fixtures import faker +from tests.fixtures.application_fixtures import faker CONFIG = get_config() From 08590ca316340637ab41513dab72abe9ef047563 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Tue, 21 Feb 2023 16:21:08 -0500 Subject: [PATCH 2/7] Fix database issues --- tests/conftest.py | 147 +++++++++++++++++---------------- tests/ctl/conftest.py | 31 +++++++ tests/ctl/core/test_dataset.py | 6 +- tests/ops/conftest.py | 42 ++++++++++ 4 files changed, 151 insertions(+), 75 deletions(-) create mode 100644 tests/ctl/conftest.py create mode 100644 tests/ops/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py index 74c33b2f16..d2906d4f58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,15 +19,17 @@ from sqlalchemy.orm import sessionmaker from toml import load as load_toml -from fides.api.ctl.database.session import sync_engine, sync_session +# from fides.api.ctl.database.session import sync_engine, sync_session from fides.api.main import app from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY from fides.api.ops.db.base import Base from fides.api.ops.models.privacy_request import generate_request_callback_jwe from fides.api.ops.schemas.messaging.messaging import MessagingServiceType -from fides.api.ops.tasks.scheduled.scheduler import scheduler + +# from fides.api.ops.tasks.scheduled.scheduler import scheduler from fides.api.ops.util.cache import get_cache -from fides.core import api + +# from fides.core import api from fides.core.config import get_config from fides.lib.cryptography.schemas.jwt import ( JWE_ISSUED_AT, @@ -122,22 +124,23 @@ async def async_session(test_client): async_engine.dispose() -@pytest.fixture(scope="session", autouse=True) -def setup_ctl_db(test_config, test_client, monkeypatch_requests): - "Sets up the database for testing." - assert CONFIG.test_mode - assert ( - requests.post == test_client.post - ) # Sanity check to make sure monkeypatch_requests fixture has run - yield api.db_action( - server_url=test_config.cli.server_url, - headers=CONFIG.user.auth_header, - action="reset", - ) +# @pytest.fixture(scope="session", autouse=True) +# @pytest.mark.usefixtures("monkeypatch_requests") +# def setup_ctl_db(test_config, test_client): +# "Sets up the database for testing." +# assert CONFIG.test_mode +# assert ( +# requests.post == test_client.post +# ) # Sanity check to make sure monkeypatch_requests fixture has run +# yield api.db_action( +# server_url=test_config.cli.server_url, +# headers=CONFIG.user.auth_header, +# action="reset", +# ) @pytest.fixture -def ctl_db(): +def db(): create_citext_extension(sync_engine) session = sync_session() @@ -146,36 +149,36 @@ def ctl_db(): session.close() -@pytest.fixture(scope="session", autouse=True) -def setup_db(api_client): - """Apply migrations at beginning and end of testing session""" - assert CONFIG.test_mode - assert requests.post != api_client.post - yield api_client.post(url=f"{CONFIG.cli.server_url}/v1/admin/db/reset") - - -@pytest.fixture(scope="session") -def db(api_client): - """Return a connection to the test DB""" - # Create the test DB engine - assert CONFIG.test_mode - assert requests.post != api_client.post - engine = get_db_engine( - database_uri=CONFIG.database.sqlalchemy_test_database_uri, - ) - - create_citext_extension(engine) - - if not scheduler.running: - scheduler.start() - SessionLocal = get_db_session(CONFIG, engine=engine) - the_session = SessionLocal() - # Setup above... - - yield the_session - # Teardown below... - the_session.close() - engine.dispose() +# @pytest.fixture(scope="session", autouse=True) +# def setup_db(api_client): +# """Apply migrations at beginning and end of testing session""" +# assert CONFIG.test_mode +# assert requests.post != api_client.post +# yield api_client.post(url=f"{CONFIG.cli.server_url}/v1/admin/db/reset") + + +# @pytest.fixture(scope="session") +# def db(api_client): +# """Return a connection to the test DB""" +# # Create the test DB engine +# assert CONFIG.test_mode +# assert requests.post != api_client.post +# engine = get_db_engine( +# database_uri=CONFIG.database.sqlalchemy_test_database_uri, +# ) +# +# create_citext_extension(engine) +# +# if not scheduler.running: +# scheduler.start() +# SessionLocal = get_db_session(CONFIG, engine=engine) +# the_session = SessionLocal() +# # Setup above... +# +# yield the_session +# # Teardown below... +# the_session.close() +# engine.dispose() @pytest.fixture(scope="session") @@ -204,32 +207,32 @@ def event_loop(): loop.close() -@pytest.fixture(autouse=True) -def clear_db_tables(db): - """Clear data from tables between tests. - - If relationships are not set to cascade on delete they will fail with an - IntegrityError if there are relationsips present. This function stores tables - that fail with this error then recursively deletes until no more IntegrityErrors - are present. - """ - yield - - def delete_data(tables): - redo = [] - for table in tables: - try: - db.execute(table.delete()) - except IntegrityError: - redo.append(table) - finally: - db.commit() - - if redo: - delete_data(redo) - - db.commit() # make sure all transactions are closed before starting deletes - delete_data(Base.metadata.sorted_tables) +# @pytest.fixture(autouse=True) +# def clear_db_tables(db): +# """Clear data from tables between tests. +# +# If relationships are not set to cascade on delete they will fail with an +# IntegrityError if there are relationsips present. This function stores tables +# that fail with this error then recursively deletes until no more IntegrityErrors +# are present. +# """ +# yield +# +# def delete_data(tables): +# redo = [] +# for table in tables: +# try: +# db.execute(table.delete()) +# except IntegrityError: +# redo.append(table) +# finally: +# db.commit() +# +# if redo: +# delete_data(redo) +# +# db.commit() # make sure all transactions are closed before starting deletes +# delete_data(Base.metadata.sorted_tables) @pytest.fixture(scope="session") diff --git a/tests/ctl/conftest.py b/tests/ctl/conftest.py new file mode 100644 index 0000000000..42f74822b6 --- /dev/null +++ b/tests/ctl/conftest.py @@ -0,0 +1,31 @@ +import pytest +import requests + +from fides.api.ctl.database.session import sync_engine, sync_session +from fides.core import api +from tests.conftest import create_citext_extension + + +@pytest.fixture(scope="session", autouse=True) +@pytest.mark.usefixtures("monkeypatch_requests") +def setup_ctl_db(test_config, test_client, config): + "Sets up the database for testing." + assert config.test_mode + assert ( + requests.post == test_client.post + ) # Sanity check to make sure monkeypatch_requests fixture has run + yield api.db_action( + server_url=test_config.cli.server_url, + headers=config.user.auth_header, + action="reset", + ) + + +@pytest.fixture +def db(): + create_citext_extension(sync_engine) + + session = sync_session() + + yield session + session.close() diff --git a/tests/ctl/core/test_dataset.py b/tests/ctl/core/test_dataset.py index a857bb3701..7021b99107 100644 --- a/tests/ctl/core/test_dataset.py +++ b/tests/ctl/core/test_dataset.py @@ -163,7 +163,7 @@ def connection_config( @pytest.mark.unit async def test_upsert_db_datasets( - test_config: FidesConfig, ctl_db: Session, connection_config, async_session + test_config: FidesConfig, db: Session, connection_config, async_session ) -> None: """ Upsert a CTL Dataset, link this to a DatasetConfig and then upsert that CTL Dataset again. @@ -227,7 +227,7 @@ async def test_upsert_db_datasets( # Create a DatasetConfig that links to the created CTL Dataset dataset_config = DatasetConfig.create( - db=ctl_db, + db=db, data={ "connection_config_id": connection_config.id, "fides_key": "new_fides_key", @@ -250,7 +250,7 @@ async def test_upsert_db_datasets( assert resp.json()["inserted"] == 0 assert resp.json()["updated"] == 1 - ctl_db.refresh(dataset_config) + db.refresh(dataset_config) assert dataset_config.ctl_dataset.name == "new name" assert dataset_config.ctl_dataset.id == ctl_dataset_id, "Id unchanged with upsert" diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py new file mode 100644 index 0000000000..0a0b74d8e0 --- /dev/null +++ b/tests/ops/conftest.py @@ -0,0 +1,42 @@ +import pytest +import requests +from sqlalchemy.exc import IntegrityError + +from fides.api.ops.db.base import Base +from fides.api.ops.tasks.scheduled.scheduler import scheduler + + +@pytest.fixture(scope="session", autouse=True) +def setup_db(api_client, config): + """Apply migrations at beginning and end of testing session""" + assert config.test_mode + assert requests.post != api_client.post + yield api_client.post(url=f"{config.cli.server_url}/v1/admin/db/reset") + + +@pytest.fixture(autouse=True) +def clear_db_tables(db): + """Clear data from tables between tests. + + If relationships are not set to cascade on delete they will fail with an + IntegrityError if there are relationsips present. This function stores tables + that fail with this error then recursively deletes until no more IntegrityErrors + are present. + """ + yield + + def delete_data(tables): + redo = [] + for table in tables: + try: + db.execute(table.delete()) + except IntegrityError: + redo.append(table) + finally: + db.commit() + + if redo: + delete_data(redo) + + db.commit() # make sure all transactions are closed before starting deletes + delete_data(Base.metadata.sorted_tables) From 3be5727503ea7fbff934394d966624b7e4a9ab78 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Wed, 22 Feb 2023 10:04:35 -0500 Subject: [PATCH 3/7] Move database fixtures --- tests/conftest.py | 87 ------------------------------------------- tests/ops/conftest.py | 26 +++++++++++++ 2 files changed, 26 insertions(+), 87 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d2906d4f58..7808304eb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,6 @@ from loguru import logger from pytest import MonkeyPatch from sqlalchemy.engine.base import Engine -from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker from toml import load as load_toml @@ -36,7 +35,6 @@ JWE_PAYLOAD_CLIENT_ID, JWE_PAYLOAD_SCOPES, ) -from fides.lib.db.session import get_db_engine, get_db_session from fides.lib.models.client import ClientDetail from fides.lib.models.fides_user import FidesUser from fides.lib.models.fides_user_permissions import FidesUserPermissions @@ -124,63 +122,6 @@ async def async_session(test_client): async_engine.dispose() -# @pytest.fixture(scope="session", autouse=True) -# @pytest.mark.usefixtures("monkeypatch_requests") -# def setup_ctl_db(test_config, test_client): -# "Sets up the database for testing." -# assert CONFIG.test_mode -# assert ( -# requests.post == test_client.post -# ) # Sanity check to make sure monkeypatch_requests fixture has run -# yield api.db_action( -# server_url=test_config.cli.server_url, -# headers=CONFIG.user.auth_header, -# action="reset", -# ) - - -@pytest.fixture -def db(): - create_citext_extension(sync_engine) - - session = sync_session() - - yield session - session.close() - - -# @pytest.fixture(scope="session", autouse=True) -# def setup_db(api_client): -# """Apply migrations at beginning and end of testing session""" -# assert CONFIG.test_mode -# assert requests.post != api_client.post -# yield api_client.post(url=f"{CONFIG.cli.server_url}/v1/admin/db/reset") - - -# @pytest.fixture(scope="session") -# def db(api_client): -# """Return a connection to the test DB""" -# # Create the test DB engine -# assert CONFIG.test_mode -# assert requests.post != api_client.post -# engine = get_db_engine( -# database_uri=CONFIG.database.sqlalchemy_test_database_uri, -# ) -# -# create_citext_extension(engine) -# -# if not scheduler.running: -# scheduler.start() -# SessionLocal = get_db_session(CONFIG, engine=engine) -# the_session = SessionLocal() -# # Setup above... -# -# yield the_session -# # Teardown below... -# the_session.close() -# engine.dispose() - - @pytest.fixture(scope="session") def api_client(): """Return a client used to make API requests""" @@ -207,34 +148,6 @@ def event_loop(): loop.close() -# @pytest.fixture(autouse=True) -# def clear_db_tables(db): -# """Clear data from tables between tests. -# -# If relationships are not set to cascade on delete they will fail with an -# IntegrityError if there are relationsips present. This function stores tables -# that fail with this error then recursively deletes until no more IntegrityErrors -# are present. -# """ -# yield -# -# def delete_data(tables): -# redo = [] -# for table in tables: -# try: -# db.execute(table.delete()) -# except IntegrityError: -# redo.append(table) -# finally: -# db.commit() -# -# if redo: -# delete_data(redo) -# -# db.commit() # make sure all transactions are closed before starting deletes -# delete_data(Base.metadata.sorted_tables) - - @pytest.fixture(scope="session") def config(): CONFIG.is_test_mode = True diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 0a0b74d8e0..b4930c212b 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -4,6 +4,8 @@ from fides.api.ops.db.base import Base from fides.api.ops.tasks.scheduled.scheduler import scheduler +from fides.lib.db.session import get_db_engine, get_db_session +from tests.conftest import create_citext_extension @pytest.fixture(scope="session", autouse=True) @@ -14,6 +16,30 @@ def setup_db(api_client, config): yield api_client.post(url=f"{config.cli.server_url}/v1/admin/db/reset") +@pytest.fixture(scope="session") +def db(api_client, config): + """Return a connection to the test DB""" + # Create the test DB engine + assert config.test_mode + assert requests.post != api_client.post + engine = get_db_engine( + database_uri=config.database.sqlalchemy_test_database_uri, + ) + + create_citext_extension(engine) + + if not scheduler.running: + scheduler.start() + SessionLocal = get_db_session(config, engine=engine) + the_session = SessionLocal() + # Setup above... + + yield the_session + # Teardown below... + the_session.close() + engine.dispose() + + @pytest.fixture(autouse=True) def clear_db_tables(db): """Clear data from tables between tests. From 3bdd12d57a910b2d1e9a0411b07dd726e4622504 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Wed, 22 Feb 2023 14:28:42 -0500 Subject: [PATCH 4/7] Add lib db fixture --- tests/conftest.py | 2 ++ tests/lib/conftest.py | 65 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tests/lib/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py index 7808304eb0..1724dd59f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,8 @@ from sqlalchemy.orm import sessionmaker from toml import load as load_toml +from fides.api.ctl.database.session import sync_engine + # from fides.api.ctl.database.session import sync_engine, sync_session from fides.api.main import app from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py new file mode 100644 index 0000000000..5d9570ef96 --- /dev/null +++ b/tests/lib/conftest.py @@ -0,0 +1,65 @@ +import pytest +import requests +from sqlalchemy.exc import IntegrityError + +from fides.api.ops.db.base import Base +from fides.api.ops.tasks.scheduled.scheduler import scheduler +from fides.lib.db.session import get_db_engine, get_db_session +from tests.conftest import create_citext_extension + + +@pytest.fixture(scope="session", autouse=True) +def setup_db(api_client, config): + """Apply migrations at beginning and end of testing session""" + assert config.test_mode + assert requests.post != api_client.post + yield api_client.post(url=f"{config.cli.server_url}/v1/admin/db/reset") + + +@pytest.fixture(scope="session") +def db(api_client, config): + """Return a connection to the test DB""" + # Create the test DB engine + assert config.test_mode + assert requests.post != api_client.post + engine = get_db_engine( + database_uri=config.database.sqlalchemy_test_database_uri, + ) + + create_citext_extension(engine) + + SessionLocal = get_db_session(config, engine=engine) + the_session = SessionLocal() + + yield the_session + + the_session.close() + engine.dispose() + + +@pytest.fixture(autouse=True) +def clear_db_tables(db): + """Clear data from tables between tests. + + If relationships are not set to cascade on delete they will fail with an + IntegrityError if there are relationsips present. This function stores tables + that fail with this error then recursively deletes until no more IntegrityErrors + are present. + """ + yield + + def delete_data(tables): + redo = [] + for table in tables: + try: + db.execute(table.delete()) + except IntegrityError: + redo.append(table) + finally: + db.commit() + + if redo: + delete_data(redo) + + db.commit() # make sure all transactions are closed before starting deletes + delete_data(Base.metadata.sorted_tables) From 9dc20ef6ea912680e999a873d17e175043804636 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Wed, 22 Feb 2023 14:51:10 -0500 Subject: [PATCH 5/7] Fix test after merging main --- tests/conftest.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1724dd59f4..d6f6e23c8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,7 @@ # from fides.core import api from fides.core.config import get_config +from fides.core.config.config_proxy import ConfigProxy from fides.lib.cryptography.schemas.jwt import ( JWE_ISSUED_AT, JWE_PAYLOAD_CLIENT_ID, @@ -41,7 +42,6 @@ from fides.lib.models.fides_user import FidesUser from fides.lib.models.fides_user_permissions import FidesUserPermissions from fides.lib.oauth.jwt import generate_jwe -from fides.lib.oauth.scopes import PRIVACY_REQUEST_READ from tests.fixtures.application_fixtures import * from tests.fixtures.bigquery_fixtures import * from tests.fixtures.email_fixtures import * @@ -601,54 +601,71 @@ def require_manual_request_approval(): @pytest.fixture -def subject_identity_verification_required(): +def subject_identity_verification_required(db): """Enable identity verification.""" original_value = CONFIG.execution.subject_identity_verification_required CONFIG.execution.subject_identity_verification_required = True + ApplicationConfig.update_config_set(db, CONFIG) yield CONFIG.execution.subject_identity_verification_required = original_value + ApplicationConfig.update_config_set(db, CONFIG) @pytest.fixture(autouse=True, scope="function") -def subject_identity_verification_not_required(): +def subject_identity_verification_not_required(db): """Disable identity verification for most tests unless overridden""" original_value = CONFIG.execution.subject_identity_verification_required CONFIG.execution.subject_identity_verification_required = False + ApplicationConfig.update_config_set(db, CONFIG) yield CONFIG.execution.subject_identity_verification_required = original_value + ApplicationConfig.update_config_set(db, CONFIG) @pytest.fixture(autouse=True, scope="function") -def privacy_request_complete_email_notification_disabled(): +def privacy_request_complete_email_notification_disabled(db): """Disable request completion email for most tests unless overridden""" original_value = CONFIG.notifications.send_request_completion_notification CONFIG.notifications.send_request_completion_notification = False + ApplicationConfig.update_config_set(db, CONFIG) yield CONFIG.notifications.send_request_completion_notification = original_value + ApplicationConfig.update_config_set(db, CONFIG) @pytest.fixture(autouse=True, scope="function") -def privacy_request_receipt_notification_disabled(): +def privacy_request_receipt_notification_disabled(db): """Disable request receipt notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = False + ApplicationConfig.update_config_set(db, CONFIG) yield CONFIG.notifications.send_request_receipt_notification = original_value + ApplicationConfig.update_config_set(db, CONFIG) @pytest.fixture(autouse=True, scope="function") -def privacy_request_review_notification_disabled(): +def privacy_request_review_notification_disabled(db): """Disable request review notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_review_notification + ApplicationConfig.update_config_set(db, CONFIG) CONFIG.notifications.send_request_review_notification = False yield CONFIG.notifications.send_request_review_notification = original_value + ApplicationConfig.update_config_set(db, CONFIG) @pytest.fixture(scope="function", autouse=True) -def set_notification_service_type_mailgun(): +def set_notification_service_type_mailgun(db): """Set default notification service type""" original_value = CONFIG.notifications.notification_service_type CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value + ApplicationConfig.update_config_set(db, CONFIG) yield CONFIG.notifications.notification_service_type = original_value + ApplicationConfig.update_config_set(db, CONFIG) + + +@pytest.fixture(scope="session") +def config_proxy(db): + return ConfigProxy(db) From 4a1efe7093e4b6f69bb57e48b5a8c8489535b35a Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Thu, 23 Feb 2023 08:30:46 -0500 Subject: [PATCH 6/7] Add db commits to fixtures to make ctl tests happy --- tests/conftest.py | 38 +++++++++------------------------ tests/ctl/cli/test_cli_utils.py | 2 +- tests/ctl/conftest.py | 32 +++++++++++++++++++++++++++ tests/lib/conftest.py | 4 ++++ tests/ops/conftest.py | 4 ++++ 5 files changed, 51 insertions(+), 29 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d6f6e23c8d..07462e4ebf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,6 @@ from fideslang import models from httpx import AsyncClient from loguru import logger -from pytest import MonkeyPatch from sqlalchemy.engine.base import Engine from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker @@ -66,33 +65,6 @@ TEST_INVALID_CONFIG_PATH = "tests/ctl/test_invalid_config.toml" TEST_DEPRECATED_CONFIG_PATH = "tests/ctl/test_deprecated_config.toml" -orig_requests_get = requests.get -orig_requests_post = requests.post -orig_requests_put = requests.put -orig_requests_patch = requests.patch -orig_requests_delete = requests.delete - - -@pytest.fixture(scope="session") -def monkeysession(): - """monkeypatch fixture at the session level instead of the function level""" - mpatch = MonkeyPatch() - yield mpatch - mpatch.undo() - - -@pytest.fixture(autouse=True, scope="session") -def monkeypatch_requests(test_client, monkeysession) -> None: - """The requests library makes requests against the running webserver - which talks to the application db. This monkeypatching operation - makes `requests` calls from src/fides/core/api.py in a test - context talk to the test db instead""" - monkeysession.setattr(requests, "get", test_client.get) - monkeysession.setattr(requests, "post", test_client.post) - monkeysession.setattr(requests, "put", test_client.put) - monkeysession.setattr(requests, "patch", test_client.patch) - monkeysession.setattr(requests, "delete", test_client.delete) - @pytest.fixture(scope="session") def test_client(): @@ -617,9 +589,11 @@ def subject_identity_verification_not_required(db): original_value = CONFIG.execution.subject_identity_verification_required CONFIG.execution.subject_identity_verification_required = False ApplicationConfig.update_config_set(db, CONFIG) + db.commit() yield CONFIG.execution.subject_identity_verification_required = original_value ApplicationConfig.update_config_set(db, CONFIG) + db.commit() @pytest.fixture(autouse=True, scope="function") @@ -628,9 +602,11 @@ def privacy_request_complete_email_notification_disabled(db): original_value = CONFIG.notifications.send_request_completion_notification CONFIG.notifications.send_request_completion_notification = False ApplicationConfig.update_config_set(db, CONFIG) + db.commit() yield CONFIG.notifications.send_request_completion_notification = original_value ApplicationConfig.update_config_set(db, CONFIG) + db.commit() @pytest.fixture(autouse=True, scope="function") @@ -639,9 +615,11 @@ def privacy_request_receipt_notification_disabled(db): original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = False ApplicationConfig.update_config_set(db, CONFIG) + db.commit() yield CONFIG.notifications.send_request_receipt_notification = original_value ApplicationConfig.update_config_set(db, CONFIG) + db.commit() @pytest.fixture(autouse=True, scope="function") @@ -649,10 +627,12 @@ def privacy_request_review_notification_disabled(db): """Disable request review notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_review_notification ApplicationConfig.update_config_set(db, CONFIG) + db.commit() CONFIG.notifications.send_request_review_notification = False yield CONFIG.notifications.send_request_review_notification = original_value ApplicationConfig.update_config_set(db, CONFIG) + db.commit() @pytest.fixture(scope="function", autouse=True) @@ -661,9 +641,11 @@ def set_notification_service_type_mailgun(db): original_value = CONFIG.notifications.notification_service_type CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value ApplicationConfig.update_config_set(db, CONFIG) + db.commit() yield CONFIG.notifications.notification_service_type = original_value ApplicationConfig.update_config_set(db, CONFIG) + db.commit() @pytest.fixture(scope="session") diff --git a/tests/ctl/cli/test_cli_utils.py b/tests/ctl/cli/test_cli_utils.py index 50989dc5cc..95455aeb19 100644 --- a/tests/ctl/cli/test_cli_utils.py +++ b/tests/ctl/cli/test_cli_utils.py @@ -6,7 +6,7 @@ import fides.cli.utils as utils from fides.api.ctl.routes.util import API_PREFIX from fides.core.config import FidesConfig -from tests.conftest import orig_requests_get +from tests.ctl.conftest import orig_requests_get @pytest.mark.unit diff --git a/tests/ctl/conftest.py b/tests/ctl/conftest.py index 42f74822b6..7e86e7e0d3 100644 --- a/tests/ctl/conftest.py +++ b/tests/ctl/conftest.py @@ -1,10 +1,42 @@ +"""This file is only for the database fixture. For all other fixtures add them to the +tests/conftest.py file. +""" + import pytest import requests +from pytest import MonkeyPatch from fides.api.ctl.database.session import sync_engine, sync_session from fides.core import api from tests.conftest import create_citext_extension +orig_requests_get = requests.get +orig_requests_post = requests.post +orig_requests_put = requests.put +orig_requests_patch = requests.patch +orig_requests_delete = requests.delete + + +@pytest.fixture(scope="session") +def monkeysession(): + """monkeypatch fixture at the session level instead of the function level""" + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + +@pytest.fixture(autouse=True, scope="session") +def monkeypatch_requests(test_client, monkeysession) -> None: + """The requests library makes requests against the running webserver + which talks to the application db. This monkeypatching operation + makes `requests` calls from src/fides/core/api.py in a test + context talk to the test db instead""" + monkeysession.setattr(requests, "get", test_client.get) + monkeysession.setattr(requests, "post", test_client.post) + monkeysession.setattr(requests, "put", test_client.put) + monkeysession.setattr(requests, "patch", test_client.patch) + monkeysession.setattr(requests, "delete", test_client.delete) + @pytest.fixture(scope="session", autouse=True) @pytest.mark.usefixtures("monkeypatch_requests") diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py index 5f77d25faa..1be17fc393 100644 --- a/tests/lib/conftest.py +++ b/tests/lib/conftest.py @@ -1,3 +1,7 @@ +"""This file is only for the database fixture. For all other fixtures add them to the +tests/conftest.py file. +""" + import pytest import requests from sqlalchemy.exc import IntegrityError diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index b4930c212b..faa99d2e86 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -1,3 +1,7 @@ +"""This file is only for the database fixture. For all other fixtures add them to the +tests/conftest.py file. +""" + import pytest import requests from sqlalchemy.exc import IntegrityError From bb0ca81e552b1a9676be22244e1dcadeef38273a Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Thu, 23 Feb 2023 09:06:11 -0500 Subject: [PATCH 7/7] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2849e8024..e83f296e82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ The types of changes are: * Add warning to 'fides deploy' when installed outside of a virtual environment [#2641](https://github.com/ethyca/fides/pull/2641) * Removed unexpected default Redis password [#2666](https://github.com/ethyca/fides/pull/2666) +### Developer Experience + +* Combined conftest.py files [#2669](https://github.com/ethyca/fides/pull/2669) + ### Fixed * Fix support for "redis.user" setting when authenticating to the Redis cache [#2666](https://github.com/ethyca/fides/pull/2666)