build(ingest): reduce dependencies for dev install

metadata-ingestion/build.gradle

@@ -44,9 +44,12 @@ task lintFix(type: Exec, dependsOn: installDev) {
 task testQuick(type: Exec, dependsOn: installDev) {
   // We can't enforce the coverage requirements if we run a subset of the tests.
   commandLine 'bash', '-x', '-c',
-    "source ${venv_name}/bin/activate && pytest -m 'not slow' -vv --cov-fail-under 0"
+    "source ${venv_name}/bin/activate && pytest -m 'not integration' -vv --cov-fail-under 0"
 }
-task testFull(type: Exec, dependsOn: installDev) {
+task installDevTest(type: Exec, dependsOn: [installDev]) {
+  commandLine "${venv_name}/bin/pip", 'install', '-e', '.[dev,integration-tests]'
+}
+task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
   commandLine 'bash', '-x', '-c',
     "source ${venv_name}/bin/activate && pytest -vv"
 }

metadata-ingestion/developing.md

@@ -94,10 +94,10 @@ Also take a look at the guide to [adding a source](./adding-source.md).
 pip install -e '.[dev]'
 
 # Run unit tests.
-pytest -m 'not slow'
+pytest -m 'not integration'
 
 # Run Docker-based integration tests.
-pytest -m 'slow'
+pytest -m 'integration'
 ```
 
 ### Sanity check code before committing

metadata-ingestion/setup.cfg

@@ -39,7 +39,7 @@ ignore_missing_imports = no
 [tool:pytest]
 addopts = --cov src --cov-report term --cov-config setup.cfg --strict-markers
 markers =
-    slow: marks tests as slow (deselect with '-m "not slow"')
+    integration: marks tests to only run in integration (deselect with '-m "not integration"')
 testpaths = 
     tests/unit
     tests/integration

metadata-ingestion/setup.py

@@ -148,14 +148,8 @@ def get_long_description():
         for plugin in [
             "bigquery",
             "bigquery-usage",
-            "mysql",
-            "mssql",
-            "mongodb",
-            "feast",
-            "ldap",
             "looker",
             "glue",
-            "hive",
             "oracle",
             "sagemaker",
             "datahub-kafka",
@@ -183,6 +177,20 @@ def get_long_description():
     "apache-airflow-providers-snowflake",
 }
 
+full_test_dev_requirements = {
+    *list(
+        dependency
+        for plugin in [
+            "feast",
+            "hive",
+            "ldap",
+            "mongodb",
+            "mssql",
+            "mysql",
+        ]
+        for dependency in plugins[plugin]
+    ),
+}
 
 entry_points = {
     "console_scripts": ["datahub = datahub.entrypoints:main"],
@@ -288,5 +296,6 @@ def get_long_description():
         ),
         "dev": list(dev_requirements),
         "dev-airflow2": list(dev_requirements_airflow_2),
+        "integration-tests": list(full_test_dev_requirements),
     },
 )

metadata-ingestion/tests/integration/feast/test_feast.py

@@ -9,7 +9,7 @@
 
 
 # make sure that mock_time is excluded here because it messes with feast
-@pytest.mark.slow
+@pytest.mark.integration
 def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/feast"
 

metadata-ingestion/tests/integration/hive/test_hive.py

@@ -8,7 +8,7 @@
 from tests.test_helpers.docker_helpers import wait_for_port
 
 
-@pytest.mark.slow
+@pytest.mark.integration
 def test_hive_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/hive"
 

metadata-ingestion/tests/integration/ldap/test_ldap.py

@@ -7,7 +7,7 @@
 from tests.test_helpers.docker_helpers import wait_for_port
 
 
-@pytest.mark.slow
+@pytest.mark.integration
 def test_ldap_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/ldap"
 

metadata-ingestion/tests/integration/mongodb/test_mongodb.py

@@ -5,7 +5,7 @@
 from tests.test_helpers.docker_helpers import wait_for_port
 
 
-@pytest.mark.slow
+@pytest.mark.integration
 def test_mongodb_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/mongodb"
 

metadata-ingestion/tests/integration/mysql/test_mysql.py

@@ -6,7 +6,7 @@
 from tests.test_helpers.docker_helpers import wait_for_port
 
 
-@pytest.mark.slow
+@pytest.mark.integration
 def test_mysql_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/mysql"
 

metadata-ingestion/tests/integration/sql_server/test_sql_server.py

@@ -9,7 +9,7 @@
 from tests.test_helpers.docker_helpers import wait_for_port
 
 
-@pytest.mark.slow
+@pytest.mark.integration
 def test_mssql_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/sql_server"
 

metadata-ingestion/tests/unit/test_hive_source.py

@@ -1,23 +1,21 @@
-import unittest
+import pytest
 
-from datahub.ingestion.source.hive import HiveConfig
 
+@pytest.mark.integration
+def test_hive_configuration_get_identifier_with_database():
+    from datahub.ingestion.source.hive import HiveConfig
 
-class HiveSinkTest(unittest.TestCase):
-    def test_hive_configuration_get_indentifier_with_database(self):
-        test_db_name = "test_database"
-        test_schema_name = "test_schema"
-        test_table_name = "test_table"
-        config_dict = {
-            "username": "test",
-            "password": "test",
-            "host_port": "test:80",
-            "database": test_db_name,
-            "scheme": "hive+https",
-        }
-        hive_config = HiveConfig.parse_obj(config_dict)
-        expected_output = f"{test_db_name}.{test_schema_name}.{test_table_name}"
-        output = hive_config.get_identifier(
-            schema=test_schema_name, table=test_table_name
-        )
-        assert output == expected_output
+    test_db_name = "test_database"
+    test_schema_name = "test_schema"
+    test_table_name = "test_table"
+    config_dict = {
+        "username": "test",
+        "password": "test",
+        "host_port": "test:80",
+        "database": test_db_name,
+        "scheme": "hive+https",
+    }
+    hive_config = HiveConfig.parse_obj(config_dict)
+    expected_output = f"{test_db_name}.{test_schema_name}.{test_table_name}"
+    output = hive_config.get_identifier(schema=test_schema_name, table=test_table_name)
+    assert output == expected_output

metadata-ingestion/tests/unit/test_ldap_source.py

@@ -1,15 +1,17 @@
 import pytest
 
-from src.datahub.ingestion.source.ldap import parse_from_attrs, strip_ldap_info
-
 
+@pytest.mark.integration
 def test_strip_ldap_info():
+    from datahub.ingestion.source.ldap import strip_ldap_info
+
     assert (
         strip_ldap_info(b"uid=firstname.surname,ou=People,dc=internal,dc=machines")
         == "firstname.surname"
     )
 
 
+@pytest.mark.integration
 @pytest.mark.parametrize(
     "input, expected",
     [
@@ -33,6 +35,8 @@ def test_strip_ldap_info():
     ],
 )
 def test_parse_from_attrs(input, expected):
+    from datahub.ingestion.source.ldap import parse_from_attrs
+
     assert (
         parse_from_attrs(
             input,

metadata-ingestion/tox.ini

@@ -13,7 +13,7 @@ extras = dev
 commands =
     # TODO: look at https://github.com/pytest-dev/pytest-cov/blob/master/examples/src-layout/tox.ini
     # For now, coverage doesn't work within tox.
-    pytest -m 'not slow' --cov-fail-under=0
+    pytest -m 'not integration' --cov-fail-under=0
 
 [testenv:py3-airflow2]
 extras = dev-airflow2