Get functional tests working against Dremio Cloud

dbt/include/dremio/macros/adapters/apply_grants.sql

@@ -22,18 +22,25 @@ limitations under the License.*/
     {%- else -%}
         {%- set relation_without_double_quotes = relation.database ~ '.' ~ relation.schema ~ '.' ~ relation.identifier-%}
     {%- endif %}
-
+
+    {%- if target.cloud_host and not target.software_host -%}
+        {%- set privileges_table = 'sys.project.privileges' -%}
+    {%- elif target.software_host and not target.cloud_host -%}
+        {%- set privileges_table = 'sys.privileges' -%}
+    {%- else -%}
+         {% do exceptions.raise_compiler_error("Invalid profile configuration: please only specify one of cloud_host or software_host in profiles.yml") %}
+    {%- endif %}
     SELECT privilege, grantee_id
-        FROM sys.privileges 
-        WHERE privileges.object_id='{{ relation_without_double_quotes }}'
+        FROM {{privileges_table}}
+        WHERE object_id='{{ relation_without_double_quotes }}'
 {% endmacro %}
 
 {%- macro dremio__get_grant_sql(relation, privilege, grantees) -%}
-    grant {{ privilege }} on {{relation.type}} {{ relation }} to user {{ grantees | join(', ') }}
+    grant {{ privilege }} on {{relation.type}} {{ relation }} to user {{adapter.quote(grantees[0])}}
 {%- endmacro -%}
 
 {%- macro default__get_revoke_sql(relation, privilege, grantees) -%}
-    revoke {{ privilege }} on {{ relation.type }} {{ relation }} from user {{ grantees | join(', ') }}
+    revoke {{ privilege }} on {{ relation.type }} {{ relation }} from user {{adapter.quote(grantees[0])}}
 {%- endmacro -%}
 
 {% macro dremio__call_dcl_statements(dcl_statement_list) %}

tests/conftest.py

@@ -13,22 +13,51 @@
 import pytest
 from dotenv import load_dotenv
 
-# Import the fuctional fixtures as a plugin
+# Import the functional fixtures as a plugin
 # Note: fixtures with session scope need to be local
 
 pytest_plugins = ["dbt.tests.fixtures.project"]
 load_dotenv()
 
 
-# The profile dictionary, used to write out profiles.yml
-@pytest.fixture(scope="class")
-def dbt_profile_target():
+def pytest_addoption(parser):
+    parser.addoption("--profile", action="store", default="dremio_cloud", type=str)
+
+
+@pytest.fixture(scope="session")
+def dbt_profile_target(request):
+    profile_type = request.config.getoption("--profile")
+    if profile_type == "dremio_software":
+        target = dremio_software_target()
+    elif profile_type == "dremio_cloud":
+        target = dremio_cloud_target()
+    else:
+        raise ValueError(f"Invalid profile type '{profile_type}'")
+    return target
+
+
+def dremio_cloud_target():
+    return {
+        "type": "dremio",
+        "threads": 1,
+        "cloud_host": os.getenv("DREMIO_CLOUD_HOST"),
+        "cloud_project_id": os.getenv("DREMIO_CLOUD_PROJECT_ID"),
+        "user": os.getenv("DREMIO_CLOUD_USERNAME"),
+        "pat": os.getenv("DREMIO_PAT"),
+        "datalake": os.getenv("DREMIO_DATALAKE"),
+        "use_ssl": True,
+        # Need to include a specific space for grants tests
+        "database": os.getenv("DREMIO_DATABASE"),
+    }
+
+
+def dremio_software_target():
     return {
         "type": "dremio",
         "threads": 1,
-        "software_host": os.getenv("DREMIO_HOST"),
-        "user": os.getenv("DREMIO_USERNAME"),
-        "password": os.getenv("DREMIO_PASSWORD"),
+        "software_host": os.getenv("DREMIO_SOFTWARE_HOST"),
+        "user": os.getenv("DREMIO_SOFTWARE_USERNAME"),
+        "password": os.getenv("DREMIO_SOFTWARE_PASSWORD"),
         "datalake": os.getenv("DREMIO_DATALAKE"),
         "use_ssl": False,
         # Need to include a specific space for grants tests

tests/fixtures/profiles.py

@@ -10,7 +10,7 @@
 # limitations under the License.
 
 import pytest
-from tests.functional.adapter.utils.test_utils import DATALAKE
+from tests.utils.util import BUCKET
 
 # This ensures the schema works with our datalake
 @pytest.fixture(scope="class")
@@ -36,7 +36,7 @@ def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update):
     }
     target = dbt_profile_target
     target["schema"] = unique_schema
-    target["root_path"] = f"{DATALAKE}.{unique_schema}"
+    target["root_path"] = f"{BUCKET}.{unique_schema}"
     profile["test"]["outputs"]["default"] = target
 
     if profiles_config_update:

tests/functional/adapter/basic/test_adapter_methods.py

@@ -49,15 +49,6 @@
 
 
 class TestBaseAdapterMethodDremio(BaseAdapterMethod):
-    @pytest.fixture(scope="class")
-    def project_config_update(self):
-        return {
-            "models": {
-                "+twin_strategy": "clone",
-            },
-            "name": "adapter_methods",
-        }
-
     @pytest.fixture(scope="class")
     def models(self):
         return {

tests/functional/adapter/basic/test_base_mat.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from tests.functional.adapter.utils.test_utils import (
+from tests.utils.util import (
     relation_from_name,
     check_relations_equal,
     check_relation_types,
@@ -29,7 +29,7 @@
     run_dbt,
     check_result_nodes_by_name,
 )
-from tests.functional.adapter.utils.test_utils import DATALAKE
+from tests.utils.util import BUCKET
 
 # Unable to insert variable into docstring, so "dbt_test_source" is hardcoded
 schema_base_yml = """
@@ -70,7 +70,7 @@ def unique_schema(self, request, prefix) -> str:
         test_file = request.module.__name__
         # We only want the last part of the name
         test_file = test_file.split(".")[-1]
-        unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
+        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
         return unique_schema
 
     @pytest.fixture(scope="class")

tests/functional/adapter/basic/test_docs_generate.py

@@ -14,10 +14,10 @@
 
 import pytest
 import os
-from tests.functional.adapter.utils.test_utils import (
+from tests.utils.util import (
     base_expected_catalog,
     expected_references_catalog,
-    DATALAKE,
+    BUCKET,
 )
 from dbt.tests.adapter.basic.test_docs_generate import (
     BaseDocsGenerate,
@@ -79,13 +79,13 @@ def models(self):
             "model.sql": models__model_sql,
         }
 
-    # Override this fixture to prepend our schema with DATALAKE
+    # Override this fixture to prepend our schema with BUCKET
     # This ensures the schema works with our datalake
     @pytest.fixture(scope="class")
     def unique_schema(self, request, prefix) -> str:
         test_file = request.module.__name__
         test_file = test_file.split(".")[-1]
-        unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
+        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
         return unique_schema
 
     # Override this fixture to prevent (twin_strategy) creating a view for seeds
@@ -165,7 +165,7 @@ class TestBaseDocsGenReferencesDremio(BaseDocsGenReferences):
     def unique_schema(self, request, prefix) -> str:
         test_file = request.module.__name__
         test_file = test_file.split(".")[-1]
-        unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
+        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
         return unique_schema
 
     # Override this fixture to allow (twin_strategy) to create a view for seeds

tests/functional/adapter/basic/test_ephemeral.py

@@ -15,7 +15,7 @@
 import pytest
 from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
 from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti
-from tests.functional.adapter.utils.test_utils import DATALAKE
+from tests.utils.util import BUCKET
 from dbt.tests.util import run_dbt
 from tests.fixtures.profiles import unique_schema, dbt_profile_data
 

tests/functional/adapter/basic/test_snapshots.py

@@ -15,7 +15,7 @@
 import pytest
 from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols
 from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
-from tests.functional.adapter.utils.test_utils import DATALAKE
+from tests.utils.util import BUCKET
 
 
 @pytest.mark.skip(reason="https://github.com/dremio/dbt-dremio/issues/20")
@@ -25,7 +25,7 @@ def unique_schema(self, request, prefix) -> str:
         test_file = request.module.__name__
         # We only want the last part of the name
         test_file = test_file.split(".")[-1]
-        unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
+        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
         return unique_schema
 
     @pytest.fixture(scope="class")
@@ -65,7 +65,7 @@ def unique_schema(self, request, prefix) -> str:
         test_file = request.module.__name__
         # We only want the last part of the name
         test_file = test_file.split(".")[-1]
-        unique_schema = f"{DATALAKE}.{prefix}_{test_file}"
+        unique_schema = f"{BUCKET}.{prefix}_{test_file}"
         return unique_schema
 
     @pytest.fixture(scope="class")

tests/functional/adapter/grants/base_grants.py

@@ -15,7 +15,7 @@
 import pytest
 
 from dbt.tests.adapter.grants.base_grants import BaseGrants
-from tests.functional.adapter.utils.test_utils import DATALAKE
+from tests.utils.util import BUCKET
 
 
 class BaseGrantsDremio(BaseGrants):
@@ -54,7 +54,7 @@ def dbt_profile_data(
         }
         target = dbt_profile_target
         target["schema"] = unique_schema
-        target["root_path"] = f"{DATALAKE}.{unique_schema}"
+        target["root_path"] = f"{BUCKET}.{unique_schema}"
         profile["test"]["outputs"]["default"] = target
 
         if profiles_config_update:

tests/functional/adapter/grants/test_incremental_grants.py

@@ -20,7 +20,7 @@
     run_dbt,
 )
 from tests.functional.adapter.grants.base_grants import BaseGrantsDremio
-from tests.functional.adapter.utils.test_utils import relation_from_name
+from tests.utils.util import relation_from_name
 from dbt.tests.adapter.grants.test_incremental_grants import (
     BaseIncrementalGrants,
     user2_incremental_model_schema_yml,

tests/functional/adapter/grants/test_invalid_grants.py

@@ -15,7 +15,7 @@
 
 from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants
 from tests.functional.adapter.grants.base_grants import BaseGrantsDremio
-from tests.functional.adapter.utils.test_utils import relation_from_name
+from tests.utils.util import relation_from_name
 from dbt.tests.util import get_connection
 
 # Currently we return an HTTP error, but need to improve error handling

tests/functional/adapter/grants/test_model_grants.py

@@ -22,7 +22,7 @@
 )
 from dbt.tests.util import run_dbt_and_capture, get_manifest, write_file, get_connection
 from tests.functional.adapter.grants.base_grants import BaseGrantsDremio
-from tests.functional.adapter.utils.test_utils import relation_from_name
+from tests.utils.util import relation_from_name
 
 
 class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants):

tests/functional/adapter/grants/test_seed_grants.py

@@ -14,7 +14,7 @@
 
 from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants
 from tests.functional.adapter.grants.base_grants import BaseGrantsDremio
-from tests.functional.adapter.utils.test_utils import relation_from_name
+from tests.utils.util import relation_from_name
 from dbt.tests.util import get_connection
 
 

tests/functional/adapter/grants/test_snapshot_grants.py

@@ -17,7 +17,11 @@
     snapshot_schema_yml,
 )
 from tests.functional.adapter.grants.base_grants import BaseGrantsDremio
-from tests.functional.adapter.utils.test_utils import relation_from_name, DATALAKE
+from tests.utils.util import (
+    relation_from_name,
+    BUCKET,
+    SOURCE,
+)
 from dbt.tests.util import get_connection
 
 # Override this model to use strategy timestamp
@@ -58,9 +62,9 @@ def dbt_profile_data(
             },
         }
         target = dbt_profile_target
-        target["schema"] = f"{DATALAKE}.{unique_schema}"
-        target["root_path"] = f"{DATALAKE}.{unique_schema}"
-        target["database"] = DATALAKE
+        target["schema"] = f"{BUCKET}.{unique_schema}"
+        target["root_path"] = f"{BUCKET}.{unique_schema}"
+        target["database"] = SOURCE
         profile["test"]["outputs"]["default"] = target
 
         if profiles_config_update:

tests/functional/adapter/utils/test_utils.py → tests/utils/util.py

@@ -24,7 +24,8 @@
 
 # Ensure we do not include dashes in our source
 # https://github.com/dremio/dbt-dremio/issues/68
-DATALAKE = "dbt_test_source"
+BUCKET = "dbtdremios3"
+SOURCE = "dbt_test_source"
 
 
 class TestProcessingException(Exception):
@@ -46,7 +47,10 @@ def relation_from_name(adapter, name: str, materialization=""):
     # only identifier was supplied.
     relation_parts = name.split(".")
     if len(relation_parts) == 1:
-        relation_parts.insert(0, credentials.schema)
+        if materialization == "view" or "view" in name:
+            relation_parts.insert(0, credentials.schema)
+        else:
+            relation_parts.insert(0, credentials.root_path)
     if len(relation_parts) == 2:
         # if the relation is a view then use database
         if materialization == "view" or "view" in name:
@@ -220,7 +224,7 @@ def case(x):
                 "unique_id": "seed.test.seed",
                 "metadata": {
                     "schema": my_schema_name,
-                    "database": DATALAKE,
+                    "database": SOURCE,
                     "name": case("seed"),
                     "type": table_type,
                     "comment": None,
@@ -235,7 +239,7 @@ def case(x):
                 "unique_id": "source.test.my_source.my_table",
                 "metadata": {
                     "schema": my_schema_name,
-                    "database": DATALAKE,
+                    "database": SOURCE,
                     "name": case("seed"),
                     "type": table_type,
                     "comment": None,
@@ -332,7 +336,7 @@ def case(x):
                 "unique_id": "seed.test.seed",
                 "metadata": {
                     "schema": my_schema_name,
-                    "database": DATALAKE,
+                    "database": SOURCE,
                     "name": case("seed"),
                     "type": table_type,
                     "comment": None,
@@ -345,7 +349,7 @@ def case(x):
                 "unique_id": "model.test.ephemeral_summary",
                 "metadata": {
                     "schema": my_schema_name,
-                    "database": DATALAKE,
+                    "database": SOURCE,
                     "name": case("ephemeral_summary"),
                     "type": table_type,
                     "comment": None,