Dbt freshness debug clean (#1321)

* Update fork from upstream * Recommit changes with proper signoff as rebase caused issues Signed-off-by: Kilian <[email protected]> * Add sources output for dbt freshness input Signed-off-by: Kilian <[email protected]> * Fix documentation of dataclass Signed-off-by: Kilian <[email protected]> Signed-off-by: Kilian <[email protected]> Co-authored-by: Kilian <[email protected]>
flyteorg · Nov 25, 2022 · 7cf5b68 · 7cf5b68
1 parent 430795d
commit 7cf5b68
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 21 deletions.
diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py b/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py
@@ -251,11 +251,8 @@ class DBTFreshnessOutput(BaseDBTOutput):
 
     Attributes
     ----------
-    raw_run_result : str
-        Raw value of DBT's ``run_result.json``.
-    raw_manifest : str
-        Raw value of DBT's ``manifest.json``.
+    raw_sources : str
+        Raw value of DBT's ``sources.json``.
     """
 
-    raw_run_result: str
-    raw_manifest: str
+    raw_sources: str
diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/task.py b/plugins/flytekit-dbt/flytekitplugins/dbt/task.py
@@ -354,18 +354,13 @@ def my_workflow() -> DBTFreshnessOutput:
             raise DBTUnhandledError(f"unhandled error while executing {full_command}", logs)
 
         output_dir = os.path.join(task_input.project_dir, task_input.output_path)
-        run_result_path = os.path.join(output_dir, "run_results.json")
-        with open(run_result_path) as file:
-            run_result = file.read()
+        sources_path = os.path.join(output_dir, "sources.json")
+        with open(sources_path) as file:
+            sources = file.read()
 
-        # read manifest.json
-        manifest_path = os.path.join(output_dir, "manifest.json")
-        with open(manifest_path) as file:
-            manifest = file.read()
 
         return DBTFreshnessOutput(
             command=full_command,
             exit_code=exit_code,
-            raw_run_result=run_result,
-            raw_manifest=manifest,
+            raw_sources=sources
         )
diff --git a/plugins/flytekit-dbt/tests/test_task.py b/plugins/flytekit-dbt/tests/test_task.py
@@ -219,10 +219,7 @@ def test_task_output(self):
             == f"dbt --log-format json source freshness --project-dir {DBT_PROJECT_DIR} --profiles-dir {DBT_PROFILES_DIR} --profile {DBT_PROFILE}"
         )
 
-        with open(f"{DBT_PROJECT_DIR}/target/run_results.json", "r") as fp:
-            exp_run_result = fp.read()
-        assert output.raw_run_result == exp_run_result
+        with open(f"{DBT_PROJECT_DIR}/target/sources.json", "r") as fp:
+            exp_sources = fp.read()
 
-        with open(f"{DBT_PROJECT_DIR}/target/manifest.json", "r") as fp:
-            exp_manifest = fp.read()
-        assert output.raw_manifest == exp_manifest
+        assert output.raw_sources == exp_sources