Add option to skip relation cache population

.changes/unreleased/Features-20230410-115538.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add --no-populate-cache to optionally skip relation cache population
+time: 2023-04-10T11:55:38.360997-05:00
+custom:
+  Author: stu-k
+  Issue: "1751"

core/dbt/adapters/base/impl.py

@@ -718,6 +718,18 @@ def list_relations(self, database: Optional[str], schema: str) -> List[BaseRelat
         # we can't build the relations cache because we don't have a
         # manifest so we can't run any operations.
         relations = self.list_relations_without_caching(schema_relation)
+
+        # if the cache is already populated, add this schema in
+        # otherwise, skip updating the cache and just ignore
+        if self.cache:
+            for relation in relations:
+                self.cache.add(relation)
+            if not relations:
+                # it's possible that there were no relations in some schemas. We want
+                # to insert the schemas we query into the cache's `.schemas` attribute
+                # so we can check it later
+                self.cache.update_schemas([(database, schema)])
+
         fire_event(
             ListRelations(
                 database=cast_to_str(database),

core/dbt/cli/main.py

@@ -147,6 +147,7 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
 @p.log_path
 @p.macro_debugging
 @p.partial_parse
+@p.populate_cache
 @p.print
 @p.printer_width
 @p.quiet

core/dbt/cli/params.py

@@ -238,6 +238,13 @@
     default=True,
 )
 
+populate_cache = click.option(
+    "--populate-cache/--no-populate-cache",
+    envvar="DBT_POPULATE_CACHE",
+    help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.",
+    default=True,
+)
+
 port = click.option(
     "--port",
     envvar=None,

core/dbt/contracts/project.py

@@ -243,25 +243,26 @@ def validate(cls, data):
 
 @dataclass
 class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract):
-    send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS
-    use_colors: Optional[bool] = None
-    use_colors_file: Optional[bool] = None
-    partial_parse: Optional[bool] = None
-    printer_width: Optional[int] = None
-    write_json: Optional[bool] = None
-    warn_error: Optional[bool] = None
-    warn_error_options: Optional[Dict[str, Union[str, List[str]]]] = None
+    cache_selected_only: Optional[bool] = None
+    debug: Optional[bool] = None
+    fail_fast: Optional[bool] = None
+    indirect_selection: Optional[str] = None
     log_format: Optional[str] = None
     log_format_file: Optional[str] = None
     log_level: Optional[str] = None
     log_level_file: Optional[str] = None
-    debug: Optional[bool] = None
-    version_check: Optional[bool] = None
-    fail_fast: Optional[bool] = None
-    use_experimental_parser: Optional[bool] = None
+    partial_parse: Optional[bool] = None
+    populate_cache: Optional[bool] = None
+    printer_width: Optional[int] = None
+    send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS
     static_parser: Optional[bool] = None
-    indirect_selection: Optional[str] = None
-    cache_selected_only: Optional[bool] = None
+    use_colors: Optional[bool] = None
+    use_colors_file: Optional[bool] = None
+    use_experimental_parser: Optional[bool] = None
+    version_check: Optional[bool] = None
+    warn_error: Optional[bool] = None
+    warn_error_options: Optional[Dict[str, Union[str, List[str]]]] = None
+    write_json: Optional[bool] = None
 
 
 @dataclass

core/dbt/task/runnable.py

@@ -371,6 +371,9 @@ def _mark_dependent_errors(self, node_id, result, cause):
             self._skipped_children[dep_node_id] = cause
 
     def populate_adapter_cache(self, adapter, required_schemas: Set[BaseRelation] = None):
+        if not self.args.populate_cache:
+            return
+
         start_populate_cache = time.perf_counter()
         if get_flags().CACHE_SELECTED_ONLY is True:
             adapter.set_relations_cache(self.manifest, required_schemas=required_schemas)

tests/adapter/dbt/tests/adapter/caching/test_caching.py

@@ -91,6 +91,20 @@ def test_cache(self, project):
         self.run_and_inspect_cache(project, run_args)
 
 
+class TestNoPopulateCache(BaseCachingTest):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "model.sql": model_sql,
+        }
+
+    def test_cache(self, project):
+        # --no-populate-cache still allows the cache to populate all relations
+        # under a schema, so the behavior here remains the same as other tests
+        run_args = ["--no-populate-cache", "run"]
+        self.run_and_inspect_cache(project, run_args)
+
+
 class TestCachingLowerCaseModel(BaseCachingLowercaseModel):
     pass