diff --git a/CHANGELOG.md b/CHANGELOG.md index 24932fb6a32..a027aa3cfb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Add deprecation warnings to anonymous usage tracking ([#2688](https://github.com/fishtown-analytics/dbt/issues/2688), [#2710](https://github.com/fishtown-analytics/dbt/issues/2710)) ### Features +- Support TTL for BigQuery tables([#2711](https://github.com/fishtown-analytics/dbt/pull/2711)) - Add better retry support when using the BigQuery adapter ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694), follow-up to [#1963](https://github.com/fishtown-analytics/dbt/pull/1963)) - Added a `dispatch` method to the context adapter and deprecated `adapter_macro`. ([#2302](https://github.com/fishtown-analytics/dbt/issues/2302), [#2679](https://github.com/fishtown-analytics/dbt/pull/2679)) - The built-in schema tests now use `adapter.dispatch`, so they can be overridden for adapter plugins ([#2415](https://github.com/fishtown-analytics/dbt/issues/2415), [#2684](https://github.com/fishtown-analytics/dbt/pull/2684)) @@ -28,7 +29,7 @@ Contributors: - [@bbhoss](https://github.com/bbhoss) ([#2677](https://github.com/fishtown-analytics/dbt/pull/2677)) -- [@kconvey](https://github.com/kconvey) ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694)) +- [@kconvey](https://github.com/kconvey) ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694), [#2711], (https://github.com/fishtown-analytics/dbt/pull/2711)) - [@vogt4nick](https://github.com/vogt4nick) ([#2702](https://github.com/fishtown-analytics/dbt/issues/2702)) - [@stephen8chang](https://github.com/stephen8chang) ([docs#106](https://github.com/fishtown-analytics/dbt-docs/pull/106), [docs#108](https://github.com/fishtown-analytics/dbt-docs/pull/108), [docs#113](https://github.com/fishtown-analytics/dbt-docs/pull/113)) diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index 36049a46d23..cbd4bd6b69f 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -104,6 +104,7 @@ class BigqueryConfig(AdapterConfig): labels: Optional[Dict[str, str]] = None partitions: Optional[List[str]] = None grant_access_to: Optional[List[Dict[str, str]]] = None + hours_to_expiration: Optional[int] = None class BigQueryAdapter(BaseAdapter): @@ -745,6 +746,12 @@ def get_table_options( expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)' opts['expiration_timestamp'] = expiration + if (config.get('hours_to_expiration') is not None) and (not temporary): + expiration = ( + 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL ' + '{} hour)').format(config.get('hours_to_expiration')) + opts['expiration_timestamp'] = expiration + if config.persist_relation_docs() and 'description' in node: description = sql_escape(node['description']) opts['description'] = '"""{}"""'.format(description) diff --git a/test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql b/test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql new file mode 100644 index 00000000000..43258a71464 --- /dev/null +++ b/test/integration/022_bigquery_test/adapter-specific-models/expiring_table.sql @@ -0,0 +1 @@ +select 1 as id diff --git a/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py b/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py new file mode 100644 index 00000000000..dcb8cf4a56b --- /dev/null +++ b/test/integration/022_bigquery_test/test_bigquery_adapter_specific.py @@ -0,0 +1,38 @@ +""""Test adapter specific config options.""" +from test.integration.base import DBTIntegrationTest, use_profile +import textwrap +import yaml + + +class TestBigqueryAdapterSpecific(DBTIntegrationTest): + + @property + def schema(self): + return "bigquery_test_022" + + @property + def models(self): + return "adapter-specific-models" + + @property + def profile_config(self): + return self.bigquery_profile() + + @property + def project_config(self): + return yaml.safe_load(textwrap.dedent('''\ + config-version: 2 + models: + test: + materialized: table + expiring_table: + hours_to_expiration: 4 + ''')) + + @use_profile('bigquery') + def test_bigquery_hours_to_expiration(self): + _, stdout = self.run_dbt_and_capture(['--debug', 'run']) + + self.assertIn( + 'expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL ' + '4 hour)', stdout) diff --git a/test/unit/test_bigquery_adapter.py b/test/unit/test_bigquery_adapter.py index ee89592d5f9..837283b3107 100644 --- a/test/unit/test_bigquery_adapter.py +++ b/test/unit/test_bigquery_adapter.py @@ -4,7 +4,7 @@ import unittest from contextlib import contextmanager from requests.exceptions import ConnectionError -from unittest.mock import patch, MagicMock, Mock +from unittest.mock import patch, MagicMock, Mock, create_autospec import hologram @@ -571,6 +571,35 @@ def test_parse_partition_by(self): } ) + def test_hours_to_expiration(self): + adapter = self.get_adapter('oauth') + mock_config = create_autospec( + dbt.context.providers.RuntimeConfigObject) + config = {'hours_to_expiration': 4} + mock_config.get.side_effect = lambda name: config.get(name) + + expected = { + 'expiration_timestamp': 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 4 hour)', + } + actual = adapter.get_table_options(mock_config, node={}, temporary=False) + self.assertEqual(expected, actual) + + + def test_hours_to_expiration_temporary(self): + adapter = self.get_adapter('oauth') + mock_config = create_autospec( + dbt.context.providers.RuntimeConfigObject) + config={'hours_to_expiration': 4} + mock_config.get.side_effect = lambda name: config.get(name) + + expected = { + 'expiration_timestamp': ( + 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'), + } + actual = adapter.get_table_options(mock_config, node={}, temporary=True) + self.assertEqual(expected, actual) + + class TestBigQueryFilterCatalog(unittest.TestCase): def test__catalog_filter_table(self):