From 2a06cf32abac7877729752fb19f0af55afec3587 Mon Sep 17 00:00:00 2001 From: "Felippe F. Caso" Date: Tue, 8 Aug 2023 18:59:13 -0300 Subject: [PATCH 1/3] feat: add s3 support to excel plugin through env --- dbt/adapters/duckdb/plugins/excel.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/duckdb/plugins/excel.py b/dbt/adapters/duckdb/plugins/excel.py index 7bdbe086..3e9d9e6a 100644 --- a/dbt/adapters/duckdb/plugins/excel.py +++ b/dbt/adapters/duckdb/plugins/excel.py @@ -1,3 +1,7 @@ +import os +from typing import Any +from typing import Dict + import pathlib import pandas as pd @@ -7,9 +11,22 @@ class Plugin(BasePlugin): + def initialize(self, plugin_config: Dict[str, Any]): + # Pass s3 settings to plugin environment + if 's3_access_key_id' in plugin_config: + os.environ['AWS_ACCESS_KEY_ID'] = plugin_config.get('s3_access_key_id') + if 's3_secret_access_key' in plugin_config: + os.environ['AWS_SECRET_ACCESS_KEY'] = plugin_config.get('s3_secret_access_key') + if 's3_region' in plugin_config: + os.environ['AWS_DEFAULT_REGION'] = plugin_config.get('s3_region') + def load(self, source_config: SourceConfig): ext_location = source_config["external_location"] ext_location = ext_location.format(**source_config.as_dict()) - source_location = pathlib.Path(ext_location.strip("'")) + if 's3' in ext_location: + # Possible to add some treatment in the future + source_location = ext_location + else: + source_location = pathlib.Path(ext_location.strip("'")) sheet_name = source_config.get("sheet_name", 0) return pd.read_excel(source_location, sheet_name=sheet_name) From c6cb22c282eb1ce97f2ae71c7ab38edd607cbc01 Mon Sep 17 00:00:00 2001 From: "Felippe F. Caso" Date: Tue, 8 Aug 2023 19:18:54 -0300 Subject: [PATCH 2/3] fix: change quoting character --- dbt/adapters/duckdb/plugins/excel.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dbt/adapters/duckdb/plugins/excel.py b/dbt/adapters/duckdb/plugins/excel.py index 3e9d9e6a..0a313c00 100644 --- a/dbt/adapters/duckdb/plugins/excel.py +++ b/dbt/adapters/duckdb/plugins/excel.py @@ -13,17 +13,17 @@ class Plugin(BasePlugin): def initialize(self, plugin_config: Dict[str, Any]): # Pass s3 settings to plugin environment - if 's3_access_key_id' in plugin_config: - os.environ['AWS_ACCESS_KEY_ID'] = plugin_config.get('s3_access_key_id') - if 's3_secret_access_key' in plugin_config: - os.environ['AWS_SECRET_ACCESS_KEY'] = plugin_config.get('s3_secret_access_key') - if 's3_region' in plugin_config: - os.environ['AWS_DEFAULT_REGION'] = plugin_config.get('s3_region') + if "s3_access_key_id" in plugin_config: + os.environ["AWS_ACCESS_KEY_ID"] = plugin_config.get("s3_access_key_id") + if "s3_secret_access_key" in plugin_config: + os.environ["AWS_SECRET_ACCESS_KEY"] = plugin_config.get("s3_secret_access_key") + if "s3_region" in plugin_config: + os.environ["AWS_DEFAULT_REGION"] = plugin_config.get("s3_region") def load(self, source_config: SourceConfig): ext_location = source_config["external_location"] ext_location = ext_location.format(**source_config.as_dict()) - if 's3' in ext_location: + if "s3" in ext_location: # Possible to add some treatment in the future source_location = ext_location else: From e56f3b3cd4415a785498b0cca065f508d1af342b Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Tue, 8 Aug 2023 16:26:35 -0700 Subject: [PATCH 3/3] Update excel.py format fixes --- dbt/adapters/duckdb/plugins/excel.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/duckdb/plugins/excel.py b/dbt/adapters/duckdb/plugins/excel.py index 0a313c00..b279fe5b 100644 --- a/dbt/adapters/duckdb/plugins/excel.py +++ b/dbt/adapters/duckdb/plugins/excel.py @@ -1,9 +1,8 @@ import os +import pathlib from typing import Any from typing import Dict -import pathlib - import pandas as pd from . import BasePlugin @@ -14,11 +13,11 @@ class Plugin(BasePlugin): def initialize(self, plugin_config: Dict[str, Any]): # Pass s3 settings to plugin environment if "s3_access_key_id" in plugin_config: - os.environ["AWS_ACCESS_KEY_ID"] = plugin_config.get("s3_access_key_id") + os.environ["AWS_ACCESS_KEY_ID"] = plugin_config["s3_access_key_id"] if "s3_secret_access_key" in plugin_config: - os.environ["AWS_SECRET_ACCESS_KEY"] = plugin_config.get("s3_secret_access_key") + os.environ["AWS_SECRET_ACCESS_KEY"] = plugin_config["s3_secret_access_key"] if "s3_region" in plugin_config: - os.environ["AWS_DEFAULT_REGION"] = plugin_config.get("s3_region") + os.environ["AWS_DEFAULT_REGION"] = plugin_config["s3_region"] def load(self, source_config: SourceConfig): ext_location = source_config["external_location"]