Skip to content

Commit

Permalink
feat: add s3 support to excel plugin through env
Browse files Browse the repository at this point in the history
  • Loading branch information
felippecaso committed Aug 8, 2023
1 parent 8b6f366 commit 2a06cf3
Showing 1 changed file with 18 additions and 1 deletion.
19 changes: 18 additions & 1 deletion dbt/adapters/duckdb/plugins/excel.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
import os
from typing import Any
from typing import Dict

import pathlib

import pandas as pd
Expand All @@ -7,9 +11,22 @@


class Plugin(BasePlugin):
def initialize(self, plugin_config: Dict[str, Any]):
# Pass s3 settings to plugin environment
if 's3_access_key_id' in plugin_config:
os.environ['AWS_ACCESS_KEY_ID'] = plugin_config.get('s3_access_key_id')
if 's3_secret_access_key' in plugin_config:
os.environ['AWS_SECRET_ACCESS_KEY'] = plugin_config.get('s3_secret_access_key')
if 's3_region' in plugin_config:
os.environ['AWS_DEFAULT_REGION'] = plugin_config.get('s3_region')

def load(self, source_config: SourceConfig):
ext_location = source_config["external_location"]
ext_location = ext_location.format(**source_config.as_dict())
source_location = pathlib.Path(ext_location.strip("'"))
if 's3' in ext_location:
# Possible to add some treatment in the future
source_location = ext_location
else:
source_location = pathlib.Path(ext_location.strip("'"))
sheet_name = source_config.get("sheet_name", 0)
return pd.read_excel(source_location, sheet_name=sheet_name)

0 comments on commit 2a06cf3

Please sign in to comment.