From 56742ed8a7e53d23743b95e761ea998dded58ca6 Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Fri, 29 Apr 2022 08:22:11 +1200 Subject: [PATCH 1/2] fix(hive): Workaround for Python 3.9 s3 transfer issue --- superset/db_engine_specs/hive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index 484e867ab0a88..05a20af9ce03d 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -65,6 +65,7 @@ def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str: # pylint: disable=import-outside-toplevel import boto3 + from boto3.s3.transfer import TransferConfig bucket_path = current_app.config["CSV_TO_HIVE_UPLOAD_S3_BUCKET"] @@ -80,6 +81,7 @@ def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str: filename, bucket_path, os.path.join(upload_prefix, table.table, os.path.basename(filename)), + Config=TransferConfig(use_threads=False), ) return location From 884ac20191846e4eee3c7311c430df720249a74d Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Tue, 3 May 2022 22:06:25 -0700 Subject: [PATCH 2/2] Update hive.py --- superset/db_engine_specs/hive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index 05a20af9ce03d..344d1f7ae9334 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -81,7 +81,7 @@ def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str: filename, bucket_path, os.path.join(upload_prefix, table.table, os.path.basename(filename)), - Config=TransferConfig(use_threads=False), + Config=TransferConfig(use_threads=False), # Threading is broken in Python 3.9. ) return location