From 59ab86ebb4c347f9ba16586217dcff1f98723fe0 Mon Sep 17 00:00:00 2001 From: Madhu Kadiri Date: Fri, 13 Dec 2024 15:11:19 +0000 Subject: [PATCH] partitioned-hash-gluejob altered - v1 --- .../glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py b/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py index 69e4b1cb67e..7a02275c150 100644 --- a/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py +++ b/terraform/environments/electronic-monitoring-data/glue-job/etl_rds_tbl_rows_hashvalue_to_s3_prq_yyyy_mm.py @@ -292,7 +292,7 @@ def write_rds_df_to_s3_parquet(df_rds_write: DataFrame, HASHBYTES('SHA2_256', CONCAT_WS('', {', '.join(all_columns_except_pkey)})), 1), 3, 66)) AS RowHash, YEAR({date_partition_column_name}) AS year, MONTH({date_partition_column_name}) AS month - FROM {rds_sqlserver_db_schema}.[{rds_sqlserver_db_table}] + FROM {rds_sqlserver_db}.{rds_sqlserver_db_schema}.{rds_sqlserver_db_table} """.strip() incremental_run_bool = args.get('incremental_run_bool', 'false') @@ -301,7 +301,7 @@ def write_rds_df_to_s3_parquet(df_rds_write: DataFrame, if rds_query_where_clause is not None: rds_db_hash_cols_query_str = rds_db_hash_cols_query_str + \ - f""" WHERE {rds_query_where_clause.lstrip()}""" + f""" WHERE {rds_query_where_clause.rstrip()}""" elif incremental_run_bool == 'true': existing_prq_hashed_rows_df = CustomPysparkMethods.get_s3_parquet_df_v2(