From 496ba2f87861c8aef148695a2dbe5d7939c0f219 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Tue, 10 Oct 2023 09:10:27 +0100 Subject: [PATCH] agg mergeSchema option --- ...ng_permit_diesel_trends_bought_in_month.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/jobs/parking/parking_permit_diesel_trends_bought_in_month.py b/scripts/jobs/parking/parking_permit_diesel_trends_bought_in_month.py index e00a8cede..c3f12c546 100644 --- a/scripts/jobs/parking/parking_permit_diesel_trends_bought_in_month.py +++ b/scripts/jobs/parking/parking_permit_diesel_trends_bought_in_month.py @@ -1,11 +1,18 @@ import sys + +from awsglue import DynamicFrame +from awsglue.context import GlueContext +from awsglue.job import Job from awsglue.transforms import * from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue import DynamicFrame -from scripts.helpers.helpers import get_glue_env_var, get_latest_partitions, PARTITION_KEYS + +from scripts.helpers.helpers import ( + PARTITION_KEYS, + get_glue_env_var, + get_latest_partitions, +) + def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFrame: for alias, frame in mapping.items(): @@ -42,6 +49,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-raw-zone", table_name="liberator_permit_vrm_update", transformation_ctx="AmazonS3_node1658998021932", + additional_options={"mergeSchema": "true"}, ) # Script generated for node SQL @@ -306,7 +314,9 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra # Script generated for node Amazon S3 AmazonS3_node1658765590649 = glueContext.getSink( - path="s3://dataplatform-" + environment + "-refined-zone/parking/liberator/Parking_Permit_diesel_Tends_Bought_in_Month/", + path="s3://dataplatform-" + + environment + + "-refined-zone/parking/liberator/Parking_Permit_diesel_Tends_Bought_in_Month/", connection_type="s3", updateBehavior="UPDATE_IN_DATABASE", partitionKeys=PARTITION_KEYS,