From 21035f79709b146955e57548df8dea9bf80c8975 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:24:47 -0800 Subject: [PATCH] [Backport 2.x] Remove manual refresh for S3 integrations (#1227) (cherry picked from commit 80f112a27c98eb0031011a24ad2a22a22ddca0e7) Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../repository/aws_elb/assets/create_mv-1.0.0.sql | 8 +++++++- .../repository/aws_elb/assets/refresh_mv-1.0.0.sql | 1 - .../__data__/repository/aws_elb/aws_elb-1.0.0.json | 5 ----- .../aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql | 8 +++++++- .../aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql | 1 - .../repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json | 5 ----- .../__data__/repository/nginx/assets/create_mv-1.0.0.sql | 6 ++++++ .../__data__/repository/nginx/assets/refresh_mv-1.0.0.sql | 1 - .../__data__/repository/nginx/nginx-1.0.0.json | 5 ----- 9 files changed, 20 insertions(+), 20 deletions(-) delete mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql delete mode 100644 server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql delete mode 100644 server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql index ecb48c084e..86a23f32f9 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql @@ -38,4 +38,10 @@ SELECT classification as `aws.elb.classification`, classification_reason as `aws.elb.classification_reason` FROM - {table_name}; + {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}/checkpoint', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql deleted file mode 100644 index 0f6ba98960..0000000000 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH MATERIALIZED VIEW {table_name}_mview; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 27480a5d07..6fbc6714fe 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -61,11 +61,6 @@ "name": "create_mv", "version": "1.0.0", "language": "sql" - }, - { - "name": "refresh_mv", - "version": "1.0.0", - "language": "sql" } ] }, diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql index 2612638e74..0706db3b5a 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql @@ -21,4 +21,10 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS ELSE 'egress' END AS `aws.vpc.flow-direction` FROM - {table_name}; + {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}/checkpoint', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql deleted file mode 100644 index 458202eaeb..0000000000 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH MATERIALIZED VIEW {table_name}_mview diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index 2a2fbf3f56..6720e0be93 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -57,11 +57,6 @@ "name": "create_mv_vpc", "version": "1.0.0", "language": "sql" - }, - { - "name": "refresh_mv_vpc", - "version": "1.0.0", - "language": "sql" } ] }, diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql index 8b29fdf0b0..cb36863843 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql @@ -8,3 +8,9 @@ SELECT body_bytes_sent AS `http.response.bytes`, 'nginx.access' AS `event.domain` FROM {table_name} +WITH ( + auto_refresh = 'true', + checkpoint_location = '{s3_bucket_location}/checkpoint', + watermark_delay = '1 Minute', + extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}' +); diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql deleted file mode 100644 index 458202eaeb..0000000000 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql +++ /dev/null @@ -1 +0,0 @@ -REFRESH MATERIALIZED VIEW {table_name}_mview diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index 7c63979225..9d95bc0b9f 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -53,11 +53,6 @@ "name": "create_mv", "version": "1.0.0", "language": "sql" - }, - { - "name": "refresh_mv", - "version": "1.0.0", - "language": "sql" } ] },