[Backport 2.x] Remove manual refresh for S3 integrations (#1227)

(cherry picked from commit 80f112a) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Jan 9, 2024 · 21035f7 · 21035f7
1 parent 5454b17
commit 21035f7
Show file tree

Hide file tree

Showing 9 changed files with 20 additions and 20 deletions.
diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql
@@ -38,4 +38,10 @@ SELECT
     classification as `aws.elb.classification`,
     classification_reason as `aws.elb.classification_reason`
 FROM
-    {table_name};
+    {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_bucket_location}/checkpoint',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);
diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql
diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json
@@ -61,11 +61,6 @@
         "name": "create_mv",
         "version": "1.0.0",
         "language": "sql"
-      },
-      {
-        "name": "refresh_mv",
-        "version": "1.0.0",
-        "language": "sql"
       }
     ]
   },

diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql
@@ -21,4 +21,10 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS
           ELSE 'egress'
         END AS `aws.vpc.flow-direction`
 FROM
-    {table_name};
+    {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_bucket_location}/checkpoint',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+)
diff --git a/...er/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql b/...er/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql
diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json
@@ -57,11 +57,6 @@
         "name": "create_mv_vpc",
         "version": "1.0.0",
         "language": "sql"
-      },
-      {
-        "name": "refresh_mv_vpc",
-        "version": "1.0.0",
-        "language": "sql"
       }
     ]
   },

diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql
@@ -8,3 +8,9 @@ SELECT
     body_bytes_sent AS `http.response.bytes`,
     'nginx.access' AS `event.domain`
 FROM {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_bucket_location}/checkpoint',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);
diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql
diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json
@@ -53,11 +53,6 @@
         "name": "create_mv",
         "version": "1.0.0",
         "language": "sql"
-      },
-      {
-        "name": "refresh_mv",
-        "version": "1.0.0",
-        "language": "sql"
       }
     ]
   },