Remove manual refresh for S3 integrations (opensearch-project#1226)

amsiglan · Nov 2, 2023 · 80f112a · 80f112a
1 parent ae76f36
commit 80f112a
Show file tree

Hide file tree

Showing 10 changed files with 27 additions and 21 deletions.
diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx
@@ -397,7 +397,13 @@ export function SetupBottomBar({
                     '{table_name}',
                     `${config.connectionDataSource}.default.${config.connectionTableName}`
                   );
-                  queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation);
+                  // We append to this URI in internal queries, so we normalize it to have no trailing slash
+                  let trimmedLocation = config.connectionLocation.trim();
+                  trimmedLocation = trimmedLocation.endsWith('/')
+                    ? trimmedLocation.slice(0, trimmedLocation.length - 1)
+                    : trimmedLocation;
+
+                  queryStr = queryStr.replaceAll('{s3_bucket_location}', trimmedLocation);
                   queryStr = queryStr.replaceAll('{object_name}', config.connectionTableName);
                   queryStr = queryStr.replaceAll(/\s+/g, ' ');
                   const result = await runQuery(queryStr, config.connectionDataSource, sessionId);

diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql
@@ -38,4 +38,10 @@ SELECT
     classification as `aws.elb.classification`,
     classification_reason as `aws.elb.classification_reason`
 FROM
-    {table_name};
+    {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_bucket_location}/checkpoint',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);
diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql
diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json
@@ -61,11 +61,6 @@
         "name": "create_mv",
         "version": "1.0.0",
         "language": "sql"
-      },
-      {
-        "name": "refresh_mv",
-        "version": "1.0.0",
-        "language": "sql"
       }
     ]
   },

diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/create_mv_vpc-1.0.0.sql
@@ -21,4 +21,10 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS
           ELSE 'egress'
         END AS `aws.vpc.flow-direction`
 FROM
-    {table_name};
+    {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_bucket_location}/checkpoint',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+)
diff --git a/...er/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql b/...er/adaptors/integrations/__data__/repository/aws_vpc_flow/assets/refresh_mv_vpc-1.0.0.sql
diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json
@@ -57,11 +57,6 @@
         "name": "create_mv_vpc",
         "version": "1.0.0",
         "language": "sql"
-      },
-      {
-        "name": "refresh_mv_vpc",
-        "version": "1.0.0",
-        "language": "sql"
       }
     ]
   },

diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_mv-1.0.0.sql
@@ -8,3 +8,9 @@ SELECT
     body_bytes_sent AS `http.response.bytes`,
     'nginx.access' AS `event.domain`
 FROM {table_name}
+WITH (
+    auto_refresh = 'true',
+    checkpoint_location = '{s3_bucket_location}/checkpoint',
+    watermark_delay = '1 Minute',
+    extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
+);
diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/refresh_mv-1.0.0.sql
diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json
@@ -49,11 +49,6 @@
         "name": "create_mv",
         "version": "1.0.0",
         "language": "sql"
-      },
-      {
-        "name": "refresh_mv",
-        "version": "1.0.0",
-        "language": "sql"
       }
     ]
   },