From 3d70fe6dc708ec9b05695e607674d21c076141d9 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 9 Jun 2023 16:59:13 +0530 Subject: [PATCH 01/27] Initial commit --- procedures_js/alert_processor.js | 2 +- procedures_js/alert_queries_runner.js | 3 ++- tables.tf | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 2466ea4..9868829 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -34,7 +34,7 @@ WHERE alert:ACTOR = ? AND correlation_id IS NOT NULL AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE - AND event_time > DATEADD(minutes, $${CORRELATION_PERIOD_MINUTES}, ?) + AND event_time > DATEADD(minutes, COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), ?) ORDER BY event_time DESC LIMIT 1 ` diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index e2a4f6d..863154d 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -61,7 +61,8 @@ SELECT '$${RUN_ID}' run_id 'DETECTOR', IFNULL(DETECTOR::VARIANT, PARSE_JSON('null')), 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), - 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')) + 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')), + 'CORRELATION_PERIOD', IFNULL(CORRELATION_PERIOD::VARIANT, PARSE_JSON('null')) ) AS alert , alert_time , event_time diff --git a/tables.tf b/tables.tf index 44ecdae..21169b8 100644 --- a/tables.tf +++ b/tables.tf @@ -65,6 +65,15 @@ resource "snowflake_table" "raw_alerts" { type = "VARIANT" } + column { + name = "CORRELATION_PERIOD" + type = "NUMBER(38,0)" + + default { + constant = 1 + } + } + comment = "A raw alerts table." depends_on = [ From 25d3763896d74cff76c8415b7904d1741117c0ef Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 9 Jun 2023 17:16:47 +0530 Subject: [PATCH 02/27] Update warehouse size --- examples/complete/variables.tf | 2 +- examples/simple/variables.tf | 2 +- variables.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index bdc8bbc..c9c4458 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -153,7 +153,7 @@ variable "snowalert_user_email" { variable "snowalert_warehouse_size" { type = string description = "Warehouse size." - default = "X-Small" + default = "XSMALL" } variable "alerts_merge_schedule" { diff --git a/examples/simple/variables.tf b/examples/simple/variables.tf index 13fe0f2..db779ac 100644 --- a/examples/simple/variables.tf +++ b/examples/simple/variables.tf @@ -153,7 +153,7 @@ variable "snowalert_user_email" { variable "snowalert_warehouse_size" { type = string description = "Warehouse size." - default = "X-Small" + default = "XSMALL" } variable "alerts_merge_schedule" { diff --git a/variables.tf b/variables.tf index 92f0128..12fbbc9 100644 --- a/variables.tf +++ b/variables.tf @@ -187,7 +187,7 @@ variable "servicenow_api_url" { variable "snowalert_warehouse_size" { type = string description = "Warehouse size." - default = "X-Small" + default = "XSMALL" } variable "alerts_merge_schedule" { From 34f993c299ce3874c14a456edc8c764e92488a86 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 9 Jun 2023 17:28:05 +0530 Subject: [PATCH 03/27] Update correlation period default --- tables.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tables.tf b/tables.tf index 21169b8..758eb12 100644 --- a/tables.tf +++ b/tables.tf @@ -68,10 +68,6 @@ resource "snowflake_table" "raw_alerts" { column { name = "CORRELATION_PERIOD" type = "NUMBER(38,0)" - - default { - constant = 1 - } } comment = "A raw alerts table." From 5ca9ba6f514f5818b65989714fc8802267cc4bb1 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Mon, 12 Jun 2023 18:12:23 +0530 Subject: [PATCH 04/27] Update tables.tf --- tables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tables.tf b/tables.tf index 758eb12..44ecdae 100644 --- a/tables.tf +++ b/tables.tf @@ -65,11 +65,6 @@ resource "snowflake_table" "raw_alerts" { type = "VARIANT" } - column { - name = "CORRELATION_PERIOD" - type = "NUMBER(38,0)" - } - comment = "A raw alerts table." depends_on = [ From 9b3c24a1460d4d7114932138e5c6895251a316a2 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Tue, 13 Jun 2023 18:40:23 +0530 Subject: [PATCH 05/27] Update alert_processor.js --- procedures_js/alert_processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 9868829..95ed8a0 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -79,7 +79,7 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) -WHERE alert:EVENT_TIME > DATEADD(minutes, $${CORRELATION_PERIOD_MINUTES}, ?) +WHERE alert:EVENT_TIME > DATEADD(minutes, COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), ?) AND alert:ALERT_ID = ? ` From 55c530032dd016e839a67832e9070c05351be2a5 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Mon, 26 Jun 2023 17:49:02 +0530 Subject: [PATCH 06/27] Update alert_processor.js --- procedures_js/alert_processor.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 95ed8a0..a6253ba 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -1,7 +1,7 @@ //args var CORRELATION_PERIOD_MINUTES -CORRELATION_PERIOD_MINUTES = CORRELATION_PERIOD_MINUTES || -60 +CORRELATION_PERIOD_MINUTES = CORRELATION_PERIOD_MINUTES || '60 minutes' var alert_correlation_result_array = [] @@ -34,7 +34,7 @@ WHERE alert:ACTOR = ? AND correlation_id IS NOT NULL AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE - AND event_time > DATEADD(minutes, COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), ?) + AND event_time > ? - INTERVAL COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}) ORDER BY event_time DESC LIMIT 1 ` @@ -79,7 +79,7 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) -WHERE alert:EVENT_TIME > DATEADD(minutes, COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), ?) +WHERE alert:EVENT_TIME > ? - INTERVAL COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}) AND alert:ALERT_ID = ? ` From 60ce877299f9a81145010ecc662ab654ebaf000a Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Mon, 26 Jun 2023 18:53:29 +0530 Subject: [PATCH 07/27] Update alert_processor.js --- procedures_js/alert_processor.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index a6253ba..35aeddc 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -34,7 +34,7 @@ WHERE alert:ACTOR = ? AND correlation_id IS NOT NULL AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE - AND event_time > ? - INTERVAL COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}) + AND event_time > IFF(alert:CORRELATION_PERIOD IS NOT NULL, ? - INTERVAL alert:CORRELATION_PERIOD, ? - INTERVAL $${CORRELATION_PERIOD_MINUTES}) ORDER BY event_time DESC LIMIT 1 ` @@ -63,7 +63,8 @@ function find_related_correlation_id(alert) { action = `["$${o}"]` } - match = exec(GET_CORRELATED_ALERT, [actor, object, action, time])[0] || {} + match = + exec(GET_CORRELATED_ALERT, [actor, object, action, time, time])[0] || {} return match['CORRELATION_ID'] || null } @@ -79,7 +80,7 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) -WHERE alert:EVENT_TIME > ? - INTERVAL COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}) +WHERE alert:EVENT_TIME > IFF(alert:CORRELATION_PERIOD IS NOT NULL, ? - INTERVAL alert:CORRELATION_PERIOD, ? - INTERVAL $${CORRELATION_PERIOD_MINUTES}) AND alert:ALERT_ID = ? ` @@ -94,6 +95,7 @@ for (const row of exec(GET_ALERTS_WITHOUT_CORRELATION_ID)) { alert_correlation_result: exec(UPDATE_ALERT_CORRELATION_ID, [ correlation_id, event_time, + event_time, alert_id, ]), }) From 28f90cf3c88824fdbdaa66b03aeae967da39007e Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Tue, 27 Jun 2023 20:15:49 +0530 Subject: [PATCH 08/27] Use case for parsing correlation period --- procedures_js/alert_processor.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 35aeddc..59c9ca9 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -34,7 +34,15 @@ WHERE alert:ACTOR = ? AND correlation_id IS NOT NULL AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE - AND event_time > IFF(alert:CORRELATION_PERIOD IS NOT NULL, ? - INTERVAL alert:CORRELATION_PERIOD, ? - INTERVAL $${CORRELATION_PERIOD_MINUTES}) + AND event_time > + DATEADD( + CASE + WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 's' THEN 'SECONDS' + WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'm' THEN 'MINUTES' + WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'd' THEN 'DAYS' + END, + - TO_NUMBER(REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+')), + ?) ORDER BY event_time DESC LIMIT 1 ` @@ -63,8 +71,7 @@ function find_related_correlation_id(alert) { action = `["$${o}"]` } - match = - exec(GET_CORRELATED_ALERT, [actor, object, action, time, time])[0] || {} + match = exec(GET_CORRELATED_ALERT, [actor, object, action, time])[0] || {} return match['CORRELATION_ID'] || null } @@ -80,7 +87,15 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) -WHERE alert:EVENT_TIME > IFF(alert:CORRELATION_PERIOD IS NOT NULL, ? - INTERVAL alert:CORRELATION_PERIOD, ? - INTERVAL $${CORRELATION_PERIOD_MINUTES}) +WHERE alert:EVENT_TIME > + DATEADD( + CASE + WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 's' THEN 'SECONDS' + WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'm' THEN 'MINUTES' + WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'd' THEN 'DAYS' + END, + - TO_NUMBER(REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+')), + ?) AND alert:ALERT_ID = ? ` @@ -95,7 +110,6 @@ for (const row of exec(GET_ALERTS_WITHOUT_CORRELATION_ID)) { alert_correlation_result: exec(UPDATE_ALERT_CORRELATION_ID, [ correlation_id, event_time, - event_time, alert_id, ]), }) From 466f8213dbe94a8730dd8263b8819575947a319f Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Wed, 28 Jun 2023 21:35:58 +0530 Subject: [PATCH 09/27] Remove unneeded trim() --- procedures_js/alert_processor.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 59c9ca9..209245c 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -37,11 +37,12 @@ WHERE alert:ACTOR = ? AND event_time > DATEADD( CASE - WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 's' THEN 'SECONDS' - WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'm' THEN 'MINUTES' - WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'd' THEN 'DAYS' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 's' THEN 'SECONDS' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'm' THEN 'MINUTES' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'h' THEN 'HOURS' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'd' THEN 'DAYS' END, - - TO_NUMBER(REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+')), + - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), ?) ORDER BY event_time DESC LIMIT 1 @@ -87,15 +88,16 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) -WHERE alert:EVENT_TIME > - DATEADD( - CASE - WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 's' THEN 'SECONDS' - WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'm' THEN 'MINUTES' - WHEN REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\D+$') = 'd' THEN 'DAYS' - END, - - TO_NUMBER(REGEXP_SUBSTR(TRIM(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+')), - ?) +WHERE alert:EVENT_TIME > + DATEADD( + CASE + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 's' THEN 'SECONDS' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'm' THEN 'MINUTES' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'h' THEN 'HOURS' + WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'd' THEN 'DAYS' + END, + - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), + ?) AND alert:ALERT_ID = ? ` From 3a3cd16243cb4313a38934152d0b1024e3662548 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Wed, 28 Jun 2023 21:57:01 +0530 Subject: [PATCH 10/27] Update alert_processor.js --- procedures_js/alert_processor.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 209245c..f4a6f03 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -37,10 +37,10 @@ WHERE alert:ACTOR = ? AND event_time > DATEADD( CASE - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 's' THEN 'SECONDS' - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'm' THEN 'MINUTES' - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'h' THEN 'HOURS' - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'd' THEN 'DAYS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN 'SECONDS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN 'MINUTES' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN 'HOURS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN 'DAYS' END, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), ?) @@ -91,10 +91,10 @@ SET correlation_id = COALESCE(?, UUID_STRING()) WHERE alert:EVENT_TIME > DATEADD( CASE - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 's' THEN 'SECONDS' - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'm' THEN 'MINUTES' - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'h' THEN 'HOURS' - WHEN REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES}), '[a-zA-Z]+') = 'd' THEN 'DAYS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN 'SECONDS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN 'MINUTES' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN 'HOURS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN 'DAYS' END, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), ?) From a083cc0e1be9ab8b3c29117eed718cac8ae46102 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Thu, 29 Jun 2023 09:56:23 +0530 Subject: [PATCH 11/27] Update alert_processor.js --- procedures_js/alert_processor.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index f4a6f03..7c85872 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -37,10 +37,10 @@ WHERE alert:ACTOR = ? AND event_time > DATEADD( CASE - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN 'SECONDS' - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN 'MINUTES' - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN 'HOURS' - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN 'DAYS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN seconds + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN minutes + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN hours + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN days END, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), ?) @@ -91,10 +91,10 @@ SET correlation_id = COALESCE(?, UUID_STRING()) WHERE alert:EVENT_TIME > DATEADD( CASE - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN 'SECONDS' - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN 'MINUTES' - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN 'HOURS' - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN 'DAYS' + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN seconds + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN minutes + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN hours + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN days END, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), ?) From 89fbd8043839f9b4f22edbcb7775c19a7b1f540a Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Thu, 29 Jun 2023 17:55:16 +0530 Subject: [PATCH 12/27] Update alert_processor.js --- procedures_js/alert_processor.js | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 7c85872..cc2dde6 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -35,15 +35,12 @@ WHERE alert:ACTOR = ? AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE AND event_time > - DATEADD( - CASE - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN seconds - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN minutes - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN hours - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN days - END, - - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), - ?) + CASE + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + END, ORDER BY event_time DESC LIMIT 1 ` @@ -89,15 +86,12 @@ UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) WHERE alert:EVENT_TIME > - DATEADD( - CASE - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN seconds - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN minutes - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN hours - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN days - END, - - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'), - ?) + CASE + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) + END, AND alert:ALERT_ID = ? ` From c955301e4416e9e1453ed0f1a07bbc77711379ab Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Thu, 29 Jun 2023 22:50:58 +0530 Subject: [PATCH 13/27] Update alert_processor.js --- procedures_js/alert_processor.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index cc2dde6..37fd2b9 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -35,12 +35,12 @@ WHERE alert:ACTOR = ? AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE AND event_time > - CASE - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - END, + CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}'')), '[a-z]') + WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + END ORDER BY event_time DESC LIMIT 1 ` @@ -86,12 +86,12 @@ UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) WHERE alert:EVENT_TIME > - CASE - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - WHEN REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '[a-z]') = 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, $${CORRELATION_PERIOD_MINUTES})), '\\d+'),?) - END, + CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}'')), '[a-z]') + WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + END AND alert:ALERT_ID = ? ` From 7c3bd8adda1b12e239d09b26df55515a52db894e Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 18:47:11 +0530 Subject: [PATCH 14/27] Update alert_processor.js --- procedures_js/alert_processor.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 37fd2b9..9167b63 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -29,17 +29,17 @@ function exec(sqlText, binds = []) { GET_CORRELATED_ALERT = ` SELECT correlation_id FROM ${results_alerts_table} -WHERE alert:ACTOR = ? - AND (alert:OBJECT::STRING = ? OR alert:ACTION::STRING = ?) +WHERE alert:ACTOR = :1 + AND (alert:OBJECT::STRING = :2 OR alert:ACTION::STRING = :3) AND correlation_id IS NOT NULL AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE AND event_time > - CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}'')), '[a-z]') - WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) - WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) - WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) - WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), '[a-z]') + WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) + WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) + WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) + WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) END ORDER BY event_time DESC LIMIT 1 @@ -84,15 +84,15 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} -SET correlation_id = COALESCE(?, UUID_STRING()) +SET correlation_id = COALESCE(:1, UUID_STRING()) WHERE alert:EVENT_TIME > - CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}'')), '[a-z]') - WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) - WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) - WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) - WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, ''$${CORRELATION_PERIOD_MINUTES}''), '\\d+')), ?) + CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), '[a-z]') + WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) + WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) + WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) + WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) END - AND alert:ALERT_ID = ? + AND alert:ALERT_ID = :3 ` for (const row of exec(GET_ALERTS_WITHOUT_CORRELATION_ID)) { From 49098f2a8d6b8f8ea8ce706c1fd3a3e67a61d6c1 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 20:04:14 +0530 Subject: [PATCH 15/27] Update alert_queries_runner.js --- procedures_js/alert_queries_runner.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index 863154d..8aed147 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -34,6 +34,21 @@ function fillArray(value, len) { return arr } +function ifColumnExists(column_name) { + column = exec( + `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = ${rules_schema} + AND TABLE_NAME = $${QUERY_NAME} + AND COLUMN_NAME = $${column_name}` + ) + + if (column == column_name) { + return `'CORRELATION_PERIOD', IFNULL(CORRELATION_PERIOD::VARIANT, PARSE_JSON('null')),` + } else { + return '' + } +} + const RUN_ID = Math.random().toString(36).substring(2).toUpperCase() const RAW_ALERTS_TABLE = `${results_raw_alerts_table}` @@ -62,7 +77,7 @@ SELECT '$${RUN_ID}' run_id 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')), - 'CORRELATION_PERIOD', IFNULL(CORRELATION_PERIOD::VARIANT, PARSE_JSON('null')) + ifColumnExists('CORRELATION_PERIOD') ) AS alert , alert_time , event_time From 5a320bbcdee558167bc45b904ada43d34ac71a83 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 20:08:22 +0530 Subject: [PATCH 16/27] Update alert_queries_runner.js --- procedures_js/alert_queries_runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index 8aed147..6532c56 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -77,7 +77,7 @@ SELECT '$${RUN_ID}' run_id 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')), - ifColumnExists('CORRELATION_PERIOD') + $${ifColumnExists('CORRELATION_PERIOD')} ) AS alert , alert_time , event_time From 7ba663ff6428f0081d784e56037a875e158678ef Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 20:20:27 +0530 Subject: [PATCH 17/27] Update procedures.tf --- procedures.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/procedures.tf b/procedures.tf index 3387ab3..8e8e32e 100644 --- a/procedures.tf +++ b/procedures.tf @@ -153,6 +153,7 @@ resource "snowflake_procedure" "alert_queries_runner_with_time" { local.snowalert_database_name, local.rules_schema, ]) + rules_schema_name = local.rules_schema }) depends_on = [ From 16c1c60ccec326d6a2977f25cadb4d374ae6a338 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 20:21:36 +0530 Subject: [PATCH 18/27] Update alert_queries_runner.js --- procedures_js/alert_queries_runner.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index 6532c56..adbb5d1 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -37,13 +37,13 @@ function fillArray(value, len) { function ifColumnExists(column_name) { column = exec( `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = ${rules_schema} + WHERE TABLE_SCHEMA = ${rules_schema_name} AND TABLE_NAME = $${QUERY_NAME} AND COLUMN_NAME = $${column_name}` ) if (column == column_name) { - return `'CORRELATION_PERIOD', IFNULL(CORRELATION_PERIOD::VARIANT, PARSE_JSON('null')),` + return `,'CORRELATION_PERIOD', IFNULL(CORRELATION_PERIOD::VARIANT, PARSE_JSON('null')),` } else { return '' } @@ -76,7 +76,7 @@ SELECT '$${RUN_ID}' run_id 'DETECTOR', IFNULL(DETECTOR::VARIANT, PARSE_JSON('null')), 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), - 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')), + 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')) $${ifColumnExists('CORRELATION_PERIOD')} ) AS alert , alert_time From 76f4891821d775a1a0fec84ffc9f261c1d22b36c Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 20:31:26 +0530 Subject: [PATCH 19/27] Update alert_queries_runner.js --- procedures_js/alert_queries_runner.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index adbb5d1..4b5e5b3 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -37,9 +37,9 @@ function fillArray(value, len) { function ifColumnExists(column_name) { column = exec( `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = ${rules_schema_name} - AND TABLE_NAME = $${QUERY_NAME} - AND COLUMN_NAME = $${column_name}` + WHERE TABLE_SCHEMA = '${rules_schema_name}' + AND TABLE_NAME = '$${QUERY_NAME}' + AND COLUMN_NAME = '$${column_name}'` ) if (column == column_name) { From 4bcc747bf4d90d0d9781d596db5ac2b1e7d58fb2 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Fri, 30 Jun 2023 20:32:09 +0530 Subject: [PATCH 20/27] Update procedures.tf --- procedures.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/procedures.tf b/procedures.tf index 8e8e32e..5d0f42f 100644 --- a/procedures.tf +++ b/procedures.tf @@ -191,6 +191,7 @@ resource "snowflake_procedure" "alert_queries_runner_without_time" { local.snowalert_database_name, local.rules_schema, ]) + rules_schema_name = local.rules_schema }) depends_on = [ @@ -223,6 +224,7 @@ resource "snowflake_procedure" "alert_queries_runner" { local.snowalert_database_name, local.rules_schema, ]) + rules_schema_name = local.rules_schema }) depends_on = [ From 59edcbc0704c160bfbacdf344904dcb8dfee64e2 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Wed, 12 Jul 2023 17:51:56 +0530 Subject: [PATCH 21/27] Use UDF to avoid repetition in statements --- functions.tf | 36 +++++++++++++++++++++++++++ procedures.tf | 10 ++++++++ procedures_js/alert_processor.js | 24 +++++------------- procedures_js/alert_queries_runner.js | 19 ++++---------- 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/functions.tf b/functions.tf index df67edf..1fe1a0c 100644 --- a/functions.tf +++ b/functions.tf @@ -338,3 +338,39 @@ depends_on = [ module.snowalert_grants ] } + +resource "snowflake_function" "convert_time_period_to_seconds" { + provider = snowflake.security_alerting_role + + database = local.snowalert_database_name + schema = local.data_schema + name = "CONVERT_TIME_PERIOD_TO_SECONDS" + + arguments { + name = "PERIOD" + type = "VARCHAR" + } + + return_type = "FLOAT" + language = "javascript" + statement = < - CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), '[a-z]') - WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) - WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) - WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) - WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :4) - END + AND event_time > DATEADD(seconds, - ${data_convert_time_period_to_seconds_function}(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), ?) ORDER BY event_time DESC LIMIT 1 ` @@ -84,15 +78,9 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} -SET correlation_id = COALESCE(:1, UUID_STRING()) -WHERE alert:EVENT_TIME > - CASE REGEXP_SUBSTR(LOWER(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), '[a-z]') - WHEN 's' THEN DATEADD(seconds, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) - WHEN 'm' THEN DATEADD(minutes, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) - WHEN 'h' THEN DATEADD(hours, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) - WHEN 'd' THEN DATEADD(days, - TO_NUMBER(REGEXP_SUBSTR(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}'), '\\\\d+')), :2) - END - AND alert:ALERT_ID = :3 +SET correlation_id = COALESCE(?, UUID_STRING()) +WHERE alert:EVENT_TIME > DATEADD(seconds, - ${data_convert_time_period_to_seconds_function}(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), ?) + AND alert:ALERT_ID = ? ` for (const row of exec(GET_ALERTS_WITHOUT_CORRELATION_ID)) { diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index 4b5e5b3..1a0574b 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -34,19 +34,8 @@ function fillArray(value, len) { return arr } -function ifColumnExists(column_name) { - column = exec( - `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = '${rules_schema_name}' - AND TABLE_NAME = '$${QUERY_NAME}' - AND COLUMN_NAME = '$${column_name}'` - ) - - if (column == column_name) { - return `,'CORRELATION_PERIOD', IFNULL(CORRELATION_PERIOD::VARIANT, PARSE_JSON('null')),` - } else { - return '' - } +function defaultNullReference(columnAndType) { + return `IFNULL(OBJECT_CONSTRUCT(*):` + columnAndType + `, PARSE_JSON('null'))` } const RUN_ID = Math.random().toString(36).substring(2).toUpperCase() @@ -77,7 +66,9 @@ SELECT '$${RUN_ID}' run_id 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')) - $${ifColumnExists('CORRELATION_PERIOD')} + 'CORRELATION_PERIOD', $${defaultNullReference( + 'CORRELATION_PERIOD::VARIANT' + )} ) AS alert , alert_time , event_time From 203c678f3a2a07eee8c02669b236c2735791f009 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Wed, 12 Jul 2023 17:58:05 +0530 Subject: [PATCH 22/27] Update procedures.tf --- procedures.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/procedures.tf b/procedures.tf index 92fe69f..ce5a91b 100644 --- a/procedures.tf +++ b/procedures.tf @@ -163,7 +163,6 @@ resource "snowflake_procedure" "alert_queries_runner_with_time" { local.snowalert_database_name, local.rules_schema, ]) - rules_schema_name = local.rules_schema }) depends_on = [ @@ -201,7 +200,6 @@ resource "snowflake_procedure" "alert_queries_runner_without_time" { local.snowalert_database_name, local.rules_schema, ]) - rules_schema_name = local.rules_schema }) depends_on = [ @@ -234,7 +232,6 @@ resource "snowflake_procedure" "alert_queries_runner" { local.snowalert_database_name, local.rules_schema, ]) - rules_schema_name = local.rules_schema }) depends_on = [ From e14b53966cb24452b6d69636a8d017e96367fdbf Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Wed, 12 Jul 2023 18:09:09 +0530 Subject: [PATCH 23/27] Add missing comma --- procedures_js/alert_queries_runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index 1a0574b..fdf223f 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -65,7 +65,7 @@ SELECT '$${RUN_ID}' run_id 'DETECTOR', IFNULL(DETECTOR::VARIANT, PARSE_JSON('null')), 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), - 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')) + 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')), 'CORRELATION_PERIOD', $${defaultNullReference( 'CORRELATION_PERIOD::VARIANT' )} From 47e20047b8e4b4bde29858a47f139e39b38ee749 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Wed, 12 Jul 2023 18:19:35 +0530 Subject: [PATCH 24/27] Fix regex --- functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions.tf b/functions.tf index 1fe1a0c..91cc07b 100644 --- a/functions.tf +++ b/functions.tf @@ -354,7 +354,7 @@ resource "snowflake_function" "convert_time_period_to_seconds" { return_type = "FLOAT" language = "javascript" statement = < Date: Wed, 12 Jul 2023 18:26:27 +0530 Subject: [PATCH 25/27] Use nullRef function for handlers --- procedures_js/alert_queries_runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procedures_js/alert_queries_runner.js b/procedures_js/alert_queries_runner.js index fdf223f..57a8c1d 100644 --- a/procedures_js/alert_queries_runner.js +++ b/procedures_js/alert_queries_runner.js @@ -65,7 +65,7 @@ SELECT '$${RUN_ID}' run_id 'DETECTOR', IFNULL(DETECTOR::VARIANT, PARSE_JSON('null')), 'EVENT_DATA', IFNULL(EVENT_DATA::VARIANT, PARSE_JSON('null')), 'SEVERITY', IFNULL(SEVERITY::VARIANT, PARSE_JSON('null')), - 'HANDLERS', IFNULL(OBJECT_CONSTRUCT(*):HANDLERS::VARIANT, PARSE_JSON('null')), + 'HANDLERS', $${defaultNullReference('HANDLERS::VARIANT')}, 'CORRELATION_PERIOD', $${defaultNullReference( 'CORRELATION_PERIOD::VARIANT' )} From 91964685786e076ff02ce93becf7a018b0e565b0 Mon Sep 17 00:00:00 2001 From: sfc-gh-nlele Date: Thu, 13 Jul 2023 15:17:32 +0530 Subject: [PATCH 26/27] Improve formatting --- procedures_js/alert_processor.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/procedures_js/alert_processor.js b/procedures_js/alert_processor.js index 0b3f884..14a89af 100644 --- a/procedures_js/alert_processor.js +++ b/procedures_js/alert_processor.js @@ -34,7 +34,11 @@ WHERE alert:ACTOR = ? AND correlation_id IS NOT NULL AND NOT IS_NULL_VALUE(alert:ACTOR) AND suppressed = FALSE - AND event_time > DATEADD(seconds, - ${data_convert_time_period_to_seconds_function}(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), ?) + AND event_time > DATEADD( + seconds, + - ${data_convert_time_period_to_seconds_function}(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), + ? + ) ORDER BY event_time DESC LIMIT 1 ` @@ -79,7 +83,11 @@ WHERE correlation_id IS NULL UPDATE_ALERT_CORRELATION_ID = ` UPDATE ${results_alerts_table} SET correlation_id = COALESCE(?, UUID_STRING()) -WHERE alert:EVENT_TIME > DATEADD(seconds, - ${data_convert_time_period_to_seconds_function}(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), ?) +WHERE alert:EVENT_TIME > DATEADD( + seconds, + - ${data_convert_time_period_to_seconds_function}(COALESCE(alert:CORRELATION_PERIOD, '$${CORRELATION_PERIOD_MINUTES}')), + ? + ) AND alert:ALERT_ID = ? ` From bdbba6b7cb2d4ac19a038434faf0cc091756d3c3 Mon Sep 17 00:00:00 2001 From: Nachiket <77716642+sfc-gh-nlele@users.noreply.github.com> Date: Mon, 17 Jul 2023 12:47:38 +0530 Subject: [PATCH 27/27] Update functions.tf --- functions.tf | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/functions.tf b/functions.tf index 91cc07b..12cc838 100644 --- a/functions.tf +++ b/functions.tf @@ -356,18 +356,12 @@ resource "snowflake_function" "convert_time_period_to_seconds" { statement = <