Merge pull request #36 from f-bader/YAMLParamMerge

Add support for scheduled YAML conversion with parameter file

README.md

@@ -2,6 +2,8 @@
 
 # Sentinel Analytics Rule converter
 
+[![PSGallery Version](https://img.shields.io/powershellgallery/v/SentinelARConverter.svg?style=flat&logo=powershell&label=PSGallery%20Version)](https://www.powershellgallery.com/packages/SentinelARConverter) [![PSGallery Downloads](https://img.shields.io/powershellgallery/dt/SentinelARConverter.svg?style=flat&logo=powershell&label=PSGallery%20Downloads)](https://www.powershellgallery.com/packages/SentinelARConverter)
+
 ## Installation
 
 ```PowerShell
@@ -72,8 +74,109 @@ Get-Content "C:\Users\User\Downloads\Azure_Sentinel_analytic_rule.yaml" | Conver
 
 If no output file path is given, the output will be send to `stdout`
 
+```PowerShell
+Convert-SentinelARYamlToArm -Filename "C:\Users\User\Downloads\Azure_Sentinel_analytic_rule.yaml" -ParameterFile "C:\Users\User\Downloads\Azure_Sentinel_analytic_rule.params.yaml" -UseOriginalFilename 
+```
+
+In this case the yaml file is converted and saved with the original file name (`Azure_Sentinel_analytic_rule.json`) but in the process of converting the file additional changes, according to the parameter file are applied.
+
+## Parameter file
+
+There are four different types of parametrization you can use. Each must be defined in it's own subsection.
+
+There is no validation of the values provided, which can result in invalid arm templates.
+
+Only [valid properties](https://learn.microsoft.com/en-us/azure/templates/microsoft.securityinsights/alertrules?pivots=deployment-language-arm-template#scheduledalertruleproperties-1) should be added.
+
+```yaml
+OverwriteProperties:
+  queryFrequency: 1h
+  queryPeriod: 1h
+PrependQuery: |
+  // Example description. Will be added to the beginning of the query.
+AppendQuery: |
+  // Example text that will be added to the end of the query.
+ReplaceQueryVariables:
+  NumberOfErrors: 200
+  ErrorCodes:
+    - "403"
+    - "404"
+```
+
+### OverwriteProperties
+
+Every key found in this section is used to either replace the existing key or is added as a new key to the resulting ARM template. Make sure to use the correct spacing to ensure that the correct keys are overwritten.
+
+```yaml
+OverwriteProperties:
+  queryFrequency: 1h
+  queryPeriod: 1h
+```
+
+In this example the `queryFrequency` and the `queryPeriod` are adjusted.
+
+### PrependQuery
+
+This text will be added to the beginning of the KQL query. If `PrependQuery: |` is used, a newline will be added automatically. If you use `PrependQuery: |-` no newline will be written.
+
+```yaml
+PrependQuery: |
+  // Example description. Will be added to the beginning of the query.
+```
+
+This example adds a description at the top of the KQL query and adds a newline.
+
+### AppendQuery
+
+Add text at the end of the KQL query. This ways you can extend the query, add additional filters or rename certain fields.
+
+```yaml
+AppendQuery: |
+  | extend TimeGenerated = StartTime
+```
+
+Adds the line to the end of the query and adds a new column named `TimeGenerated` based on value of the `StartTime` column.
+
+### ReplaceQueryVariables
+
+This section allows you to use variable names in your original YAML file. They will be replaced by the value provided in the parameter file. There is support for simple string replacement and arrays.
+
+All variables must be named using two percent sign at the beginning and the end e.g. `%%VARIABLENAME%%`.
+
+* String values are replaced as is.
+* Array values are joined together using `","` and a single `"` is added at the start and end. The resulting string is used to replace the variable.
+
+```yaml
+ReplaceQueryVariables:
+  NumberOfErrors: 200
+  ErrorCodes:
+    - 403
+    - 404
+```
+
+* The variable `%%NumberOfErrors%%` will be replaced by the string value `200`
+* Before the variable `%%ErrorCodes%%` will be replaced, the `ErrorCodes` array will be converted into a single string `"403","404"`
+
+This way the following KQL query will be converted...
+
+```kql
+| where Message in (%%ErrorCodes%%)
+| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP
+| where NumberOfErrors > %%NumberOfErrors%%
+```
+...to this result:
+
+```kql
+| where Message in ("403","404")
+| summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP
+| where NumberOfErrors > 200
+```
+
 ## Changelog
 
+### 2.3.0
+ * FEATURE: Add the option to specify a parameter file. This gives a maximum of flexbility to manipulate existing YAML files.
+
 ### v2.2.3
 
  * FEATURE: Add validation and auto-correction of invalid MITRE ATT&CK® tactics and techniques when converting YAML files to ARM templates

src/SentinelARConverter.psd1

@@ -12,7 +12,7 @@
     RootModule        = 'SentinelARConverter.psm1'
 
     # Version number of this module.
-    ModuleVersion     = '2.2.4'
+    ModuleVersion     = '2.3.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @()

src/public/Convert-SentinelARYamlToArm.ps1

@@ -103,6 +103,9 @@ function Convert-SentinelARYamlToArm {
         [Parameter()]
         [string]$Severity,
 
+        [Parameter()]
+        [string]$ParameterFile,
+
         [Parameter()]
         [datetime]$StartRunningAt,
 
@@ -120,6 +123,16 @@ function Convert-SentinelARYamlToArm {
                 throw "File not found"
             }
         }
+
+        if ($ParameterFile) {
+            try {
+                if (-not (Test-Path $ParameterFile)) {
+                    Write-Error -Exception
+                }
+            } catch {
+                throw "Parameters file not found"
+            }
+        }
     }
 
     process {
@@ -143,6 +156,74 @@ function Convert-SentinelARYamlToArm {
             throw "Could not convert source file. YAML might be corrupted"
         }
 
+        try {
+            if ($ParameterFile) {
+                Write-Verbose "Read parameters file `"$ParameterFile`""
+                $Parameters = Get-Content $ParameterFile | ConvertFrom-Yaml
+            } else {
+                Write-Verbose "No parameters file provided"
+            }
+        } catch {
+            throw "Could not convert parameters file. YAML might be corrupted"
+        }
+
+        #region Parameter file handling
+        if ($Parameters) {
+            #region Overwrite values from parameters file
+            if ($Parameters.OverwriteProperties) {
+                foreach ($Key in $Parameters.OverwriteProperties.Keys) {
+                    if ($analyticRule.ContainsKey($Key)) {
+                        Write-Verbose "Overwriting property $Key with $($Parameters.OverwriteProperties[$Key])"
+                        $analyticRule[$Key] = $Parameters.OverwriteProperties[$Key]
+                    } else {
+                        Write-Verbose "Add new property $Key with $($Parameters.OverwriteProperties[$Key])"
+                        $analyticRule.Add($Key, $Parameters.OverwriteProperties[$Key])
+                    }
+                }
+            } else {
+                Write-Verbose "No properties to overwrite in provided parameters file"
+            }
+            #endregion Overwrite values from parameters file
+
+            #region Prepend KQL query with data from parameters file
+            if ($Parameters.PrependQuery) {
+                $analyticRule.query = $Parameters.PrependQuery + $analyticRule.query
+            } else {
+                Write-Verbose "No query to prepend in provided parameters file"
+            }
+            #endregion Prepend KQL query with data from parameters file
+
+            #region Append KQL query with data from parameters file
+            if ($Parameters.AppendQuery) {
+                $analyticRule.query = $analyticRule.query + $Parameters.AppendQuery
+            } else {
+                Write-Verbose "No query to append in provided parameters file"
+            }
+            #endregion Append KQL query with data from parameters file
+
+            #region Replace variables in KQL query with data from parameters file
+            if ($Parameters.ReplaceQueryVariables) {
+                foreach ($Key in $Parameters.ReplaceQueryVariables.Keys) {
+                    if ($Parameters.ReplaceQueryVariables[$Key].Count -gt 1) {
+                        # Join array values with comma and wrap in quotes
+                        $ReplaceValue = $Parameters.ReplaceQueryVariables[$Key] -join '","'
+                        $ReplaceValue = '"' + $ReplaceValue + '"'
+                    } else {
+                        # Use single value
+                        $ReplaceValue = $Parameters.ReplaceQueryVariables[$Key]
+                    }
+                    Write-Verbose "Replacing variable %%$Key%% with $($ReplaceValue)"
+                    $analyticRule.query = $analyticRule.query -replace "%%$($Key)%%", $ReplaceValue
+                }
+            } else {
+                Write-Verbose "No variables to replace in provided parameters file"
+            }
+            #endregion Replace variables in KQL query with data from parameters file
+
+            Write-Verbose "$($analyticRule | ConvertTo-Json -Depth 99)"
+        }
+        #endregion Parameter file handling
+
         if ( [string]::IsNullOrWhiteSpace($analyticRule.name) -or [string]::IsNullOrWhiteSpace($analyticRule.id) ) {
             throw "Analytics Rule name or id is empty. YAML might be corrupted"
         }

tests/Convert-SentinelARYamlToArm.tests.ps1

@@ -12,6 +12,12 @@ param(
     [String]
     $exampleScheduledTTPFilePath = "./tests/examples/TTPWithTacticsNTechniques.yaml",
     [Parameter()]
+    [String]
+    $exampleScheduledWithVariablesFilePath = "./tests/examples/ScheduledParam.yaml",
+    [Parameter()]
+    [String]
+    $exampleScheduledParameterFilePath = "./tests/examples/ScheduledParam.params.yaml",
+    [Parameter()]
     [Switch]
     $RetainTestFiles = $false
 )
@@ -393,6 +399,41 @@ Describe "Convert-SentinelARYamlToArm" {
         }
     }
 
+    Context "Scheduled with parameter file provided" {
+        BeforeAll {
+            Copy-Item -Path $exampleScheduledWithVariablesFilePath -Destination "TestDrive:/Scheduled.yaml" -Force
+            Copy-Item -Path $exampleScheduledParameterFilePath -Destination "TestDrive:/ScheduledParam.params.yaml" -Force
+            Convert-SentinelARYamlToArm -Filename "TestDrive:/Scheduled.yaml" -OutFile "TestDrive:/Scheduled.json" -ParameterFile "TestDrive:/ScheduledParam.params.yaml"
+            $armTemplate = Get-Content -Path "TestDrive:/Scheduled.json" -Raw | ConvertFrom-Json
+        }
+
+        AfterEach {
+            if ( -not $RetainTestFiles) {
+                Remove-Item -Path "TestDrive:/*" -Include *.json -Force
+            }
+        }
+
+        It "Should have the correct replaced parameter values" {
+            $armTemplate.resources.properties.queryPeriod | Should -Be "PT1H"
+            $armTemplate.resources.properties.queryFrequency | Should -Be "PT1H"
+            $armTemplate.resources.properties.description | Should -Match "'403' or '404'"
+        }
+
+        It "Should have all new properties from parameter file" {
+            $armTemplate.resources.properties.customDetails | Should -Not -BeNullOrEmpty
+        }
+
+        It "Should have the correct added query values" {
+            $armTemplate.resources.properties.query | Should -Match "// Example description. Will be added to the beginning of the query."
+            $armTemplate.resources.properties.query | Should -Match "// Example text that will be added to the end of the query."
+        }
+
+        It "Should have the correct variables replaced" {
+            $armTemplate.resources.properties.query | Should -Match 'where Message in \("403","404"\)'
+            $armTemplate.resources.properties.query | Should -Match 'where NumberOfErrors > 200'
+        }
+    }
+
     AfterAll {
         Remove-Module SentinelARConverter -Force
     }

tests/examples/ScheduledParam.params.yaml

@@ -0,0 +1,18 @@
+OverwriteProperties:
+  description: |-
+    Identifies instances where Wazuh logged over 200 '403' or '404' Web Errors from one IP Address.
+    To onboard Wazuh data into Sentinel please view: https://github.com/wazuh/wazuh-documentation/blob/master/source/azure/monitoring%20activity.rst
+  queryFrequency: 1h
+  queryPeriod: 1h
+  customDetails:
+    HostName: HostName
+PrependQuery: |
+  // Example description. Will be added to the beginning of the query.
+AppendQuery: |
+  // Example text that will be added to the end of the query.
+  | extend TimeGenerated = StartTime
+ReplaceQueryVariables:
+  NumberOfErrors: 200
+  ErrorCodes:
+    - 403
+    - "404"

tests/examples/ScheduledParam.yaml

@@ -0,0 +1,42 @@
+id: 2790795b-7dba-483e-853f-44aa0bc9c985
+name: Wazuh - Large Number of Web errors from an IP
+description: |
+  'Identifies instances where Wazuh logged over 400 '403' Web Errors from one IP Address. To onboard Wazuh data into Sentinel please view: https://github.com/wazuh/wazuh-documentation/blob/master/source/azure/monitoring%20activity.rst'
+severity: Low
+requiredDataConnectors: []
+queryFrequency: 1d
+queryPeriod: 1d
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - Persistence
+query: |
+  CommonSecurityLog
+  | where DeviceProduct =~ "Wazuh"
+  | where Activity has "Web server 400 error code."
+  | where Message in (%%ErrorCodes%%)
+  | extend HostName=substring(split(DeviceCustomString1,")")[0],1)
+  | summarize StartTime = min(TimeGenerated), EndTime = max(TimeGenerated), NumberOfErrors = dcount(SourceIP) by HostName, SourceIP
+  | where NumberOfErrors > %%NumberOfErrors%%
+  | sort by NumberOfErrors desc
+  | extend timestamp = StartTime
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: HostName
+        columnName: HostName
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: SourceIP
+version: 1.0.3
+kind: Scheduled
+metadata:
+  source:
+    kind: Community
+  author:
+    name: Jordan Ross
+  support:
+    tier: Community
+  categories:
+    domains: ["Security - Others", "Networking"]