Update bicep and powershell for managing pipeline logs kusto (#7624)

* Update scripts for deploying kusto resources
* Update kusto schema from prod database
* Update kusto ingestion deployment templates

tools/pipeline-witness/infrastructure/Extract-KustoScripts.ps1

@@ -0,0 +1,100 @@
+<#
+.SYNOPSIS
+    Downloads Kusto schema from a Kusto cluster and saves it to the file system.
+#>
+
+param(
+    [string]$ClusterUri = "https://azsdkengsys.westus2.kusto.windows.net",
+    [string]$DatabaseName = "Pipelines",
+    [string]$OutputPath = (Join-Path $PSScriptRoot "kusto")
+)
+
+$accessToken = az account get-access-token --resource "https://api.kusto.windows.net" --query "accessToken" --output tsv
+$headers = @{ Authorization="Bearer $accessToken" }
+$nl = [System.Environment]::NewLine
+
+function InvokeKustoCommand($command) {
+    $response = Invoke-RestMethod -Uri "$ClusterUri/v1/rest/mgmt" -Headers $headers -Method Post -Body (@{csl=$command; db=$DatabaseName} | ConvertTo-Json) -ContentType "application/json"
+    $columns = $response.Tables[0].Columns
+    $rows = $response.Tables[0].Rows
+
+    $results = @()
+    foreach ($row in $rows) {
+        $obj = @{}
+        for ($i = 0; $i -lt $columns.Length; $i++) {
+            $obj[$columns[$i].ColumnName] = $row[$i]
+        }
+        $results += [PSCustomObject]$obj
+    }
+    return $results
+}
+
+function ToCamelCase($str) {
+    return $str.Substring(0, 1).ToLower() + $str.Substring(1)
+}
+
+function Quote($str, $quoteChar = '"') {
+    $str ??= ""
+    return $quoteChar + $str.Replace('\', '\\').Replace($quoteChar, "\" + $quoteChar) + $quoteChar
+}
+
+function WriteKustoFile($kustoObject, $folder, $fileContents) {
+    $parentFolder = Join-Path $OutputPath $folder
+
+    if($kustoObject.Folder) {
+        $parentFolder = Join-Path $parentFolder $kustoObject.Folder
+    }
+
+    if (-not (Test-Path $parentFolder -PathType Container)) {
+        New-Item -ItemType Directory -Path $parentFolder | Out-Null
+    }
+
+    $filePath = Join-Path $parentFolder "$($kustoObject.Name).kql"
+
+    Set-Content -Path $filePath -Value $fileContents -Encoding ASCII
+}
+
+function ExtractTables {
+    $clusterSchema = (InvokeKustoCommand '.show database schema as json').DatabaseSchema | ConvertFrom-Json
+    $tables = $clusterSchema.Databases.$DatabaseName.Tables.PSObject.Properties.Value
+
+    foreach ($table in $tables) {
+        $fileContents = (
+                ".create-merge table $($table.Name) (",
+                (($table.OrderedColumns | ForEach-Object { "    $($_.Name): $($_.CslType)" }) -join ",$nl"),
+                ") with (folder=$(Quote $table.Folder "'"), docstring=$(Quote $table.DocString "'"))",
+                "",
+                ".create-or-alter table $($table.Name) ingestion json mapping '$($table.Name)`_mapping' ``````[",
+                (($table.OrderedColumns | ForEach-Object { "    { `"column`": `"$($_.Name)`", `"path`": `"$['$(ToCamelCase $_.Name)']`" }" }) -join ",$nl"),
+                "]``````"
+        ) -join $nl
+
+        WriteKustoFile $table "tables" $fileContents
+    }
+}
+
+function ExtractFunctions {
+    $functions = InvokeKustoCommand '.show functions'
+    foreach ($function in $functions) {
+        $fileContents = ".create-or-alter function with (folder=$(Quote $function.Folder "'"), docstring=$(Quote $function.DocString "'")) $($function.Name)$($function.Parameters)$nl$($function.Body)"
+        WriteKustoFile $function "functions" $fileContents
+    }
+}
+
+function ExtractViews {
+    $materializedViews = InvokeKustoCommand '.show materialized-views'
+    foreach ($view in $materializedViews) {
+        $fileContents = (
+                ".create-or-alter materialized-view with (folder=$(Quote $view.Folder "'"), docstring=$(Quote $view.DocString "'")) $($view.Name) on table $($view.SourceTable)",
+                "{",
+                $view.Query,
+                "}"
+        ) -join $nl
+
+        WriteKustoFile $view "views" $fileContents
+    }
+}
+
+ExtractTables
+ExtractFunctions
+ExtractViews

tools/pipeline-witness/infrastructure/Merge-KustoScripts.ps1

@@ -0,0 +1,37 @@
+<#
+.SYNOPSIS
+    Merges Kusto scripts into a single file.
+#>
+param (
+  [Parameter(Mandatory)]
+  [string]$OutputPath
+)
+
+function ReadFiles([IO.FileInfo[]] $files) {
+  foreach ($file in $files) {
+      Write-Output '/////////////////////////////////////////////////////////////////////////////////////////'
+      Write-Output "// Imported from $(Resolve-Path $file.FullName -Relative)"
+      Write-Output ''
+      Get-Content $file
+      Write-Output ''
+      Write-Output ''
+  }
+}
+
+$outputFolder = Split-Path $OutputPath -Parent
+if (-not (Test-Path $outputFolder)) {
+  New-Item -ItemType Directory -Force -Path $outputFolder | Out-Null
+}
+
+$lines = @()
+
+Push-Location "$PSScriptRoot/kusto"
+try {
+  $lines += ReadFiles (Get-ChildItem -Path "./tables/" -Include "*.kql" -Recurse)
+  $lines += ReadFiles (Get-ChildItem -Path "./views/" -Include "*.kql" -Recurse)
+  $lines += ReadFiles (Get-ChildItem -Path "./functions/" -Include "*.kql" -Recurse)
+} finally {
+  Pop-Location
+}
+
+$lines | Set-Content $OutputPath

tools/pipeline-witness/infrastructure/bicep/pipelinelogs.production.json

@@ -2,11 +2,14 @@
   "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
   "contentVersion": "1.0.0.0",
   "parameters": {
+    "resourceGroupName": {
+      "value": "pipelinelogs"
+    },
     "location": {
       "value": "westus2"
     },
     "storageAccountName": {
-      "value": "pipelinelogs"
+      "value": "azsdkengsyspipelinelogs"
     },
     "kustoClusterName": {
       "value": "azsdkengsys"

tools/pipeline-witness/infrastructure/bicep/pipelinelogs.staging.json

@@ -2,6 +2,9 @@
   "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
   "contentVersion": "1.0.0.0",
   "parameters": {
+    "resourceGroupName": {
+      "value": "pipelinelogs"
+    },
     "location": {
       "value": "westus2"
     },

tools/pipeline-witness/infrastructure/bicep/pipelinelogs.test.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "resourceGroupName": {
+      "value": "pipelinelogs-test"
+    },
+    "location": {
+      "value": "westus2"
+    },
+    "storageAccountName": {
+      "value": "pipelinelogstest"
+    },
+    "kustoClusterName": {
+      "value": "azsdkengsystest"
+    },
+    "kustoDatabaseName": {
+      "value": "test"
+    }
+  }
+}

tools/pipeline-witness/infrastructure/bicep/resourceGroup.bicep

@@ -0,0 +1,24 @@
+targetScope='subscription'
+
+param resourceGroupName string
+param location string
+param storageAccountName string
+param kustoClusterName string
+param kustoDatabaseName string
+param deploymentName string = deployment().name
+
+resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = {
+  name: resourceGroupName
+  location: location
+}
+
+module resources 'resources.bicep' = {
+  name: deploymentName
+  scope: resourceGroup
+  params: {
+    storageAccountName: storageAccountName
+    kustoClusterName: kustoClusterName
+    kustoDatabaseName: kustoDatabaseName
+    location: location
+  }
+}

tools/pipeline-witness/infrastructure/bicep/pipelinelogs.bicep → tools/pipeline-witness/infrastructure/bicep/resources.bicep

@@ -1,7 +1,7 @@
-param location string
 param storageAccountName string
 param kustoClusterName string
 param kustoDatabaseName string
+param location string
 
 var tables = [
   {
@@ -25,23 +25,27 @@ var tables = [
     container: 'buildtimelinerecords'
   }
   {
-    name: 'PipelineOwners'
+    name: 'PipelineOwner'
     container: 'pipelineowners'
   }
   {
     name: 'TestRun'
     container: 'testruns'
   }
+  {
+    name: 'TestRunResult'
+    container: 'testrunresults'
+  }
 ]
 
-var kustoScript = loadTextContent('pipelinelogs.kql')
+var kustoScript = loadTextContent('../artifacts/merged.kql')
 
 // Storage Account
 resource storageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = {
   name: storageAccountName
   location: location
   sku: {
-    name: 'Standard_RAGRS'
+    name: 'Standard_LRS'
   }
   kind: 'StorageV2'
   properties: {
@@ -100,7 +104,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = {
 
 // Event Grid
 resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' = {
-  name: storageAccount.name
+  name: storageAccountName
   location: location
   properties: {
     source: storageAccount.id
@@ -109,8 +113,8 @@ resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' = {
 }
 
 // Event Hub
-resource eventhubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = {
-  name: storageAccount.name
+resource eventHubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = {
+  name: storageAccountName
   location: location
   sku: {
     name: 'Standard'
@@ -195,7 +199,7 @@ resource containers 'Microsoft.Storage/storageAccounts/blobServices/containers@2
 }]
 
 resource eventHubs 'Microsoft.EventHub/namespaces/eventhubs@2022-01-01-preview' = [for (table, i) in tables: {
-  parent: eventhubNamespace
+  parent: eventHubNamespace
   name: table.container
   properties: {
     messageRetentionInDays: 7
@@ -230,7 +234,7 @@ resource eventGridSubscriptions 'Microsoft.EventGrid/systemTopics/eventSubscript
 
 resource kustoDataConnections 'Microsoft.Kusto/Clusters/Databases/DataConnections@2022-02-01' = [for (table, i) in tables: {
   parent: kustoCluster::database
-  name: table.container
+  name: '${kustoDatabaseName}-${table.container}'
   location: location
   kind: 'EventGrid'
   properties: {

tools/pipeline-witness/infrastructure/build.ps1

@@ -0,0 +1,26 @@
+<#
+.SYNOPSIS
+    Validates the bicep file build.
+#>
+
+Push-Location $PSScriptRoot
+try {
+    ./Merge-KustoScripts.ps1 -OutputPath "./artifacts/merged.kql"
+    if($?) {
+        Write-Host "Merged KQL files"
+    } else {
+        Write-Error "Failed to merge KQL files"
+        exit 1
+    }
+
+    az bicep build --file "./bicep/resourceGroup.bicep" --outdir "./artifacts"
+    if($?) {
+        Write-Host "Built Bicep files"
+    } else {
+        Write-Error "Failed to build Bicep files"
+        exit 1
+    }
+}
+finally {
+    Pop-Location
+}

tools/pipeline-witness/infrastructure/deploy.ps1

@@ -0,0 +1,39 @@
+<#
+.SYNOPSIS
+    Deploys the kusto cluster, storage account and associated ingestion queue resources to the specified environment.
+#>
+param(
+    [Parameter(Mandatory)]
+    [validateSet('production', 'staging', 'test')]
+    [string]$target
+)
+
+Push-Location $PSScriptRoot
+try {
+    $deploymentName = "pipelinelogs-$target-$(Get-Date -Format 'yyyyMMddHHmm')"
+    $parametersFile = "./bicep/pipelinelogs.$target.json"
+
+    $subscription = az account show --query name -o tsv
+
+    $parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters
+    $location = $parameters.location.value
+    $resourceGroupName = $parameters.resourceGroupName.value
+
+    ./Merge-KustoScripts.ps1 -OutputPath "./artifacts/merged.kql"
+    if($?) {
+        Write-Host "Merged KQL files"
+    } else {
+        Write-Error "Failed to merge KQL files"
+        exit 1
+    }
+
+    Write-Host "Deploying resources to:`n" + `
+               "  Subscription: $subscription`n" + `
+               "  Resource Group: $resourceGroupName`n" + `
+               "  Location: $location`n"
+
+    Write-Host "> az deployment sub create --template-file './bicep/resourceGroup.bicep' --parameters $parametersFile --location $location --name $deploymentName"
+    az deployment sub create --template-file './bicep/resourceGroup.bicep' --parameters $parametersFile --location $location --name $deploymentName
+} finally {
+    Pop-Location
+}