diff --git a/tools/pipeline-witness/infrastructure/Extract-KustoScripts.ps1 b/tools/pipeline-witness/infrastructure/Extract-KustoScripts.ps1 new file mode 100644 index 00000000000..77c0035daf9 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/Extract-KustoScripts.ps1 @@ -0,0 +1,100 @@ +<# +.SYNOPSIS + Downloads Kusto schema from a Kusto cluster and saves it to the file system. +#> + +param( + [string]$ClusterUri = "https://azsdkengsys.westus2.kusto.windows.net", + [string]$DatabaseName = "Pipelines", + [string]$OutputPath = (Join-Path $PSScriptRoot "kusto") +) + +$accessToken = az account get-access-token --resource "https://api.kusto.windows.net" --query "accessToken" --output tsv +$headers = @{ Authorization="Bearer $accessToken" } +$nl = [System.Environment]::NewLine + +function InvokeKustoCommand($command) { + $response = Invoke-RestMethod -Uri "$ClusterUri/v1/rest/mgmt" -Headers $headers -Method Post -Body (@{csl=$command; db=$DatabaseName} | ConvertTo-Json) -ContentType "application/json" + $columns = $response.Tables[0].Columns + $rows = $response.Tables[0].Rows + + $results = @() + foreach ($row in $rows) { + $obj = @{} + for ($i = 0; $i -lt $columns.Length; $i++) { + $obj[$columns[$i].ColumnName] = $row[$i] + } + $results += [PSCustomObject]$obj + } + return $results +} + +function ToCamelCase($str) { + return $str.Substring(0, 1).ToLower() + $str.Substring(1) +} + +function Quote($str, $quoteChar = '"') { + $str ??= "" + return $quoteChar + $str.Replace('\', '\\').Replace($quoteChar, "\" + $quoteChar) + $quoteChar +} + +function WriteKustoFile($kustoObject, $folder, $fileContents) { + $parentFolder = Join-Path $OutputPath $folder + + if($kustoObject.Folder) { + $parentFolder = Join-Path $parentFolder $kustoObject.Folder + } + + if (-not (Test-Path $parentFolder -PathType Container)) { + New-Item -ItemType Directory -Path $parentFolder | Out-Null + } + + $filePath = Join-Path $parentFolder "$($kustoObject.Name).kql" + + Set-Content -Path $filePath -Value $fileContents -Encoding ASCII +} + +function ExtractTables { + $clusterSchema = (InvokeKustoCommand '.show database schema as json').DatabaseSchema | ConvertFrom-Json + $tables = $clusterSchema.Databases.$DatabaseName.Tables.PSObject.Properties.Value + + foreach ($table in $tables) { + $fileContents = ( + ".create-merge table $($table.Name) (", + (($table.OrderedColumns | ForEach-Object { " $($_.Name): $($_.CslType)" }) -join ",$nl"), + ") with (folder=$(Quote $table.Folder "'"), docstring=$(Quote $table.DocString "'"))", + "", + ".create-or-alter table $($table.Name) ingestion json mapping '$($table.Name)`_mapping' ``````[", + (($table.OrderedColumns | ForEach-Object { " { `"column`": `"$($_.Name)`", `"path`": `"$['$(ToCamelCase $_.Name)']`" }" }) -join ",$nl"), + "]``````" + ) -join $nl + + WriteKustoFile $table "tables" $fileContents + } +} + +function ExtractFunctions { + $functions = InvokeKustoCommand '.show functions' + foreach ($function in $functions) { + $fileContents = ".create-or-alter function with (folder=$(Quote $function.Folder "'"), docstring=$(Quote $function.DocString "'")) $($function.Name)$($function.Parameters)$nl$($function.Body)" + WriteKustoFile $function "functions" $fileContents + } +} + +function ExtractViews { + $materializedViews = InvokeKustoCommand '.show materialized-views' + foreach ($view in $materializedViews) { + $fileContents = ( + ".create-or-alter materialized-view with (folder=$(Quote $view.Folder "'"), docstring=$(Quote $view.DocString "'")) $($view.Name) on table $($view.SourceTable)", + "{", + $view.Query, + "}" + ) -join $nl + + WriteKustoFile $view "views" $fileContents + } +} + +ExtractTables +ExtractFunctions +ExtractViews diff --git a/tools/pipeline-witness/infrastructure/Merge-KustoScripts.ps1 b/tools/pipeline-witness/infrastructure/Merge-KustoScripts.ps1 new file mode 100644 index 00000000000..329fe54dd7d --- /dev/null +++ b/tools/pipeline-witness/infrastructure/Merge-KustoScripts.ps1 @@ -0,0 +1,37 @@ +<# +.SYNOPSIS + Merges Kusto scripts into a single file. +#> +param ( + [Parameter(Mandatory)] + [string]$OutputPath +) + +function ReadFiles([IO.FileInfo[]] $files) { + foreach ($file in $files) { + Write-Output '/////////////////////////////////////////////////////////////////////////////////////////' + Write-Output "// Imported from $(Resolve-Path $file.FullName -Relative)" + Write-Output '' + Get-Content $file + Write-Output '' + Write-Output '' + } +} + +$outputFolder = Split-Path $OutputPath -Parent +if (-not (Test-Path $outputFolder)) { + New-Item -ItemType Directory -Force -Path $outputFolder | Out-Null +} + +$lines = @() + +Push-Location "$PSScriptRoot/kusto" +try { + $lines += ReadFiles (Get-ChildItem -Path "./tables/" -Include "*.kql" -Recurse) + $lines += ReadFiles (Get-ChildItem -Path "./views/" -Include "*.kql" -Recurse) + $lines += ReadFiles (Get-ChildItem -Path "./functions/" -Include "*.kql" -Recurse) +} finally { + Pop-Location +} + +$lines | Set-Content $OutputPath diff --git a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.production.json b/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.production.json index 27bb9738ab6..3fcddf20a24 100644 --- a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.production.json +++ b/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.production.json @@ -2,11 +2,14 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { + "resourceGroupName": { + "value": "pipelinelogs" + }, "location": { "value": "westus2" }, "storageAccountName": { - "value": "pipelinelogs" + "value": "azsdkengsyspipelinelogs" }, "kustoClusterName": { "value": "azsdkengsys" diff --git a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.staging.json b/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.staging.json index 9123b703665..74d5f27ca3c 100644 --- a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.staging.json +++ b/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.staging.json @@ -2,6 +2,9 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { + "resourceGroupName": { + "value": "pipelinelogs" + }, "location": { "value": "westus2" }, diff --git a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.test.json b/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.test.json new file mode 100644 index 00000000000..a70cd6b9824 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.test.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "resourceGroupName": { + "value": "pipelinelogs-test" + }, + "location": { + "value": "westus2" + }, + "storageAccountName": { + "value": "pipelinelogstest" + }, + "kustoClusterName": { + "value": "azsdkengsystest" + }, + "kustoDatabaseName": { + "value": "test" + } + } +} \ No newline at end of file diff --git a/tools/pipeline-witness/infrastructure/bicep/resourceGroup.bicep b/tools/pipeline-witness/infrastructure/bicep/resourceGroup.bicep new file mode 100644 index 00000000000..f2991037a4f --- /dev/null +++ b/tools/pipeline-witness/infrastructure/bicep/resourceGroup.bicep @@ -0,0 +1,24 @@ +targetScope='subscription' + +param resourceGroupName string +param location string +param storageAccountName string +param kustoClusterName string +param kustoDatabaseName string +param deploymentName string = deployment().name + +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: location +} + +module resources 'resources.bicep' = { + name: deploymentName + scope: resourceGroup + params: { + storageAccountName: storageAccountName + kustoClusterName: kustoClusterName + kustoDatabaseName: kustoDatabaseName + location: location + } +} diff --git a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.bicep b/tools/pipeline-witness/infrastructure/bicep/resources.bicep similarity index 93% rename from tools/pipeline-witness/infrastructure/bicep/pipelinelogs.bicep rename to tools/pipeline-witness/infrastructure/bicep/resources.bicep index cd21aa8ec8c..57f9c85c4b2 100644 --- a/tools/pipeline-witness/infrastructure/bicep/pipelinelogs.bicep +++ b/tools/pipeline-witness/infrastructure/bicep/resources.bicep @@ -1,7 +1,7 @@ -param location string param storageAccountName string param kustoClusterName string param kustoDatabaseName string +param location string var tables = [ { @@ -25,23 +25,27 @@ var tables = [ container: 'buildtimelinerecords' } { - name: 'PipelineOwners' + name: 'PipelineOwner' container: 'pipelineowners' } { name: 'TestRun' container: 'testruns' } + { + name: 'TestRunResult' + container: 'testrunresults' + } ] -var kustoScript = loadTextContent('pipelinelogs.kql') +var kustoScript = loadTextContent('../artifacts/merged.kql') // Storage Account resource storageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { name: storageAccountName location: location sku: { - name: 'Standard_RAGRS' + name: 'Standard_LRS' } kind: 'StorageV2' properties: { @@ -100,7 +104,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2021-09-01' = { // Event Grid resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' = { - name: storageAccount.name + name: storageAccountName location: location properties: { source: storageAccount.id @@ -109,8 +113,8 @@ resource eventGridTopic 'Microsoft.EventGrid/systemTopics@2022-06-15' = { } // Event Hub -resource eventhubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = { - name: storageAccount.name +resource eventHubNamespace 'Microsoft.EventHub/namespaces@2022-01-01-preview' = { + name: storageAccountName location: location sku: { name: 'Standard' @@ -195,7 +199,7 @@ resource containers 'Microsoft.Storage/storageAccounts/blobServices/containers@2 }] resource eventHubs 'Microsoft.EventHub/namespaces/eventhubs@2022-01-01-preview' = [for (table, i) in tables: { - parent: eventhubNamespace + parent: eventHubNamespace name: table.container properties: { messageRetentionInDays: 7 @@ -230,7 +234,7 @@ resource eventGridSubscriptions 'Microsoft.EventGrid/systemTopics/eventSubscript resource kustoDataConnections 'Microsoft.Kusto/Clusters/Databases/DataConnections@2022-02-01' = [for (table, i) in tables: { parent: kustoCluster::database - name: table.container + name: '${kustoDatabaseName}-${table.container}' location: location kind: 'EventGrid' properties: { diff --git a/tools/pipeline-witness/infrastructure/build.ps1 b/tools/pipeline-witness/infrastructure/build.ps1 new file mode 100644 index 00000000000..47af9ea0ddb --- /dev/null +++ b/tools/pipeline-witness/infrastructure/build.ps1 @@ -0,0 +1,26 @@ +<# +.SYNOPSIS + Validates the bicep file build. +#> + +Push-Location $PSScriptRoot +try { + ./Merge-KustoScripts.ps1 -OutputPath "./artifacts/merged.kql" + if($?) { + Write-Host "Merged KQL files" + } else { + Write-Error "Failed to merge KQL files" + exit 1 + } + + az bicep build --file "./bicep/resourceGroup.bicep" --outdir "./artifacts" + if($?) { + Write-Host "Built Bicep files" + } else { + Write-Error "Failed to build Bicep files" + exit 1 + } +} +finally { + Pop-Location +} diff --git a/tools/pipeline-witness/infrastructure/deploy.ps1 b/tools/pipeline-witness/infrastructure/deploy.ps1 new file mode 100644 index 00000000000..f8625bbac5c --- /dev/null +++ b/tools/pipeline-witness/infrastructure/deploy.ps1 @@ -0,0 +1,39 @@ +<# +.SYNOPSIS + Deploys the kusto cluster, storage account and associated ingestion queue resources to the specified environment. +#> +param( + [Parameter(Mandatory)] + [validateSet('production', 'staging', 'test')] + [string]$target +) + +Push-Location $PSScriptRoot +try { + $deploymentName = "pipelinelogs-$target-$(Get-Date -Format 'yyyyMMddHHmm')" + $parametersFile = "./bicep/pipelinelogs.$target.json" + + $subscription = az account show --query name -o tsv + + $parameters = (Get-Content -Path $parametersFile -Raw | ConvertFrom-Json).parameters + $location = $parameters.location.value + $resourceGroupName = $parameters.resourceGroupName.value + + ./Merge-KustoScripts.ps1 -OutputPath "./artifacts/merged.kql" + if($?) { + Write-Host "Merged KQL files" + } else { + Write-Error "Failed to merge KQL files" + exit 1 + } + + Write-Host "Deploying resources to:`n" + ` + " Subscription: $subscription`n" + ` + " Resource Group: $resourceGroupName`n" + ` + " Location: $location`n" + + Write-Host "> az deployment sub create --template-file './bicep/resourceGroup.bicep' --parameters $parametersFile --location $location --name $deploymentName" + az deployment sub create --template-file './bicep/resourceGroup.bicep' --parameters $parametersFile --location $location --name $deploymentName +} finally { + Pop-Location +} \ No newline at end of file diff --git a/tools/pipeline-witness/infrastructure/kusto/Merge-KustoScripts.ps1 b/tools/pipeline-witness/infrastructure/kusto/Merge-KustoScripts.ps1 deleted file mode 100644 index 690661c8f7f..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/Merge-KustoScripts.ps1 +++ /dev/null @@ -1,23 +0,0 @@ -param ( - [Parameter(Mandatory=$true)] - [string]$OutputPath -) - -function ReadFiles([IO.FileInfo[]] $files) { - foreach ($file in $files) { - Write-Output '/////////////////////////////////////////////////////////////////////////////////////////' - Write-Output "// Imported from $(Resolve-Path $file.FullName -Relative)" - Write-Output '' - Get-Content $file - Write-Output '' - Write-Output '' - } -} - -$lines = @() - -$lines += ReadFiles (Get-ChildItem -Path "$PSScriptRoot/tables/" -Include "*.kql" -Recurse) -$lines += ReadFiles (Get-ChildItem -Path "$PSScriptRoot/views/" -Include "*.kql" -Recurse) -$lines += ReadFiles (Get-ChildItem -Path "$PSScriptRoot/functions/" -Include "*.kql" -Recurse) - -$lines | Set-Content $OutputPath diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/DashboardQuery.kql b/tools/pipeline-witness/infrastructure/kusto/functions/DashboardQuery.kql deleted file mode 100644 index dc6154621e7..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/DashboardQuery.kql +++ /dev/null @@ -1,14 +0,0 @@ -.create-or-alter function DashboardQuery(platformFilter:dynamic, serviceFilter:dynamic, buildTypeFilter:dynamic, triggerFilter:dynamic, resultFilter:dynamic, sourceBranchFilter:string, definitionNameFilter:string, ownerFilter:string) -{ - ManagedDefinitionBuild(sourceBranchFilter) - | where isnull(platformFilter) or Platform in (platformFilter) - | where isnull(serviceFilter) or Service in (serviceFilter) - | where isnull(buildTypeFilter) or BuildType in (buildTypeFilter) - | where isnull(triggerFilter) or Trigger in (triggerFilter) - | where isnull(resultFilter) or Result in (resultFilter) - | where BuildDefinitionName matches regex coalesce(definitionNameFilter, ".") - | where Owners contains (ownerFilter) - | join kind = leftouter BuildFailure on ProjectId, BuildDefinitionId, $left.FinishTime == $right.BuildFinishTime - | project BuildDefinitionId, FinishTime, BuildDefinitionName, ErrorClassification, Platform, Service, Result, BuildType, Trigger, SourceBranch, Owners, ProjectName, ProjectId - | extend BuildDefinitionIdUrl = strcat("https://dev.azure.com/azure-sdk/", ProjectName, "/_build?definitionId=", tostring(BuildDefinitionId)) -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/LatestBuild.kql b/tools/pipeline-witness/infrastructure/kusto/functions/LatestBuild.kql index 66a97cba7db..3518ec22e2f 100644 --- a/tools/pipeline-witness/infrastructure/kusto/functions/LatestBuild.kql +++ b/tools/pipeline-witness/infrastructure/kusto/functions/LatestBuild.kql @@ -1,5 +1,5 @@ -.create-or-alter function LatestBuild(sourceBranchFilter:string) -{ +.create-or-alter function with (folder='', docstring='') LatestBuild(sourceBranchFilter:string) +{ Build | where SourceBranch matches regex sourceBranchFilter | summarize arg_max(FinishTime, *) by DefinitionId diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/LatestDefinition.kql b/tools/pipeline-witness/infrastructure/kusto/functions/LatestDefinition.kql index ebce887e38f..23782890d98 100644 --- a/tools/pipeline-witness/infrastructure/kusto/functions/LatestDefinition.kql +++ b/tools/pipeline-witness/infrastructure/kusto/functions/LatestDefinition.kql @@ -1,5 +1,5 @@ -.create-or-alter function LatestDefinition() -{ +.create-or-alter function with (folder='', docstring='') LatestDefinition() +{ BuildDefinition | summarize arg_max(BuildDefinitionRevision, *) by BuildDefinitionId, ProjectId } diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/LatestOwners.kql b/tools/pipeline-witness/infrastructure/kusto/functions/LatestOwners.kql index 3e868a8482d..3f2222c92dd 100644 --- a/tools/pipeline-witness/infrastructure/kusto/functions/LatestOwners.kql +++ b/tools/pipeline-witness/infrastructure/kusto/functions/LatestOwners.kql @@ -1,5 +1,5 @@ -.create-or-alter function LatestOwners() -{ +.create-or-alter function with (folder='', docstring='') LatestOwners() +{ PipelineOwner | summarize arg_max(Timestamp, *) by BuildDefinitionId } diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedBuild.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedBuild.kql deleted file mode 100644 index ff7a1000b9b..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedBuild.kql +++ /dev/null @@ -1,37 +0,0 @@ -.create-or-alter function ManagedBuild(platformFilter:dynamic, serviceFilter:dynamic, buildTypeFilter:dynamic, triggerFilter:dynamic, resultFilter:dynamic, sourceBranchFilter:string, definitionNameFilter:string) -{ - ManagedDefinition - | extend BuildDefinitionId = BuildDefinitionId - | join kind=inner PipelineOwner on BuildDefinitionId - | extend BuildDefinitionId = tolong(BuildDefinitionId) - | join kind = inner Build on ProjectId, $left.BuildDefinitionId == $right.DefinitionId - | join kind = leftouter BuildFailure on ProjectId, BuildDefinitionId, $left.FinishTime == $right.BuildFinishTime - | project - ProjectId, - ProjectName, - BuildDefinitionName, - BuildDefinitionId, - BuildDefinitionRevision, - Platform, - Service = Feature, - BuildType = Category, - Trigger = Reason, - Result, - SourceBranch, - FinishTime, - ErrorClassification, - Owners - | summarize arg_max(FinishTime, *) by BuildDefinitionId, ProjectId - | where isnull(platformFilter) or Platform in (platformFilter) - | where isnull(serviceFilter) or Service in (serviceFilter) - | where isnull(buildTypeFilter) or BuildType in (buildTypeFilter) - | where isnull(triggerFilter) or Trigger in (triggerFilter) - | where isnull(resultFilter) or Result in (resultFilter) - | where BuildDefinitionName matches regex coalesce(definitionNameFilter, ".") - | where SourceBranch matches regex (sourceBranchFilter) - | extend Ranking = case(Result == 'failed', 1, Result == 'partiallySucceeded', 2, Result == 'canceled', 3, 4) - | order by Ranking asc - | project-away Ranking, BuildDefinitionRevision - | extend BuildDefinitionIdUrl = strcat("https://dev.azure.com/azure-sdk/", ProjectName, "/_build?definitionId=", tostring(BuildDefinitionId)) - | mv-expand Platform to typeof(string), Service to typeof(string), BuildType to typeof(string) -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedBuild2.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedBuild2.kql deleted file mode 100644 index b7fb62aa195..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedBuild2.kql +++ /dev/null @@ -1,40 +0,0 @@ -.create-or-alter function ManagedBuild2(platform:dynamic=dynamic(null), service:dynamic=dynamic(null), buildType:dynamic=dynamic(null), trigger:dynamic=dynamic(null), result:dynamic=dynamic(null), sourceBranch:string="", definitionName:string="") -{ - ManagedDefinition - | where isnull(platform) or Platform in (platform) - | where isnull(service) or Feature in (service) - | where isnull(buildType) or Category in (buildType) - | join kind = leftouter ( - PipelineOwner - | summarize arg_min(Timestamp, *) by OrganizationName, BuildDefinitionId - ) on OrganizationName, BuildDefinitionId - | join kind = leftouter ( - Build - | where isnull(trigger) or Reason in (trigger) - | where isnull(result) or Result in (result) - | where SourceBranch matches regex coalesce(sourceBranch, ".") - | where DefinitionName matches regex coalesce(definitionName, ".") - | summarize arg_max(FinishTime, *) by DefinitionId, ProjectId - ) on ProjectId, $left.BuildDefinitionId == $right.DefinitionId - | join kind = leftouter ( - BuildFailure - | summarize arg_min(RecordFinishTime, *) by BuildDefinitionId, BuildId, ProjectId - ) on ProjectId, BuildDefinitionId, BuildId, $left.FinishTime == $right.BuildFinishTime - | project - ProjectId, - ProjectName, - BuildDefinitionName, - BuildDefinitionId, - Platform, - BuildType = Category, - Service = Feature, - Owners, - Trigger = Reason, - Result, - SourceBranch, - FinishTime, - ErrorClassification, - BuildDefinitionIdUrl = strcat("https://dev.azure.com/azure-sdk/", ProjectName, "/_build?definitionId=", tostring(BuildDefinitionId)) - | order by case(Result == 'failed', 1, Result == 'partiallySucceeded', 2, Result == 'canceled', 3, 4) asc - | mv-expand Platform to typeof(string), Service to typeof(string), BuildType to typeof(string) -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinition.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinition.kql deleted file mode 100644 index 49602cfa1e4..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinition.kql +++ /dev/null @@ -1,10 +0,0 @@ -.create-or-alter function ManagedDefinition -{ - LatestDefinition - | where Variables contains "meta" - | extend metas = parse_json(Variables) - | extend Platform = metas["meta.platform"].value - | extend Service = metas["meta.component"].value - | extend BuildType = metas["meta.category"].value - | project-away metas -} \ No newline at end of file diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionBuild.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionBuild.kql deleted file mode 100644 index b5c97966e82..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionBuild.kql +++ /dev/null @@ -1,8 +0,0 @@ -.create-or-alter function ManagedDefinitionBuild(sourceBranchFilter:string) -{ - ManagedDefinitionFunc() - | join kind=inner LatestBuild(sourceBranchFilter) on $left.BuildDefinitionId == $right.DefinitionId - | join kind=inner LatestOwners() on BuildDefinitionId - | project-away OrganizationName1, OrganizationName2, BuildDefinitionId1, Tags1, ProjectId1, ProjectState1, QueueName1, RepositoryType1, RepositoryCheckoutSubmodules1, QueuePoolName1, ProjectName1, QueueId1, RepositoryId1 - | project-rename DefinitionUri = Uri, BuildUri = Uri1, DefinitionData = Data, BuildData = Data1, Trigger = Reason -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionBuildX.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionBuildX.kql deleted file mode 100644 index 3aff5c1bf68..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionBuildX.kql +++ /dev/null @@ -1,25 +0,0 @@ -.create-or-alter function ManagedDefinitionBuildX() -{ - ManagedDefinition - | project-rename - DefinitionUri = Uri, - DefinitionData = Data - | join kind=inner ( - LatestOwners - | project BuildDefinitionId, Owners - ) on BuildDefinitionId - | project-away BuildDefinitionId1 - | join kind=inner ( - Build - | project - BuildDefinitionId = DefinitionId, - BuildUri = Uri, - BuildData = Data, - BuildId, - Result, - SourceBranch, - Trigger = Reason, - FinishTime - ) on BuildDefinitionId - | project-away BuildDefinitionId1 -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionFn.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionFn.kql deleted file mode 100644 index 210af0e0708..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionFn.kql +++ /dev/null @@ -1,10 +0,0 @@ -.create-or-alter function ManagedDefinitionFn() -{ - LatestDefinition() - | where Variables contains "meta" - | extend metas = parse_json(Variables) - | extend Platform = metas["meta.platform"].value - | extend Feature = metas["meta.component"].value - | extend Category = metas["meta.category"].value - | project-away metas -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionFunc.kql b/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionFunc.kql deleted file mode 100644 index b894e058b42..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/ManagedDefinitionFunc.kql +++ /dev/null @@ -1,10 +0,0 @@ -.create-or-alter function ManagedDefinitionFunc() -{ - LatestDefinition() - | where Variables contains "meta" - | extend metas = parse_json(Variables) - | extend Platform = metas["meta.platform"].value - | extend Service = metas["meta.component"].value - | extend BuildType = metas["meta.category"].value - | project-away metas -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/SuccessRateCalc.kql b/tools/pipeline-witness/infrastructure/kusto/functions/SuccessRateCalc.kql deleted file mode 100644 index 37645b9d3f9..00000000000 --- a/tools/pipeline-witness/infrastructure/kusto/functions/SuccessRateCalc.kql +++ /dev/null @@ -1,21 +0,0 @@ -.create-or-alter function SuccessRateCalc(startTimeFrame:datetime, endTimeFrame:datetime, platformFilter:dynamic=dynamic(null), serviceFilter:dynamic=dynamic(null), buildTypeFilter:dynamic=dynamic(null)) -{ - let ['language']=dynamic(null); - Build - | extend BuildDefinitionId = tolong(DefinitionId) - | join kind = inner ManagedDefinition on ProjectId, BuildDefinitionId - | extend Platform = tostring(Platform), Service = tostring(Feature), Category = tostring(Category) - | where FinishTime between (startTimeFrame .. endTimeFrame) - | where SourceBranch == "refs/heads/main" - | where Platform in ('cpp', 'go', 'java', 'js', 'net', 'python', 'ios', 'android') - | where isempty(['language']) or Platform in (['language']) - | where Result in ("succeeded", "failed") - | where isnull(platformFilter) or Platform in (platformFilter) - | where isnull(serviceFilter) or Service in (serviceFilter) - | where isnull(buildTypeFilter) or Category in (buildTypeFilter) - | summarize Count = count(), Succeeded = countif(Result == "succeeded") by BuildDefinitionId, DefinitionName, Platform, Service, Category - | summarize - BuildCount = sum(Count), - Succeeded = sum(Succeeded), - SuccessRate = floor(sum(Succeeded) / todouble(sum(Count)), decimal(0.001)) * 100 by BuildDefinitionId, DefinitionName, Platform, Service, Category -} diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/WeightedAverageCalc.kql b/tools/pipeline-witness/infrastructure/kusto/functions/WeightedAverageCalc.kql index 1e61d6c3326..58c2269f2c0 100644 --- a/tools/pipeline-witness/infrastructure/kusto/functions/WeightedAverageCalc.kql +++ b/tools/pipeline-witness/infrastructure/kusto/functions/WeightedAverageCalc.kql @@ -1,4 +1,4 @@ -.create-or-alter function WeightedAverageCalc(startTimeFrame:datetime, endTimeFrame:datetime, platformFilter:dynamic=dynamic(null), serviceFilter:dynamic=dynamic(null), buildTypeFilter:dynamic=dynamic(null)) +.create-or-alter function with (folder='', docstring='') WeightedAverageCalc(startTimeFrame:datetime, endTimeFrame:datetime, platformFilter:dynamic=dynamic(null), serviceFilter:dynamic=dynamic(null), buildTypeFilter:dynamic=dynamic(null)) { let ['language']=dynamic(null); let buildCountScorePercent = 25; @@ -19,7 +19,7 @@ | extend SuccessRate = todouble(Succeeded)/Count); let maxCount = toscalar (perOwner | summarize max(Count)); perOwner - | extend + | extend SuccessRatePoints = SuccessRate * (100 - buildCountScorePercent), BuildCountPoints = todouble(Count)/maxCount * buildCountScorePercent } diff --git a/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql b/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql index a573450cabf..106d18ab2b9 100644 --- a/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql +++ b/tools/pipeline-witness/infrastructure/kusto/functions/users/kojamroz/SpecsPipelinesBuildsJobLogs.kql @@ -1,25 +1,17 @@ -.create-or-alter function with ( - folder='users/kojamroz', - docstring="Build logs for specific ADO job in the specs pipelines builds. See source for details." - ) - SpecsPipelinesBuildsJobLogs( - jobName:string, - start:datetime, - end:datetime, - buildId:string = "*") +.create-or-alter function with (folder='users/kojamroz', docstring='Build logs for specific ADO job in the specs pipelines builds. See source for details.') SpecsPipelinesBuildsJobLogs(jobName:string, start:datetime, end:datetime, buildId:string="*") { // ABOUT THIS FUNCTION // -// This function returns build logs for the -// Azure.azure-rest-api-specs-pipeline [1] +// This function returns build logs for the +// Azure.azure-rest-api-specs-pipeline [1] // and Azure.azure-rest-api-specs-pipeline-staging [2] builds, // for job named 'jobName', e.g. "BreakingChange" or "LintDiff". // // The logs are from time period between 'start' and 'end'. // // Optionally, you can pass 'buildId'. By default it is any build, denoted as "*". -// -// If you pass specific 'buildId', consider narrowing down 'start' and 'end' +// +// If you pass specific 'buildId', consider narrowing down 'start' and 'end' // to the time period the build generated logs, to keep the function call performant. // // Run this function against cluster Azsdkengsys, database Pipelines. @@ -33,7 +25,7 @@ // --------------------------------------------------------------- // let filteredJobFirstMessage = strcat("##[section]Starting: ", jobName); -let allPipelineLogs = +let allPipelineLogs = cluster('azsdkengsys.westus2.kusto.windows.net').database('Pipelines').BuildLogLine | where Timestamp between (start..end) | where BuildDefinitionId in ("1736", "3268") @@ -43,20 +35,20 @@ let logIds = allPipelineLogs | where Message == filteredJobFirstMessage | distinct BuildId, LogId - // Here we ensure only one LogId is taken from each BuildId. This is necessary as given LogId can appear twice: + // Here we ensure only one LogId is taken from each BuildId. This is necessary as given LogId can appear twice: // once with log line offset for given job, and once with offset for given job. // Alternatively, the rows could be deduplicated by appropriate join to the BuildTimelineRecord table. | summarize arg_min(LogId, *) by BuildId ; let filteredLogs = -logIds +logIds | join kind=inner allPipelineLogs on BuildId, LogId | project-away BuildId1, LogId1 - | extend - PacificTime = datetime_utc_to_local(Timestamp, 'US/Pacific'), + | extend + PacificTime = datetime_utc_to_local(Timestamp, 'US/Pacific'), buildUrl = strcat("https://dev.azure.com/azure-sdk/internal/_build/results?buildId=", BuildId, "&view=results") | project-reorder Timestamp, PacificTime, buildUrl, BuildDefinitionId, BuildId, LineNumber, Message - // Get rid of duplicate logs. This is a known issue with no known proper fix - see the comment in the + // Get rid of duplicate logs. This is a known issue with no known proper fix - see the comment in the // definition of "logIds" above. | distinct * | order by Timestamp desc diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/Build.kql b/tools/pipeline-witness/infrastructure/kusto/tables/Build.kql index fab4bc3ab7d..d542320d915 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/Build.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/Build.kql @@ -57,9 +57,9 @@ ValidationResults: string, Data: string, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table Build ingestion json mapping 'Build_mapping' ```[ +.create-or-alter table Build ingestion json mapping 'Build_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, { "column": "BuildId", "path": "$['buildId']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/BuildDefinition.kql b/tools/pipeline-witness/infrastructure/kusto/tables/BuildDefinition.kql index a427ac12976..eb310174fdc 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/BuildDefinition.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/BuildDefinition.kql @@ -57,9 +57,9 @@ Tags: string, Data: string, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table BuildDefinition ingestion json mapping 'BuildDefinition_mapping' ```[ +.create-or-alter table BuildDefinition ingestion json mapping 'BuildDefinition_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, { "column": "BuildDefinitionId", "path": "$['buildDefinitionId']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/BuildFailure.kql b/tools/pipeline-witness/infrastructure/kusto/tables/BuildFailure.kql index 0db4e38fe04..93bc4cb6715 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/BuildFailure.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/BuildFailure.kql @@ -12,9 +12,9 @@ RecordFinishTime: datetime, ErrorClassification: string, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table BuildFailure ingestion json mapping 'BuildFailure_mapping' ```[ +.create-or-alter table BuildFailure ingestion json mapping 'BuildFailure_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, { "column": "ProjectName", "path": "$['projectName']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/BuildLogLine.kql b/tools/pipeline-witness/infrastructure/kusto/tables/BuildLogLine.kql index 3280f3302c7..3811e36b124 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/BuildLogLine.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/BuildLogLine.kql @@ -11,9 +11,9 @@ Timestamp: datetime, Message: string, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table BuildLogLine ingestion json mapping 'BuildLogLine_mapping' ```[ +.create-or-alter table BuildLogLine ingestion json mapping 'BuildLogLine_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, { "column": "ProjectName", "path": "$['projectName']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/BuildTimelineRecord.kql b/tools/pipeline-witness/infrastructure/kusto/tables/BuildTimelineRecord.kql index 3d1af6512f0..c3c9d76380f 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/BuildTimelineRecord.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/BuildTimelineRecord.kql @@ -33,9 +33,9 @@ Type: string, Issues: string, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table BuildTimelineRecord ingestion json mapping 'BuildTimelineRecord_mapping' ```[ +.create-or-alter table BuildTimelineRecord ingestion json mapping 'BuildTimelineRecord_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, { "column": "ProjectName", "path": "$['projectName']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/Dashboards/PackageReleaseData.kql b/tools/pipeline-witness/infrastructure/kusto/tables/Dashboards/PackageReleaseData.kql new file mode 100644 index 00000000000..6d682135232 --- /dev/null +++ b/tools/pipeline-witness/infrastructure/kusto/tables/Dashboards/PackageReleaseData.kql @@ -0,0 +1,15 @@ +.create-merge table PackageReleaseData ( + Date: datetime, + Package: string, + Version: string, + Language: string, + Properties: string +) with (folder='Dashboards', docstring='') + +.create-or-alter table PackageReleaseData ingestion json mapping 'PackageReleaseData_mapping' ```[ + { "column": "Date", "path": "$['date']" }, + { "column": "Package", "path": "$['package']" }, + { "column": "Version", "path": "$['version']" }, + { "column": "Language", "path": "$['language']" }, + { "column": "Properties", "path": "$['properties']" } +]``` diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/PipelineOwner.kql b/tools/pipeline-witness/infrastructure/kusto/tables/PipelineOwner.kql index aabedc279ac..8106686d50b 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/PipelineOwner.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/PipelineOwner.kql @@ -4,9 +4,9 @@ Owners: dynamic, Timestamp: datetime, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table PipelineOwner ingestion json mapping 'PipelineOwner_mapping' ```[ +.create-or-alter table PipelineOwner ingestion json mapping 'PipelineOwner_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "BuildDefinitionId", "path": "$['buildDefinitionId']" }, { "column": "Owners", "path": "$['owners']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/TestRun.kql b/tools/pipeline-witness/infrastructure/kusto/tables/TestRun.kql index a89d2a6233e..5f35a9fcdaa 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/TestRun.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/TestRun.kql @@ -1,6 +1,11 @@ .create-merge table TestRun ( OrganizationName: string, ProjectId: string, + ProjectName: string, + BuildDefinitionId: long, + BuildDefinitionPath: string, + BuildDefinitionName: string, + BuildId: long, TestRunId: int, Title: string, StartedDate: datetime, @@ -8,8 +13,6 @@ ResultDurationSeconds: real, RunDurationSeconds: real, BranchName: string, - BuildDefinitionId: int, - BuildId: long, HasDetail: bool, IsAutomated: bool, ResultAbortedCount: int, @@ -30,11 +33,16 @@ OrganizationId: string, Data: string, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table TestRun ingestion json mapping 'TestRun_mapping' ```[ +.create-or-alter table TestRun ingestion json mapping 'TestRun_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, + { "column": "ProjectName", "path": "$['projectName']" }, + { "column": "BuildDefinitionId", "path": "$['buildDefinitionId']" }, + { "column": "BuildDefinitionPath", "path": "$['buildDefinitionPath']" }, + { "column": "BuildDefinitionName", "path": "$['buildDefinitionName']" }, + { "column": "BuildId", "path": "$['buildId']" }, { "column": "TestRunId", "path": "$['testRunId']" }, { "column": "Title", "path": "$['title']" }, { "column": "StartedDate", "path": "$['startedDate']" }, @@ -42,8 +50,6 @@ { "column": "ResultDurationSeconds", "path": "$['resultDurationSeconds']" }, { "column": "RunDurationSeconds", "path": "$['runDurationSeconds']" }, { "column": "BranchName", "path": "$['branchName']" }, - { "column": "BuildDefinitionId", "path": "$['buildDefinitionId']" }, - { "column": "BuildId", "path": "$['buildId']" }, { "column": "HasDetail", "path": "$['hasDetail']" }, { "column": "IsAutomated", "path": "$['isAutomated']" }, { "column": "ResultAbortedCount", "path": "$['resultAbortedCount']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/tables/TestRunResult.kql b/tools/pipeline-witness/infrastructure/kusto/tables/TestRunResult.kql index f2d3bbe4b97..60839cc061f 100644 --- a/tools/pipeline-witness/infrastructure/kusto/tables/TestRunResult.kql +++ b/tools/pipeline-witness/infrastructure/kusto/tables/TestRunResult.kql @@ -2,12 +2,12 @@ OrganizationName: string, ProjectId: string, ProjectName: string, - BuildDefinitionId: int, + BuildDefinitionId: long, BuildDefinitionPath: string, BuildDefinitionName: string, - BuildId: int, + BuildId: long, TestRunId: int, - TestCaseId: int, + TestCaseId: int, TestCaseReferenceId: int, TestCaseTitle: string, Outcome: string, @@ -19,9 +19,9 @@ StartedDate: datetime, CompletedDate: datetime, EtlIngestDate: datetime -) +) with (folder='', docstring='') -.alter table TestRunResult ingestion json mapping 'TestRunResult_mapping' ```[ +.create-or-alter table TestRunResult ingestion json mapping 'TestRunResult_mapping' ```[ { "column": "OrganizationName", "path": "$['organizationName']" }, { "column": "ProjectId", "path": "$['projectId']" }, { "column": "ProjectName", "path": "$['projectName']" }, diff --git a/tools/pipeline-witness/infrastructure/kusto/views/ManagedDefinition.kql b/tools/pipeline-witness/infrastructure/kusto/views/ManagedDefinition.kql index 8a90c60383a..8d82bcc04be 100644 --- a/tools/pipeline-witness/infrastructure/kusto/views/ManagedDefinition.kql +++ b/tools/pipeline-witness/infrastructure/kusto/views/ManagedDefinition.kql @@ -1,4 +1,4 @@ -.create-or-alter materialized-view ManagedDefinition on table BuildDefinition +.create-or-alter materialized-view with (folder='', docstring='') ManagedDefinition on table BuildDefinition { BuildDefinition | where Variables contains "meta" diff --git a/tools/pipeline-witness/infrastructure/kusto/views/Metrics.kql b/tools/pipeline-witness/infrastructure/kusto/views/Metrics.kql index 46307e73bde..355d90b726c 100644 --- a/tools/pipeline-witness/infrastructure/kusto/views/Metrics.kql +++ b/tools/pipeline-witness/infrastructure/kusto/views/Metrics.kql @@ -1,4 +1,4 @@ -.create-or-alter materialized-view with (docstring = 'Log extracted metrics') Metrics on table BuildLogLine +.create-or-alter materialized-view with (folder='', docstring='Log extracted metrics') Metrics on table BuildLogLine { BuildLogLine | where Message has_all ("logmetric", "name", "value", "timestamp")