Skip to content

Commit

Permalink
Remove email fields from kusto schema (#7785)
Browse files Browse the repository at this point in the history 
* Remove email fields from kusto schema
* Remove unnecessary columns and add useable URL
  • Loading branch information
@hallipr
hallipr authored Feb 29, 2024
1 parent 2d364d5 commit c236a88
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 97 deletions.
36 changes: 3 additions & 33 deletions tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/BlobUploadProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,24 +398,8 @@ private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinitio
Path = definition.Path,
RepositoryId = definition.Repository.Id,
RepositoryName = definition.Repository.Name,
AuthoredByDescriptor = definition.AuthoredBy.Descriptor.ToString(),
AuthoredByDisplayName = definition.AuthoredBy.DisplayName,
AuthoredById = definition.AuthoredBy.Id,
AuthoredByIsContainer = definition.AuthoredBy.IsContainer,
AuthoredByUniqueName = definition.AuthoredBy.UniqueName,
CreatedDate = definition.CreatedDate,
DefaultBranch = definition.Repository.DefaultBranch,
DraftOfId = default(string),
DraftOfName = default(string),
DraftOfProjectId = default(string),
DraftOfProjectName = default(string),
DraftOfProjectRevision = default(string),
DraftOfProjectState = default(string),
DraftOfProjectVisibility = default(string),
DraftOfQueueStatus = default(string),
DraftOfRevision = default(string),
DraftOfType = default(string),
DraftOfUri = default(string),
ProjectName = definition.Project.Name,
ProjectRevision = definition.Project.Revision,
ProjectState = definition.Project.State,
Expand All @@ -427,7 +411,7 @@ private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinitio
QueuePoolName = definition.Queue?.Pool?.Name,
QueueStatus = definition.QueueStatus,
Type = definition.Type,
Uri = definition.Uri,
Url = $"https://dev.azure.com/{account}/{definition.Project.Name}/_build?definitionId={definition.Id}",
BadgeEnabled = definition.BadgeEnabled,
BuildNumberFormat = definition.BuildNumberFormat,
Comment = definition.Comment,
Expand All @@ -445,7 +429,7 @@ private async Task UploadBuildDefinitionBlobAsync(string account, BuildDefinitio
Options = definition.Options,
Variables = definition.Variables,
Tags = definition.Tags,
Data = definition,
Triggers = definition.Triggers,
EtlIngestDate = DateTimeOffset.UtcNow
}, jsonSettings);

Expand Down Expand Up @@ -548,10 +532,6 @@ private async Task UploadBuildBlobAsync(string account, Build build)
StartTime = build.StartTime,
FinishTime = build.FinishTime,
KeepForever = build.KeepForever,
LastChangedByDisplayName = build.LastChangedBy?.DisplayName,
LastChangedById = build.LastChangedBy?.Id,
LastChangedByIsContainer = build.LastChangedBy?.IsContainer,
LastChangedByUniqueName = build.LastChangedBy?.UniqueName,
LastChangedDate = build.LastChangedDate,
LogsId = build.Logs?.Id,
LogsType = build.Logs?.Type,
Expand All @@ -569,23 +549,14 @@ private async Task UploadBuildBlobAsync(string account, Build build)
Reason = build.Reason,
RepositoryCheckoutSubmodules = build.Repository?.CheckoutSubmodules,
RepositoryType = build.Repository?.Type,
RequestedByDisplayName = build.RequestedBy?.DisplayName,
RequestedById = build.RequestedBy?.Id,
RequestedByIsContainer = build.RequestedBy?.IsContainer,
RequestedByUniqueName = build.RequestedBy?.UniqueName,
RequestedForDisplayName = build.RequestedFor?.DisplayName,
RequestedForId = build.RequestedFor?.Id,
RequestedForIsContainer = build.RequestedFor?.IsContainer,
RequestedForUniqueName = build.RequestedFor?.UniqueName,
Result = build.Result,
RetainedByRelease = build.RetainedByRelease,
SourceBranch = build.SourceBranch,
SourceVersion = build.SourceVersion,
Status = build.Status,
Tags = build.Tags?.Any() == true ? JsonConvert.SerializeObject(build.Tags, jsonSettings) : null,
Uri = build.Uri,
Url = $"https://dev.azure.com/{account}/{build.Project.Name}/_build/results?buildId={build.Id}",
ValidationResults = build.ValidationResults,
Data = JsonConvert.SerializeObject(build, jsonSettings),
EtlIngestDate = DateTime.UtcNow,
}, jsonSettings);

Expand Down Expand Up @@ -869,7 +840,6 @@ private async Task UploadTestRunBlobAsync(string account, Build build, TestRun t
: testRun.Release?.Id > 0 ? "Release"
: "",
OrganizationId = default(string),
Data = default(string),
EtlIngestDate = DateTime.UtcNow,
}, jsonSettings);

Expand Down
30 changes: 2 additions & 28 deletions tools/pipeline-witness/infrastructure/kusto/tables/Build.kql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,6 @@
StartTime: datetime,
FinishTime: datetime,
KeepForever: bool,
LastChangedByDisplayName: string,
LastChangedById: string,
LastChangedByIsContainer: bool,
LastChangedByUniqueName: string,
LastChangedDate: datetime,
LogsId: long,
LogsType: string,
Expand All @@ -39,23 +35,14 @@
Reason: string,
RepositoryCheckoutSubmodules: bool,
RepositoryType: string,
RequestedByDisplayName: string,
RequestedById: string,
RequestedByIsContainer: bool,
RequestedByUniqueName: string,
RequestedForDisplayName: string,
RequestedForId: string,
RequestedForIsContainer: bool,
RequestedForUniqueName: string,
Result: string,
RetainedByRelease: string,
SourceBranch: string,
SourceVersion: string,
Status: string,
Tags: string,
Uri: string,
Url: string,
ValidationResults: string,
Data: string,
EtlIngestDate: datetime
) with (folder='', docstring='')

Expand All @@ -79,10 +66,6 @@
{ "column": "StartTime", "path": "$['startTime']" },
{ "column": "FinishTime", "path": "$['finishTime']" },
{ "column": "KeepForever", "path": "$['keepForever']" },
{ "column": "LastChangedByDisplayName", "path": "$['lastChangedByDisplayName']" },
{ "column": "LastChangedById", "path": "$['lastChangedById']" },
{ "column": "LastChangedByIsContainer", "path": "$['lastChangedByIsContainer']" },
{ "column": "LastChangedByUniqueName", "path": "$['lastChangedByUniqueName']" },
{ "column": "LastChangedDate", "path": "$['lastChangedDate']" },
{ "column": "LogsId", "path": "$['logsId']" },
{ "column": "LogsType", "path": "$['logsType']" },
Expand All @@ -100,22 +83,13 @@
{ "column": "Reason", "path": "$['reason']" },
{ "column": "RepositoryCheckoutSubmodules", "path": "$['repositoryCheckoutSubmodules']" },
{ "column": "RepositoryType", "path": "$['repositoryType']" },
{ "column": "RequestedByDisplayName", "path": "$['requestedByDisplayName']" },
{ "column": "RequestedById", "path": "$['requestedById']" },
{ "column": "RequestedByIsContainer", "path": "$['requestedByIsContainer']" },
{ "column": "RequestedByUniqueName", "path": "$['requestedByUniqueName']" },
{ "column": "RequestedForDisplayName", "path": "$['requestedForDisplayName']" },
{ "column": "RequestedForId", "path": "$['requestedForId']" },
{ "column": "RequestedForIsContainer", "path": "$['requestedForIsContainer']" },
{ "column": "RequestedForUniqueName", "path": "$['requestedForUniqueName']" },
{ "column": "Result", "path": "$['result']" },
{ "column": "RetainedByRelease", "path": "$['retainedByRelease']" },
{ "column": "SourceBranch", "path": "$['sourceBranch']" },
{ "column": "SourceVersion", "path": "$['sourceVersion']" },
{ "column": "Status", "path": "$['status']" },
{ "column": "Tags", "path": "$['tags']" },
{ "column": "Uri", "path": "$['uri']" },
{ "column": "Url", "path": "$['url']" },
{ "column": "ValidationResults", "path": "$['validationResults']" },
{ "column": "Data", "path": "$['data']" },
{ "column": "EtlIngestDate", "path": "$['etlIngestDate']" }
]```
40 changes: 4 additions & 36 deletions tools/pipeline-witness/infrastructure/kusto/tables/BuildDefinition.kql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,8 @@
Path: string,
RepositoryId: string,
RepositoryName: string,
AuthoredByDescriptor: string,
AuthoredByDisplayName: string,
AuthoredById: string,
AuthoredByIsContainer: string,
AuthoredByUniqueName: string,
CreatedDate: datetime,
DefaultBranch: string,
DraftOfId: long,
DraftOfName: string,
DraftOfProjectId: string,
DraftOfProjectName: string,
DraftOfProjectRevision: long,
DraftOfProjectState: string,
DraftOfProjectVisibility: string,
DraftOfQueueStatus: string,
DraftOfRevision: long,
DraftOfType: string,
DraftOfUri: string,
ProjectName: string,
ProjectRevision: long,
ProjectState: string,
Expand All @@ -36,7 +20,7 @@
QueuePoolName: string,
QueueStatus: string,
Type: string,
Uri: string,
Url: string,
BadgeEnabled: bool,
BuildNumberFormat: string,
Comment: string,
Expand All @@ -55,7 +39,7 @@
Options: string,
Variables: string,
Tags: string,
Data: string,
Triggers: dynamic,
EtlIngestDate: datetime
) with (folder='', docstring='')

Expand All @@ -68,24 +52,8 @@
{ "column": "Path", "path": "$['path']" },
{ "column": "RepositoryId", "path": "$['repositoryId']" },
{ "column": "RepositoryName", "path": "$['repositoryName']" },
{ "column": "AuthoredByDescriptor", "path": "$['authoredByDescriptor']" },
{ "column": "AuthoredByDisplayName", "path": "$['authoredByDisplayName']" },
{ "column": "AuthoredById", "path": "$['authoredById']" },
{ "column": "AuthoredByIsContainer", "path": "$['authoredByIsContainer']" },
{ "column": "AuthoredByUniqueName", "path": "$['authoredByUniqueName']" },
{ "column": "CreatedDate", "path": "$['createdDate']" },
{ "column": "DefaultBranch", "path": "$['defaultBranch']" },
{ "column": "DraftOfId", "path": "$['draftOfId']" },
{ "column": "DraftOfName", "path": "$['draftOfName']" },
{ "column": "DraftOfProjectId", "path": "$['draftOfProjectId']" },
{ "column": "DraftOfProjectName", "path": "$['draftOfProjectName']" },
{ "column": "DraftOfProjectRevision", "path": "$['draftOfProjectRevision']" },
{ "column": "DraftOfProjectState", "path": "$['draftOfProjectState']" },
{ "column": "DraftOfProjectVisibility", "path": "$['draftOfProjectVisibility']" },
{ "column": "DraftOfQueueStatus", "path": "$['draftOfQueueStatus']" },
{ "column": "DraftOfRevision", "path": "$['draftOfRevision']" },
{ "column": "DraftOfType", "path": "$['draftOfType']" },
{ "column": "DraftOfUri", "path": "$['draftOfUri']" },
{ "column": "ProjectName", "path": "$['projectName']" },
{ "column": "ProjectRevision", "path": "$['projectRevision']" },
{ "column": "ProjectState", "path": "$['projectState']" },
Expand All @@ -97,7 +65,7 @@
{ "column": "QueuePoolName", "path": "$['queuePoolName']" },
{ "column": "QueueStatus", "path": "$['queueStatus']" },
{ "column": "Type", "path": "$['type']" },
{ "column": "Uri", "path": "$['uri']" },
{ "column": "Url", "path": "$['url']" },
{ "column": "BadgeEnabled", "path": "$['badgeEnabled']" },
{ "column": "BuildNumberFormat", "path": "$['buildNumberFormat']" },
{ "column": "Comment", "path": "$['comment']" },
Expand All @@ -116,6 +84,6 @@
{ "column": "Options", "path": "$['options']" },
{ "column": "Variables", "path": "$['variables']" },
{ "column": "Tags", "path": "$['tags']" },
{ "column": "Data", "path": "$['data']" },
{ "column": "Triggers", "path": "$['triggers']" },
{ "column": "EtlIngestDate", "path": "$['etlIngestDate']" }
]```

0 comments on commit c236a88

Please sign in to comment.