forked from actions/runner
-
-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Provide errors for missing and unexpected parameters in azure pipelin…
…es (#238) * azp: detect unrecognized parameters and missing required parameters * add test framework for azpipelines * Revert Test.csproj (Due to build failure) * removed try/catch, fix for null reference types * fix tests on linux and run them in ci --------- Co-authored-by: ChristopherHX <[email protected]>
- Loading branch information
1 parent
c7ee3e7
commit efdebf7
Showing
26 changed files
with
708 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
src/Sdk.Tests/AzurePipelines/AzurePipelinesYamlValidationTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace Runner.Server.Azure.Devops | ||
| { | ||
| public class AzurePipelinesYamlValidationTests | ||
| { | ||
| private TestContext Context; | ||
|
|
||
| public AzurePipelinesYamlValidationTests(ITestOutputHelper output) | ||
| { | ||
| Context = TestContext.Create(Directory.GetCurrentDirectory()).AddOutputToTest(output); | ||
| } | ||
|
|
||
| [Theory] | ||
| [ClassData(typeof(AzPipelineTestWorkflows))] | ||
| public void ValidateYamlFormat(TestWorkflow workflow) | ||
| { | ||
| // arrange | ||
| Context.SetWorkingDirectory(TestUtil.GetAzPipelineFolder(workflow.WorkingDirectory)); | ||
| foreach(var localRepo in workflow.LocalRepository) | ||
| { | ||
| var repositoryAndRef = localRepo.Split("="); | ||
| Context.AddRepo(repositoryAndRef[0], TestUtil.GetAzPipelineFolder(repositoryAndRef[1])); | ||
| } | ||
|
|
||
| // act | ||
| var act = new Action(() => Context.Evaluate(workflow.File).ToYaml()); | ||
|
|
||
| // assert | ||
| if (workflow.ExpectedException != null) | ||
| { | ||
| var message = Should.Throw(act, workflow.ExpectedException).Message; | ||
|
|
||
| if (workflow.ExpectedErrorMessage != null) | ||
| { | ||
| message.ShouldContain(workflow.ExpectedErrorMessage); | ||
| } | ||
|
|
||
| } | ||
| else | ||
| { | ||
| act.Invoke(); | ||
| } | ||
| } | ||
|
|
||
| [Fact] | ||
| public void TestWorkflowResolve() | ||
| { | ||
| var results = TestGenerator.ResolveWorkflows(TestUtil.GetAzPipelineFolder()).ToArray(); | ||
| } | ||
| } | ||
| } |
20 changes: 20 additions & 0 deletions
20
src/Sdk.Tests/AzurePipelines/TestData/AzPipelineTestWorkflows.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| using System.Collections; | ||
|
|
||
| namespace Runner.Server.Azure.Devops | ||
| { | ||
| public class AzPipelineTestWorkflows : IEnumerable<object[]> | ||
| { | ||
| public IEnumerator<object[]> GetEnumerator() | ||
| { | ||
| foreach(var result in TestGenerator.ResolveWorkflows(TestUtil.GetAzPipelineFolder())) | ||
| { | ||
| yield return new object[] { result }; | ||
| } | ||
| } | ||
|
|
||
| IEnumerator IEnumerable.GetEnumerator() | ||
| { | ||
| return this.GetEnumerator(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| using GitHub.DistributedTask.ObjectTemplating; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Text.RegularExpressions; | ||
|
|
||
| namespace Runner.Server.Azure.Devops | ||
| { | ||
| public class TestGenerator | ||
| { | ||
| public static IEnumerable<TestWorkflow> ResolveWorkflows(string basePath, string folder = "") | ||
| { | ||
| var currentFolder = Path.Combine(basePath, folder); | ||
| var files = Directory.GetFiles(currentFolder, "*.yml"); | ||
|
|
||
| bool found = false; | ||
|
|
||
| if (files.Any()) | ||
| { | ||
| bool asPipeline = false; | ||
|
|
||
| // prioritize filtering pipeline*.yml files | ||
| var pipelines = files.Where(i => i.Contains($"{Path.DirectorySeparatorChar}pipeline")); | ||
| if (pipelines.Any()) | ||
| { | ||
| asPipeline = true; | ||
| files = pipelines.ToArray(); | ||
| } | ||
|
|
||
| foreach(var file in files) | ||
| { | ||
| if (TryReadPipeline(basePath, file, asPipeline, out var result)) | ||
| { | ||
| found = true; // this folder contains pipelines, stop recursion | ||
| yield return result; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (!found) | ||
| { | ||
| // loop through subfolder | ||
| foreach(var directory in Directory.GetDirectories(currentFolder)) | ||
| { | ||
| var folderName = Path.GetRelativePath(basePath, directory); | ||
| foreach(var item in ResolveWorkflows(basePath, folderName)) | ||
| { | ||
| yield return item; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static bool TryReadPipeline(string basePath, string file, bool isPipeline, [MaybeNullWhen(returnValue:false)] out TestWorkflow result) | ||
| { | ||
| result = null; | ||
|
|
||
| // read contents of files | ||
| using(var reader = new StreamReader(file)) | ||
| { | ||
| var content = reader.ReadToEnd(); | ||
|
|
||
| var parser = new TestWorkflowParser(content); | ||
|
|
||
| if (isPipeline || parser.HasMeta()) | ||
| { | ||
| var relativePath = Path.GetRelativePath(basePath, file); | ||
| var workingDir = Path.GetDirectoryName(relativePath) ?? basePath; | ||
| var fileName = Path.GetFileName(relativePath); | ||
|
|
||
| result = new TestWorkflow(workingDir, fileName) | ||
| { | ||
| Name = parser.Name, | ||
| LocalRepository = parser.LocalRepository, | ||
| ExpectedException = parser.ExpectedException, | ||
| ExpectedErrorMessage = parser.ExpectedError | ||
| }; | ||
|
|
||
| return true; | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Obtains meta-data stored in the yaml file comments | ||
| /// </summary> | ||
| class TestWorkflowParser | ||
| { | ||
| public string? Name { get; private set; } | ||
| public Type? ExpectedException { get; private set; } | ||
| public string? ExpectedError { get; private set; } | ||
| public string[] LocalRepository { get; private set; } | ||
|
|
||
| public bool HasMeta() | ||
| { | ||
| return Name != null || ExpectedException != null | ExpectedError != null | LocalRepository?.Length > 0; | ||
| } | ||
|
|
||
| public TestWorkflowParser(string content) | ||
| { | ||
| Name = GetMeta(content, "Name")[0]; | ||
| ExpectedError = GetMeta(content, "ExpectedErrorMessage")[0]; | ||
| ExpectedException = LoadType(GetMeta(content, "ExpectedException")[0]); | ||
| LocalRepository = GetMeta(content, "LocalRepository").Where(i => i != null).Cast<string>().ToArray(); | ||
| } | ||
|
|
||
| private static string?[] GetMeta(string content, string name) | ||
| { | ||
| var matches = Regex.Matches(content, $"^# {name}: (.*?)\r?$", RegexOptions.Multiline | RegexOptions.IgnoreCase); | ||
| return matches.Where(i => i.Success).Select(i => i.Groups[1].Value).DefaultIfEmpty().ToArray(); | ||
| } | ||
|
|
||
| public static Type? LoadType(string? typeName) | ||
| { | ||
| if (typeName == null) | ||
| { | ||
| return null; | ||
| } | ||
|
|
||
| var referenceTypes = new Type[] | ||
| { | ||
| typeof(TemplateValidationError), | ||
| typeof(Exception) | ||
| }; | ||
|
|
||
| // locate and resolve type | ||
| return referenceTypes.Select(i => Type.GetType($"{i.Namespace}.{typeName}, {i.Assembly.GetName().Name}", false)).FirstOrDefault(i => i != null); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace Runner.Server.Azure.Devops | ||
| { | ||
| public class TestWorkflow : IXunitSerializable | ||
| { | ||
| /// <summary> | ||
| /// Serialization Constructor | ||
| /// </summary> | ||
| #pragma warning disable CS8618 | ||
| public TestWorkflow() { } | ||
| #pragma warning restore CS8618 | ||
|
|
||
| public TestWorkflow(string workingDirectory, string file) | ||
| { | ||
| WorkingDirectory = workingDirectory; | ||
| File = file.Replace(@"\", "/"); | ||
| LocalRepository = Array.Empty<string>(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Base Working Directory for Test | ||
| /// </summary> | ||
| public string WorkingDirectory { get; private set; } | ||
|
|
||
| /// <summary> | ||
| /// Path to Pipeline | ||
| /// </summary> | ||
| public string File { get; private set; } | ||
|
|
||
| #region meta-data | ||
| /// <summary> | ||
| /// Display Name for Test | ||
| /// </summary> | ||
| public string? Name { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Additional repository information needed for workflow | ||
| /// </summary> | ||
| public string[] LocalRepository { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Expected Exception for YAML Parsing scenario | ||
| /// </summary> | ||
| public Type? ExpectedException { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Expected Exception Message for YAML Parsing scenario | ||
| /// </summary> | ||
| public string? ExpectedErrorMessage { get; set; } | ||
| #endregion | ||
|
|
||
| #region IXUnitSerializable | ||
|
|
||
| /// <summary> | ||
| /// Hydrate Test data from xUnit Test discovery | ||
| /// </summary> | ||
| public void Deserialize(IXunitSerializationInfo info) | ||
| { | ||
| Name = info.GetValue<string>(nameof(Name)); | ||
| WorkingDirectory = info.GetValue<string>(nameof(WorkingDirectory)); | ||
| File = info.GetValue<string>(nameof(File)); | ||
| LocalRepository = info.GetValue<string?>(nameof(LocalRepository))?.Split(";") ?? Array.Empty<string>(); | ||
| ExpectedErrorMessage = info.GetValue<string?>(nameof(ExpectedErrorMessage)); | ||
| string? exceptionType = info.GetValue<string?>(nameof(ExpectedException)); | ||
| if (exceptionType != null) | ||
| { | ||
| ExpectedException = Type.GetType(exceptionType, false); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Persist test data from xUnit Test discovery | ||
| /// </summary> | ||
| public void Serialize(IXunitSerializationInfo info) | ||
| { | ||
| info.AddValue(nameof(Name), Name); | ||
| info.AddValue(nameof(WorkingDirectory), WorkingDirectory); | ||
| info.AddValue(nameof(File), File); | ||
| info.AddValue(nameof(LocalRepository), LocalRepository?.Length > 0 ? string.Join(";", LocalRepository) : null); | ||
| info.AddValue(nameof(ExpectedException), $"{ExpectedException?.FullName},{ExpectedException?.Assembly.GetName().Name}"); | ||
| info.AddValue(nameof(ExpectedErrorMessage), ExpectedErrorMessage); | ||
| } | ||
| #endregion | ||
|
|
||
| /// <summary> | ||
| /// Provide a display value for the testdata | ||
| /// </summary> | ||
| /// <returns></returns> | ||
| public override string ToString() | ||
| { | ||
| if (Name != null) | ||
| { | ||
| return Name; | ||
| } | ||
|
|
||
| // working_directory_pipeline_name | ||
| return string.Format( | ||
| "{0}{1}", | ||
| WorkingDirectory, | ||
| File == "pipeline.yml" ? "" : $"_{File.Replace(".yml","")}" | ||
| ); | ||
| } | ||
|
|
||
| } | ||
| } |
Oops, something went wrong.