diff --git a/.github/workflows/continuous-build.yaml b/.github/workflows/continuous-build.yaml index bb2679b2..f55dcb12 100644 --- a/.github/workflows/continuous-build.yaml +++ b/.github/workflows/continuous-build.yaml @@ -35,4 +35,5 @@ jobs: - name: File-base integration tests (in-proc) run: | dotnet test code/DeltaKustoFileIntegrationTest --configuration Release \ - --no-build --verbosity normal --blame-hang-timeout 3s + --no-build --verbosity normal --blame-hang-timeout 3s \ + -p:ParallelizeTestCollections=false diff --git a/code/DeltaKustoAdxIntegrationTest/AdxIntegrationTestBase.cs b/code/DeltaKustoAdxIntegrationTest/AdxIntegrationTestBase.cs index 5f5f90a3..6d87534c 100644 --- a/code/DeltaKustoAdxIntegrationTest/AdxIntegrationTestBase.cs +++ b/code/DeltaKustoAdxIntegrationTest/AdxIntegrationTestBase.cs @@ -19,8 +19,6 @@ namespace DeltaKustoAdxIntegrationTest { public abstract class AdxIntegrationTestBase : IntegrationTestBase { - private static readonly TimeSpan TIME_OUT = TimeSpan.FromSeconds(90); - private readonly Uri _clusterUri; private readonly string _currentDb; private readonly string _targetDb; @@ -84,15 +82,11 @@ protected AdxIntegrationTestBase() protected async Task TestAdxToFile(string statesFolderPath, string outputFolder) { - var cancellationToken = new CancellationTokenSource(TIME_OUT); - var ct = cancellationToken.Token; - await LoopThroughStateFilesAsync( statesFolderPath, - ct, async (fromFile, toFile) => { - await PrepareDbAsync(fromFile, true, ct); + await PrepareDbAsync(fromFile, true); var outputPath = outputFolder + Path.GetFileNameWithoutExtension(fromFile) @@ -104,15 +98,14 @@ await LoopThroughStateFilesAsync( .Append(("jobs.main.action.filePath", outputPath)); var parameters = await RunParametersAsync( "adx-to-file-params.json", - ct, overrides); - var outputCommands = await LoadScriptAsync(outputPath); + var outputCommands = await LoadScriptAsync("", outputPath); var targetCommands = CommandBase.FromScript( await File.ReadAllTextAsync(toFile)); - await ApplyCommandsAsync(outputCommands, true, ct); + await ApplyCommandsAsync(outputCommands, true); - var finalCommands = await FetchDbCommandsAsync(true, ct); + var finalCommands = await FetchDbCommandsAsync(true); var targetModel = DatabaseModel.FromCommands(targetCommands); var finalModel = DatabaseModel.FromCommands(finalCommands); var finalScript = string.Join(";\n\n", finalCommands.Select(c => c.ToScript())); @@ -126,15 +119,11 @@ await LoopThroughStateFilesAsync( protected async Task TestFileToAdx(string statesFolderPath, string outputFolder) { - var cancellationToken = new CancellationTokenSource(TIME_OUT); - var ct = cancellationToken.Token; - await LoopThroughStateFilesAsync( statesFolderPath, - ct, async (fromFile, toFile) => { - await PrepareDbAsync(toFile, false, ct); + await PrepareDbAsync(toFile, false); var outputPath = outputFolder + Path.GetFileNameWithoutExtension(fromFile) @@ -146,9 +135,8 @@ await LoopThroughStateFilesAsync( .Append(("jobs.main.action.filePath", outputPath)); var parameters = await RunParametersAsync( "file-to-adx-params.json", - ct, overrides); - var outputCommands = await LoadScriptAsync(outputPath); + var outputCommands = await LoadScriptAsync("", outputPath); var currentCommands = CommandBase.FromScript( await File.ReadAllTextAsync(fromFile)); var targetCommands = CommandBase.FromScript( @@ -156,10 +144,9 @@ await LoopThroughStateFilesAsync( await ApplyCommandsAsync( currentCommands.Concat(outputCommands), - true, - ct); + true); - var finalCommands = await FetchDbCommandsAsync(true, ct); + var finalCommands = await FetchDbCommandsAsync(true); var targetModel = DatabaseModel.FromCommands(targetCommands); var finalModel = DatabaseModel.FromCommands(finalCommands); var finalScript = string.Join(";\n\n", finalCommands.Select(c => c.ToScript())); @@ -173,17 +160,13 @@ await ApplyCommandsAsync( protected async Task TestAdxToAdx(string statesFolderPath, string outputFolder) { - var cancellationToken = new CancellationTokenSource(TIME_OUT); - var ct = cancellationToken.Token; - await LoopThroughStateFilesAsync( statesFolderPath, - ct, async (fromFile, toFile) => { await Task.WhenAll( - PrepareDbAsync(fromFile, true, ct), - PrepareDbAsync(toFile, false, ct)); + PrepareDbAsync(fromFile, true), + PrepareDbAsync(toFile, false)); var outputPath = outputFolder + Path.GetFileNameWithoutExtension(fromFile) @@ -195,11 +178,10 @@ await Task.WhenAll( .Append(("jobs.main.action.filePath", outputPath)); var parameters = await RunParametersAsync( "adx-to-adx-params.json", - ct, overrides); var targetCommands = CommandBase.FromScript( await File.ReadAllTextAsync(toFile)); - var finalCommands = await FetchDbCommandsAsync(false, ct); + var finalCommands = await FetchDbCommandsAsync(false); var targetModel = DatabaseModel.FromCommands(targetCommands); var finalModel = DatabaseModel.FromCommands(finalCommands); var finalScript = string.Join(";\n\n", finalCommands.Select(c => c.ToScript())); @@ -213,7 +195,6 @@ await Task.WhenAll( private async Task LoopThroughStateFilesAsync( string folderPath, - CancellationToken ct, Func loopFunction) { var stateFiles = Directory.GetFiles(folderPath); @@ -225,7 +206,7 @@ private async Task LoopThroughStateFilesAsync( foreach (var toFile in stateFiles) { Console.WriteLine($"Current loop: ({fromFile}, {toFile})"); - await CleanDatabasesAsync(ct); + await CleanDatabasesAsync(); await loopFunction(fromFile, toFile); } } @@ -233,26 +214,23 @@ private async Task LoopThroughStateFilesAsync( private async Task ApplyCommandsAsync( IEnumerable commands, - bool isCurrent, - CancellationToken ct) + bool isCurrent) { var gateway = CreateKustoManagementGateway(isCurrent); // Apply commands to the db - await gateway.ExecuteCommandsAsync(commands, ct); + await gateway.ExecuteCommandsAsync(commands); } - private async Task> FetchDbCommandsAsync( - bool isCurrent, - CancellationToken ct) + private async Task> FetchDbCommandsAsync(bool isCurrent) { var gateway = CreateKustoManagementGateway(isCurrent); var dbProvider = (IDatabaseProvider)new KustoDatabaseProvider( new ConsoleTracer(false), gateway); var emptyProvider = (IDatabaseProvider)new EmptyDatabaseProvider(); - var finalDb = await dbProvider.RetrieveDatabaseAsync(ct); - var emptyDb = await emptyProvider.RetrieveDatabaseAsync(ct); + var finalDb = await dbProvider.RetrieveDatabaseAsync(); + var emptyDb = await emptyProvider.RetrieveDatabaseAsync(); // Use the delta from an empty db to get var finalCommands = emptyDb.ComputeDelta(finalDb); @@ -261,7 +239,6 @@ private async Task> FetchDbCommandsAsync( protected override Task RunParametersAsync( string parameterFilePath, - CancellationToken ct, IEnumerable<(string path, string value)>? overrides = null) { var adjustedOverrides = overrides != null @@ -273,20 +250,19 @@ protected override Task RunParametersAsync( .Append(("tokenProvider.login.clientId", _servicePrincipalId)) .Append(("tokenProvider.login.secret", _servicePrincipalSecret)); - return base.RunParametersAsync(parameterFilePath, ct, adjustedOverrides); + return base.RunParametersAsync(parameterFilePath, adjustedOverrides); } - protected async Task CleanDatabasesAsync(CancellationToken ct) + protected async Task CleanDatabasesAsync() { await Task.WhenAll( - CleanDbAsync(true, ct), - CleanDbAsync(false, ct)); + CleanDbAsync(true), + CleanDbAsync(false)); } protected async Task PrepareDbAsync( string scriptPath, - bool isCurrent, - CancellationToken ct) + bool isCurrent) { var script = await File.ReadAllTextAsync(scriptPath); @@ -295,7 +271,7 @@ protected async Task PrepareDbAsync( var commands = CommandBase.FromScript(script); var gateway = CreateKustoManagementGateway(isCurrent); - await gateway.ExecuteCommandsAsync(commands, ct); + await gateway.ExecuteCommandsAsync(commands); } catch (Exception ex) { @@ -333,18 +309,18 @@ private ITokenProvider CreateTokenProvider() return tokenProvider!; } - private async Task CleanDbAsync(bool isCurrent, CancellationToken ct) + private async Task CleanDbAsync(bool isCurrent) { var emptyDbProvider = (IDatabaseProvider)new EmptyDatabaseProvider(); var kustoGateway = CreateKustoManagementGateway(isCurrent); var dbProvider = (IDatabaseProvider)new KustoDatabaseProvider( new ConsoleTracer(false), kustoGateway); - var emptyDb = await emptyDbProvider.RetrieveDatabaseAsync(ct); - var db = await dbProvider.RetrieveDatabaseAsync(ct); + var emptyDb = await emptyDbProvider.RetrieveDatabaseAsync(); + var db = await dbProvider.RetrieveDatabaseAsync(); var currentDeltaCommands = db.ComputeDelta(emptyDb); - await kustoGateway.ExecuteCommandsAsync(currentDeltaCommands, ct); + await kustoGateway.ExecuteCommandsAsync(currentDeltaCommands); } } } \ No newline at end of file diff --git a/code/DeltaKustoAdxIntegrationTest/FailIfDataLoss/no-fail.json b/code/DeltaKustoAdxIntegrationTest/FailIfDataLoss/no-fail.json index f0b2f0e0..358261b9 100644 --- a/code/DeltaKustoAdxIntegrationTest/FailIfDataLoss/no-fail.json +++ b/code/DeltaKustoAdxIntegrationTest/FailIfDataLoss/no-fail.json @@ -12,7 +12,7 @@ "current": { "scripts": [ { - "filePath": "FailIfDataLoss/source.kql" + "filePath": "source.kql" } ] }, @@ -23,7 +23,7 @@ } }, "action": { - "filePath": "outputs/fail-if-data-loss/no-fail.kql" + "filePath": "outputs/no-fail.kql" } } } diff --git a/code/DeltaKustoAdxIntegrationTest/FailIfDataLossTest.cs b/code/DeltaKustoAdxIntegrationTest/FailIfDataLossTest.cs index 7a4e3957..094649dd 100644 --- a/code/DeltaKustoAdxIntegrationTest/FailIfDataLossTest.cs +++ b/code/DeltaKustoAdxIntegrationTest/FailIfDataLossTest.cs @@ -18,12 +18,11 @@ public class FailIfDataLossTest : AdxIntegrationTestBase public async Task TestFailIfDropsNoDrop() { var toFile = "FailIfDataLoss/target.kql"; - var ct = CreateCancellationToken(); - await CleanDatabasesAsync(ct); - await PrepareDbAsync(toFile, false, ct); + await CleanDatabasesAsync(); + await PrepareDbAsync(toFile, false); - await RunParametersAsync("FailIfDataLoss/no-fail.json", ct, TargetDbOverrides); + await RunParametersAsync("FailIfDataLoss/no-fail.json", TargetDbOverrides); // We just test that this doesn't fail } @@ -32,10 +31,9 @@ public async Task TestFailIfDropsNoDrop() public async Task TestFailIfDrops() { var toFile = "FailIfDataLoss/target.kql"; - var ct = CreateCancellationToken(); - await CleanDatabasesAsync(ct); - await PrepareDbAsync(toFile, false, ct); + await CleanDatabasesAsync(); + await PrepareDbAsync(toFile, false); var overrides = TargetDbOverrides .Append(("failIfDrops", "true")); @@ -44,7 +42,7 @@ public async Task TestFailIfDrops() { // The "Main" will return non-zero which will throw an exception var parameters = - await RunParametersAsync("FailIfDataLoss/no-fail.json", ct, overrides); + await RunParametersAsync("FailIfDataLoss/no-fail.json", overrides); Assert.True(parameters.FailIfDataLoss); Assert.False(true, "Should have thrown by now"); @@ -53,8 +51,5 @@ public async Task TestFailIfDrops() { } } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(8)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/EmptyDelta/empty-delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/EmptyDelta/empty-delta-params.yaml index d6274c25..8d63cc1c 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/EmptyDelta/empty-delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/EmptyDelta/empty-delta-params.yaml @@ -2,6 +2,6 @@ main: target: scripts: - - filePath: Functions/EmptyCurrent/EmptyDelta/target.kql + - filePath: target.kql action: - filePath: outputs/functions/empty-current/empty-delta.kql + filePath: outputs/empty-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/FunctionEmptyCurrentTest.cs b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/FunctionEmptyCurrentTest.cs index d3030ed5..40621f7a 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/FunctionEmptyCurrentTest.cs +++ b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/FunctionEmptyCurrentTest.cs @@ -11,11 +11,10 @@ public class FunctionEmptyCurrentTest : IntegrationTestBase [Fact] public async Task EmptyDelta() { - var parameters = await RunParametersAsync( - "Functions/EmptyCurrent/EmptyDelta/empty-delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Functions/EmptyCurrent/EmptyDelta/empty-delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Empty(commands); } @@ -23,13 +22,12 @@ public async Task EmptyDelta() [Fact] public async Task OneFunctionDelta() { - var parameters = await RunParametersAsync( - "Functions/EmptyCurrent/OneFunctionDelta/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Functions/EmptyCurrent/OneFunctionDelta/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var inputPath = parameters.Jobs!.First().Value.Target!.Scripts!.First().FilePath!; var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var inputCommands = await LoadScriptAsync(inputPath); - var outputCommands = await LoadScriptAsync(outputPath); + var inputCommands = await LoadScriptAsync(paramPath, inputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.True(inputCommands.SequenceEqual(outputCommands)); } @@ -37,18 +35,14 @@ public async Task OneFunctionDelta() [Fact] public async Task TwoFunctionsDelta() { - var parameters = await RunParametersAsync( - "Functions/EmptyCurrent/TwoFunctionsDelta/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Functions/EmptyCurrent/TwoFunctionsDelta/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var inputPath = parameters.Jobs!.First().Value.Target!.Scripts!.First().FilePath!; var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var inputCommands = await LoadScriptAsync(inputPath); - var outputCommands = await LoadScriptAsync(outputPath); + var inputCommands = await LoadScriptAsync(paramPath, inputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.True(inputCommands.SequenceEqual(outputCommands)); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/OneFunctionDelta/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/OneFunctionDelta/delta-params.yaml index a1aad5d0..9183757a 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/OneFunctionDelta/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/OneFunctionDelta/delta-params.yaml @@ -2,6 +2,6 @@ main: target: scripts: - - filePath: Functions/EmptyCurrent/OneFunctionDelta/target.kql + - filePath: target.kql action: - filePath: outputs/functions/empty-current/one-function-delta.kql \ No newline at end of file + filePath: outputs/one-function-delta.kql \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/TwoFunctionsDelta/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/TwoFunctionsDelta/delta-params.yaml index f92ef425..0ba6650c 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/TwoFunctionsDelta/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Functions/EmptyCurrent/TwoFunctionsDelta/delta-params.yaml @@ -2,6 +2,6 @@ main: target: scripts: - - filePath: Functions/EmptyCurrent/TwoFunctionsDelta/target.kql + - filePath: target.kql action: - filePath: outputs/functions/empty-current/two-functions-delta.kql \ No newline at end of file + filePath: outputs/two-functions-delta.kql \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/EmptyBoth/delta-params.json b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/EmptyBoth/delta-params.json index ad27a632..4e56e3bf 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/EmptyBoth/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/EmptyBoth/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "filePath": "Functions/WithCurrent/EmptyBoth/current.kql" + "filePath": "current.kql" } ] }, "target": { "scripts": [ { - "filePath": "Functions/WithCurrent/EmptyBoth/target.kql" + "filePath": "target.kql" } ] }, "action": { - "filePath": "outputs/functions/with-current/empty-both.kql" + "filePath": "outputs/empty-both.kql" } } } diff --git a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/FunctionWithCurrentTest.cs b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/FunctionWithCurrentTest.cs index 3cf5dba2..97a20c46 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/FunctionWithCurrentTest.cs +++ b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/FunctionWithCurrentTest.cs @@ -13,11 +13,10 @@ public class FunctionWithCurrentTest : IntegrationTestBase [Fact] public async Task EmptyBoth() { - var parameters = await RunParametersAsync( - "Functions/WithCurrent/EmptyBoth/delta-params.json", - CreateCancellationToken()); + var paramPath = "Functions/WithCurrent/EmptyBoth/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Empty(commands); } @@ -25,11 +24,10 @@ public async Task EmptyBoth() [Fact] public async Task Same() { - var parameters = await RunParametersAsync( - "Functions/WithCurrent/Same/delta-params.json", - CreateCancellationToken()); + var paramPath = "Functions/WithCurrent/Same/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Empty(commands); } @@ -37,11 +35,10 @@ public async Task Same() [Fact] public async Task TargetMore() { - var parameters = await RunParametersAsync( - "Functions/WithCurrent/TargetMore/delta-params.json", - CreateCancellationToken()); + var paramPath = "Functions/WithCurrent/TargetMore/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -53,12 +50,11 @@ public async Task TargetMore() [Fact] public async Task TargetLess() { - var parameters = await RunParametersAsync( - "Functions/WithCurrent/TargetLess/delta-params.json", - CreateCancellationToken()); + var paramPath = "Functions/WithCurrent/TargetLess/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputRootFolder = parameters.Jobs!.First().Value.Action!.FolderPath!; var outputPath = Path.Combine(outputRootFolder, "functions/drop/YourFunction.kql"); - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -66,8 +62,5 @@ public async Task TargetLess() Assert.Equal("YourFunction", function.FunctionName.Name); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/Same/delta-params.json b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/Same/delta-params.json index 1c73a213..1efd6643 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/Same/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/Same/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "filePath": "Functions/WithCurrent/Same/current.kql" + "filePath": "current.kql" } ] }, "target": { "scripts": [ { - "filePath": "Functions/WithCurrent/Same/target.kql" + "filePath": "target.kql" } ] }, "action": { - "filePath": "outputs/functions/with-current/same.kql" + "filePath": "outputs/same.kql" } } } diff --git a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetLess/delta-params.json b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetLess/delta-params.json index 4895a648..031a52dd 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetLess/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetLess/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "filePath": "Functions/WithCurrent/TargetLess/current.kql" + "filePath": "current.kql" } ] }, "target": { "scripts": [ { - "filePath": "Functions/WithCurrent/TargetLess/target.kql" + "filePath": "target.kql" } ] }, "action": { - "folderPath": "outputs/functions/with-current/target-less" + "folderPath": "outputs/target-less" } } } diff --git a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetMore/delta-params.json b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetMore/delta-params.json index ee00635c..c2ae3eb1 100644 --- a/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetMore/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Functions/WithCurrent/TargetMore/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "folderPath": "Functions/WithCurrent/TargetMore/Current" + "folderPath": "Current" } ] }, "target": { "scripts": [ { - "folderPath": "Functions/WithCurrent/TargetMore/Target/" + "folderPath": "Target/" } ] }, "action": { - "filePath": "outputs/functions/with-current/target-more.kql" + "filePath": "outputs/target-more.kql" } } } diff --git a/code/DeltaKustoFileIntegrationTest/IntegrationTestBase.cs b/code/DeltaKustoFileIntegrationTest/IntegrationTestBase.cs index e11a805a..283e57ad 100644 --- a/code/DeltaKustoFileIntegrationTest/IntegrationTestBase.cs +++ b/code/DeltaKustoFileIntegrationTest/IntegrationTestBase.cs @@ -54,6 +54,8 @@ private class ProjectSetting } #endregion + private static readonly TimeSpan PROCESS_TIMEOUT = TimeSpan.FromSeconds(20); + private readonly string? _executablePath; static IntegrationTestBase() @@ -85,9 +87,9 @@ static IntegrationTestBase() } protected SimpleHttpClientFactory HttpClientFactory { get; } - + protected IKustoManagementGatewayFactory GatewayFactory { get; } - + protected ITokenProviderFactory TokenProviderFactory { get; } protected IntegrationTestBase() @@ -109,15 +111,13 @@ protected IntegrationTestBase() } var tracer = new ConsoleTracer(false); - + HttpClientFactory = new SimpleHttpClientFactory(tracer); GatewayFactory = new KustoManagementGatewayFactory(tracer, HttpClientFactory); TokenProviderFactory = new TokenProviderFactory(tracer, HttpClientFactory); } - protected async virtual Task RunMainAsync( - CancellationToken ct, - params string[] args) + protected async virtual Task RunMainAsync(params string[] args) { if (_executablePath == null) { @@ -148,6 +148,8 @@ protected async virtual Task RunMainAsync( if (started) { + var ct = new CancellationTokenSource(PROCESS_TIMEOUT).Token; + await process.WaitForExitAsync(ct); return process.ExitCode; @@ -161,18 +163,15 @@ protected async virtual Task RunMainAsync( } } - protected async virtual Task RunSuccessfulMainAsync( - CancellationToken ct, - params string[] args) + protected async virtual Task RunSuccessfulMainAsync(params string[] args) { - var returnedValue = await RunMainAsync(ct, args); + var returnedValue = await RunMainAsync(args); Assert.Equal(0, returnedValue); } protected async virtual Task RunParametersAsync( string parameterFilePath, - CancellationToken ct, IEnumerable<(string path, string value)>? overrides = null) { var pathOverrides = overrides != null @@ -182,7 +181,7 @@ protected async virtual Task RunParametersAsync( var cliParameters = overrides != null ? baseParameters.Append("-o").Concat(pathOverrides) : baseParameters; - var returnedValue = await RunMainAsync(ct, cliParameters.ToArray()); + var returnedValue = await RunMainAsync(cliParameters.ToArray()); if (returnedValue != 0) { @@ -203,14 +202,19 @@ protected async virtual Task RunParametersAsync( return parameters; } - protected async virtual Task> LoadScriptAsync(string scriptPath) + protected async virtual Task> LoadScriptAsync( + string paramPath, + string scriptPath) { - if(!File.Exists(scriptPath)) + var rootFolder = Path.GetDirectoryName(paramPath) ?? ""; + var path = Path.Combine(rootFolder!, scriptPath); + + if (!File.Exists(path)) { - throw new InvalidOperationException($"Can't find '{scriptPath}'"); + throw new InvalidOperationException($"Can't find '{path}'"); } - var script = await File.ReadAllTextAsync(scriptPath); + var script = await File.ReadAllTextAsync(path); var commands = CommandBase.FromScript(script); return commands; diff --git a/code/DeltaKustoFileIntegrationTest/JsonSchema/HelpSamples/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/JsonSchema/HelpSamples/delta-params.yaml index fe9d20f2..b16d76ed 100644 --- a/code/DeltaKustoFileIntegrationTest/JsonSchema/HelpSamples/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/JsonSchema/HelpSamples/delta-params.yaml @@ -1,6 +1,6 @@ jobs: main: target: - jsonFilePath: JsonSchema/HelpSamples/schema.json + jsonFilePath: schema.json action: - filePath: outputs/json-schema/help-samples.kql \ No newline at end of file + filePath: outputs/help-samples.kql \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/JsonSchema/JsonSchemaTest.cs b/code/DeltaKustoFileIntegrationTest/JsonSchema/JsonSchemaTest.cs index a6ef8eea..f21223d6 100644 --- a/code/DeltaKustoFileIntegrationTest/JsonSchema/JsonSchemaTest.cs +++ b/code/DeltaKustoFileIntegrationTest/JsonSchema/JsonSchemaTest.cs @@ -12,11 +12,10 @@ public class JsonSchemaTest : IntegrationTestBase [Fact] public async Task HelpSamples() { - var parameters = await RunParametersAsync( - "JsonSchema/HelpSamples/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "JsonSchema/HelpSamples/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var outputCommands = await LoadScriptAsync(outputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); // Mostly a check we can process the json file var oneTable = outputCommands .Where(c => c is CreateTableCommand) @@ -25,8 +24,5 @@ public async Task HelpSamples() Assert.Single(oneTable); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Mappings/MappingTest.cs b/code/DeltaKustoFileIntegrationTest/Mappings/MappingTest.cs index 5fb1e503..1c1d7551 100644 --- a/code/DeltaKustoFileIntegrationTest/Mappings/MappingTest.cs +++ b/code/DeltaKustoFileIntegrationTest/Mappings/MappingTest.cs @@ -13,11 +13,10 @@ public class MappingTest : IntegrationTestBase [Fact] public async Task NoneToOne() { - var parameters = await RunParametersAsync( - "Mappings/NoneToOne/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Mappings/NoneToOne/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var outputCommands = await LoadScriptAsync(outputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(outputCommands); @@ -33,11 +32,10 @@ public async Task NoneToOne() [Fact] public async Task OneToTwo() { - var parameters = await RunParametersAsync( - "Mappings/OneToTwo/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Mappings/OneToTwo/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var outputCommands = await LoadScriptAsync(outputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.Equal(2, outputCommands.Count()); @@ -58,11 +56,10 @@ public async Task OneToTwo() [Fact] public async Task TwoToOne() { - var parameters = await RunParametersAsync( - "Mappings/TwoToOne/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Mappings/TwoToOne/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var outputCommands = await LoadScriptAsync(outputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(outputCommands); @@ -70,8 +67,5 @@ public async Task TwoToOne() Assert.Equal("csv", dropMappingCommand.MappingKind); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Mappings/NoneToOne/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Mappings/NoneToOne/delta-params.yaml index e9baaa72..4554cdb0 100644 --- a/code/DeltaKustoFileIntegrationTest/Mappings/NoneToOne/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Mappings/NoneToOne/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Mappings/NoneToOne/current.kql + - filePath: current.kql target: scripts: - - filePath: Mappings/NoneToOne/target.kql + - filePath: target.kql action: - filePath: outputs/mappings/none-to-one.kql + filePath: outputs/none-to-one.kql diff --git a/code/DeltaKustoFileIntegrationTest/Mappings/OneToTwo/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Mappings/OneToTwo/delta-params.yaml index e75b246d..eb561a18 100644 --- a/code/DeltaKustoFileIntegrationTest/Mappings/OneToTwo/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Mappings/OneToTwo/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Mappings/OneToTwo/current.kql + - filePath: current.kql target: scripts: - - filePath: Mappings/OneToTwo/target.kql + - filePath: target.kql action: - filePath: outputs/mappings/one-to-two.kql + filePath: outputs/one-to-two.kql diff --git a/code/DeltaKustoFileIntegrationTest/Mappings/TwoToOne/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Mappings/TwoToOne/delta-params.yaml index 8cc76126..e7a99c5e 100644 --- a/code/DeltaKustoFileIntegrationTest/Mappings/TwoToOne/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Mappings/TwoToOne/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Mappings/TwoToOne/current.kql + - filePath: current.kql target: scripts: - - filePath: Mappings/TwoToOne/target.kql + - filePath: target.kql action: - filePath: outputs/mappings/two-to-one.kql + filePath: outputs/two-to-one.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/OneTableDelta/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/OneTableDelta/delta-params.yaml index 8230fff3..50ac20e9 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/OneTableDelta/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/OneTableDelta/delta-params.yaml @@ -2,6 +2,6 @@ main: target: scripts: - - filePath: Tables/EmptyCurrent/OneTableDelta/target.kql + - filePath: target.kql action: - filePath: outputs/tables/empty-current/one-table-delta.kql \ No newline at end of file + filePath: outputs/one-table-delta.kql \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TableEmptyCurrentTest.cs b/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TableEmptyCurrentTest.cs index 2727c894..ce9f0b1a 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TableEmptyCurrentTest.cs +++ b/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TableEmptyCurrentTest.cs @@ -11,13 +11,12 @@ public class TableEmptyCurrentTest : IntegrationTestBase [Fact] public async Task OneTableDelta() { - var parameters = await RunParametersAsync( - "Tables/EmptyCurrent/OneTableDelta/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/EmptyCurrent/OneTableDelta/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var inputPath = parameters.Jobs!.First().Value.Target!.Scripts!.First().FilePath!; var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var inputCommands = await LoadScriptAsync(inputPath); - var outputCommands = await LoadScriptAsync(outputPath); + var inputCommands = await LoadScriptAsync(paramPath, inputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.True(inputCommands.SequenceEqual(outputCommands)); } @@ -25,18 +24,14 @@ public async Task OneTableDelta() [Fact] public async Task TwoFunctionsDelta() { - var parameters = await RunParametersAsync( - "Tables/EmptyCurrent/TwoTablesDelta/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/EmptyCurrent/TwoTablesDelta/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var inputPath = parameters.Jobs!.First().Value.Target!.Scripts!.First().FilePath!; var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var inputCommands = await LoadScriptAsync(inputPath); - var outputCommands = await LoadScriptAsync(outputPath); + var inputCommands = await LoadScriptAsync(paramPath, inputPath); + var outputCommands = await LoadScriptAsync(paramPath, outputPath); Assert.True(inputCommands.SequenceEqual(outputCommands)); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TwoTablesDelta/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TwoTablesDelta/delta-params.yaml index 2393b285..9c3abbfe 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TwoTablesDelta/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/EmptyCurrent/TwoTablesDelta/delta-params.yaml @@ -2,6 +2,6 @@ main: target: scripts: - - filePath: Tables/EmptyCurrent/TwoTablesDelta/target.kql + - filePath: target.kql action: - filePath: outputs/tables/empty-current/two-tables-delta.kql \ No newline at end of file + filePath: outputs/two-tables-delta.kql \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddDocStringOnTable/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddDocStringOnTable/delta-params.yaml index 199041c1..92e85c31 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddDocStringOnTable/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddDocStringOnTable/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Tables/Scenarios/AddDocStringOnTable/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/AddDocStringOnTable/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/add-doc-string-on-table-delta.kql + filePath: outputs/add-doc-string-on-table-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddFolderOnTable/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddFolderOnTable/delta-params.yaml index 12bf8521..0225958b 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddFolderOnTable/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/AddFolderOnTable/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Tables/Scenarios/AddFolderOnTable/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/AddFolderOnTable/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/add-folder-on-table-delta.kql + filePath: outputs/add-folder-on-table-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnDoc/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnDoc/delta-params.yaml index 88ff2baf..9bea4136 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnDoc/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnDoc/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Tables/Scenarios/ChangeColumnDoc/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/ChangeColumnDoc/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/change-column-doc-delta.kql + filePath: outputs/change-column-doc-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnType/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnType/delta-params.yaml index dcf2d03a..20c4d847 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnType/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/ChangeColumnType/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Tables/Scenarios/ChangeColumnType/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/ChangeColumnType/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/change-column-type-delta.kql + filePath: outputs/change-column-type-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumn/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumn/delta-params.yaml index 22a1636f..a298e3ca 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumn/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumn/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Tables/Scenarios/DropColumn/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/DropColumn/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/drop-column-delta.kql + filePath: outputs/drop-column-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumnDoc/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumnDoc/delta-params.yaml index 705f1b0b..02c6f6dc 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumnDoc/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropColumnDoc/delta-params.yaml @@ -2,9 +2,9 @@ main: current: scripts: - - filePath: Tables/Scenarios/DropColumnDoc/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/DropColumnDoc/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/drop-column-doc-delta.kql + filePath: outputs/drop-column-doc-delta.kql diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropMultipleTables/delta-params.yaml b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropMultipleTables/delta-params.yaml index d901a44c..6875488c 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropMultipleTables/delta-params.yaml +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/DropMultipleTables/delta-params.yaml @@ -2,10 +2,10 @@ main: current: scripts: - - filePath: Tables/Scenarios/DropMultipleTables/current.kql + - filePath: current.kql target: scripts: - - filePath: Tables/Scenarios/DropMultipleTables/target.kql + - filePath: target.kql action: - filePath: outputs/tables/scenarios/drop-multiple-tables-delta.kql + filePath: outputs/drop-multiple-tables-delta.kql usePluralForms: true diff --git a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/TableScenariosTest.cs b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/TableScenariosTest.cs index 0b7a64af..f8da91e7 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/TableScenariosTest.cs +++ b/code/DeltaKustoFileIntegrationTest/Tables/Scenarios/TableScenariosTest.cs @@ -13,11 +13,10 @@ public class TableScenariosTest : IntegrationTestBase [Fact] public async Task AddFolderOnTable() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/AddFolderOnTable/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/AddFolderOnTable/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -29,11 +28,10 @@ public async Task AddFolderOnTable() [Fact] public async Task AddDocStringOnTable() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/AddDocStringOnTable/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/AddDocStringOnTable/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -45,11 +43,10 @@ public async Task AddDocStringOnTable() [Fact] public async Task ChangeColumnType() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/ChangeColumnType/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/ChangeColumnType/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -61,11 +58,10 @@ public async Task ChangeColumnType() [Fact] public async Task DropColumn() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/DropColumn/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/DropColumn/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -78,11 +74,10 @@ public async Task DropColumn() [Fact] public async Task ChangeColumnDocString() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/ChangeColumnDoc/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/ChangeColumnDoc/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -99,11 +94,10 @@ public async Task ChangeColumnDocString() [Fact] public async Task DropColumnDocString() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/DropColumnDoc/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/DropColumnDoc/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -120,11 +114,10 @@ public async Task DropColumnDocString() [Fact] public async Task DropMultipleTablesTablesString() { - var parameters = await RunParametersAsync( - "Tables/Scenarios/DropMultipleTables/delta-params.yaml", - CreateCancellationToken()); + var paramPath = "Tables/Scenarios/DropMultipleTables/delta-params.yaml"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -132,8 +125,5 @@ public async Task DropMultipleTablesTablesString() Assert.Equal(3, dropTables.TableNames.Count); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/EmptyBoth/delta-params.json b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/EmptyBoth/delta-params.json index e125fb7f..4e56e3bf 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/EmptyBoth/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/EmptyBoth/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "filePath": "Tables/WithCurrent/EmptyBoth/current.kql" + "filePath": "current.kql" } ] }, "target": { "scripts": [ { - "filePath": "Tables/WithCurrent/EmptyBoth/target.kql" + "filePath": "target.kql" } ] }, "action": { - "filePath": "outputs/tables/with-current/empty-both.kql" + "filePath": "outputs/empty-both.kql" } } } diff --git a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/Same/delta-params.json b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/Same/delta-params.json index 344bc2d8..1efd6643 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/Same/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/Same/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "filePath": "Tables/WithCurrent/Same/current.kql" + "filePath": "current.kql" } ] }, "target": { "scripts": [ { - "filePath": "Tables/WithCurrent/Same/target.kql" + "filePath": "target.kql" } ] }, "action": { - "filePath": "outputs/tables/with-current/same.kql" + "filePath": "outputs/same.kql" } } } diff --git a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TableWithCurrentTest.cs b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TableWithCurrentTest.cs index 054b2766..98e5793f 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TableWithCurrentTest.cs +++ b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TableWithCurrentTest.cs @@ -13,11 +13,10 @@ public class TableWithCurrentTest : IntegrationTestBase [Fact] public async Task EmptyBoth() { - var parameters = await RunParametersAsync( - "Tables/WithCurrent/EmptyBoth/delta-params.json", - CreateCancellationToken()); + var paramPath = "Tables/WithCurrent/EmptyBoth/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Empty(commands); } @@ -25,11 +24,10 @@ public async Task EmptyBoth() [Fact] public async Task Same() { - var parameters = await RunParametersAsync( - "Tables/WithCurrent/Same/delta-params.json", - CreateCancellationToken()); + var paramPath = "Tables/WithCurrent/Same/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Empty(commands); } @@ -37,11 +35,10 @@ public async Task Same() [Fact] public async Task TargetMore() { - var parameters = await RunParametersAsync( - "Tables/WithCurrent/TargetMore/delta-params.json", - CreateCancellationToken()); + var paramPath = "Tables/WithCurrent/TargetMore/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputPath = parameters.Jobs!.First().Value.Action!.FilePath!; - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -53,12 +50,11 @@ public async Task TargetMore() [Fact] public async Task TargetLess() { - var parameters = await RunParametersAsync( - "Tables/WithCurrent/TargetLess/delta-params.json", - CreateCancellationToken()); + var paramPath = "Tables/WithCurrent/TargetLess/delta-params.json"; + var parameters = await RunParametersAsync(paramPath); var outputRootFolder = parameters.Jobs!.First().Value.Action!.FolderPath!; var outputPath = Path.Combine(outputRootFolder, "tables/drop/your-table.kql"); - var commands = await LoadScriptAsync(outputPath); + var commands = await LoadScriptAsync(paramPath, outputPath); Assert.Single(commands); @@ -66,8 +62,5 @@ public async Task TargetLess() Assert.Equal("your-table", table.TableName.Name); } - - private CancellationToken CreateCancellationToken() => - new CancellationTokenSource(TimeSpan.FromSeconds(2)).Token; } } \ No newline at end of file diff --git a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetLess/delta-params.json b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetLess/delta-params.json index 78b3a9eb..031a52dd 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetLess/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetLess/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "filePath": "Tables/WithCurrent/TargetLess/current.kql" + "filePath": "current.kql" } ] }, "target": { "scripts": [ { - "filePath": "Tables/WithCurrent/TargetLess/target.kql" + "filePath": "target.kql" } ] }, "action": { - "folderPath": "outputs/tables/with-current/target-less" + "folderPath": "outputs/target-less" } } } diff --git a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetMore/delta-params.json b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetMore/delta-params.json index fa453203..c2ae3eb1 100644 --- a/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetMore/delta-params.json +++ b/code/DeltaKustoFileIntegrationTest/Tables/WithCurrent/TargetMore/delta-params.json @@ -4,19 +4,19 @@ "current": { "scripts": [ { - "folderPath": "Tables/WithCurrent/TargetMore/Current" + "folderPath": "Current" } ] }, "target": { "scripts": [ { - "folderPath": "Tables/WithCurrent/TargetMore/Target/" + "folderPath": "Target/" } ] }, "action": { - "filePath": "outputs/tables/with-current/target-more.kql" + "filePath": "outputs/target-more.kql" } } } diff --git a/code/DeltaKustoIntegration/Action/IActionProvider.cs b/code/DeltaKustoIntegration/Action/IActionProvider.cs index ce312f77..64b79e7c 100644 --- a/code/DeltaKustoIntegration/Action/IActionProvider.cs +++ b/code/DeltaKustoIntegration/Action/IActionProvider.cs @@ -12,6 +12,6 @@ public interface IActionProvider Task ProcessDeltaCommandsAsync( bool doNotProcessIfDataLoss, ActionCommandCollection commands, - CancellationToken ct); + CancellationToken ct = default); } } \ No newline at end of file diff --git a/code/DeltaKustoIntegration/CancellationTokenHelper.cs b/code/DeltaKustoIntegration/CancellationTokenHelper.cs new file mode 100644 index 00000000..c9dd2182 --- /dev/null +++ b/code/DeltaKustoIntegration/CancellationTokenHelper.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace DeltaKustoIntegration +{ + internal static class CancellationTokenHelper + { + public static CancellationToken MergeCancellationToken( + CancellationToken ct, + TimeSpan timeout) + { + var timeoutSource = new CancellationTokenSource(timeout); + var merged = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutSource.Token); + + return merged.Token; + } + } +} \ No newline at end of file diff --git a/code/DeltaKustoIntegration/Database/IDatabaseProvider.cs b/code/DeltaKustoIntegration/Database/IDatabaseProvider.cs index 5b7783bb..7076bd43 100644 --- a/code/DeltaKustoIntegration/Database/IDatabaseProvider.cs +++ b/code/DeltaKustoIntegration/Database/IDatabaseProvider.cs @@ -9,6 +9,6 @@ namespace DeltaKustoIntegration.Database { public interface IDatabaseProvider { - Task RetrieveDatabaseAsync(CancellationToken ct); + Task RetrieveDatabaseAsync(CancellationToken ct = default); } } \ No newline at end of file diff --git a/code/DeltaKustoIntegration/Database/ScriptDatabaseProvider.cs b/code/DeltaKustoIntegration/Database/ScriptDatabaseProvider.cs index d342992e..9101f07a 100644 --- a/code/DeltaKustoIntegration/Database/ScriptDatabaseProvider.cs +++ b/code/DeltaKustoIntegration/Database/ScriptDatabaseProvider.cs @@ -62,10 +62,9 @@ private async Task> LoadScriptsAsync( } else if (fileParametrization.FolderPath != null) { - var scripts = _fileGateway.GetFolderContentsAsync( - fileParametrization.FolderPath, - fileParametrization.Extensions, - ct); + var scripts = _fileGateway + .ChangeFolder(fileParametrization.FolderPath) + .GetFolderContentsAsync(fileParametrization.Extensions, ct); var contents = (await scripts.ToEnumerableAsync()) .Select(t => t.content); diff --git a/code/DeltaKustoIntegration/FileGateway.cs b/code/DeltaKustoIntegration/FileGateway.cs index b140cb15..7e543352 100644 --- a/code/DeltaKustoIntegration/FileGateway.cs +++ b/code/DeltaKustoIntegration/FileGateway.cs @@ -12,14 +12,34 @@ namespace DeltaKustoIntegration { public class FileGateway : IFileGateway { + private static readonly TimeSpan TIMEOUT = TimeSpan.FromSeconds(2); + private readonly string _rootFolder; + + public FileGateway() : this(string.Empty) + { + } + + private FileGateway(string rootFolder) + { + _rootFolder = rootFolder; + } + + IFileGateway IFileGateway.ChangeFolder(string folderPath) + { + var newRootFolder = Path.Combine(_rootFolder, folderPath); + + return new FileGateway(newRootFolder); + } + async Task IFileGateway.GetFileContentAsync( string filePath, CancellationToken ct) { + var path = Path.Combine(_rootFolder, filePath); var text = await File.ReadAllTextAsync( - filePath, + path, Encoding.UTF8, - ct); + CancellationTokenHelper.MergeCancellationToken(ct, TIMEOUT)); return text; } @@ -29,7 +49,8 @@ async Task IFileGateway.SetFileContentAsync( string content, CancellationToken ct) { - var directory = Path.GetDirectoryName(filePath); + var path = Path.Combine(_rootFolder, filePath); + var directory = Path.GetDirectoryName(path); if (!string.IsNullOrWhiteSpace(directory)) { @@ -37,37 +58,39 @@ async Task IFileGateway.SetFileContentAsync( } await File.WriteAllTextAsync( - filePath, + path, content, - ct); + CancellationTokenHelper.MergeCancellationToken(ct, TIMEOUT)); } async IAsyncEnumerable<(string path, string content)> IFileGateway.GetFolderContentsAsync( - string folderPath, IEnumerable? extensions, [EnumeratorCancellation] CancellationToken ct) { var fileGateway = (IFileGateway)this; - var directories = Directory.GetDirectories(folderPath); - var files = Directory.GetFiles(folderPath); + var directories = Directory.GetDirectories(_rootFolder); + var files = Directory.GetFiles(_rootFolder); foreach (var file in files) { if (HasExtension(file, extensions)) { - var script = await fileGateway.GetFileContentAsync(file, ct); + var fileName = Path.GetFileName(file); + var script = await fileGateway.GetFileContentAsync(fileName, ct); - yield return (file, script); + yield return (fileName, script); } } foreach (var directory in directories) { - var scripts = fileGateway.GetFolderContentsAsync(directory, extensions, ct); + var directoryName = Path.GetDirectoryName(directory)!; + var localFileGateway = fileGateway.ChangeFolder(directoryName); + var scripts = fileGateway.GetFolderContentsAsync(extensions, ct); await foreach (var script in scripts) { - yield return script; + yield return (Path.Combine(directoryName, script.path), script.content); } } } diff --git a/code/DeltaKustoIntegration/IFileGateway.cs b/code/DeltaKustoIntegration/IFileGateway.cs index 634d717e..c2928a14 100644 --- a/code/DeltaKustoIntegration/IFileGateway.cs +++ b/code/DeltaKustoIntegration/IFileGateway.cs @@ -8,13 +8,14 @@ namespace DeltaKustoIntegration { public interface IFileGateway { - Task GetFileContentAsync(string filePath, CancellationToken ct); + IFileGateway ChangeFolder(string folderPath); - Task SetFileContentAsync(string filePath, string content, CancellationToken ct); + Task GetFileContentAsync(string filePath, CancellationToken ct = default); + + Task SetFileContentAsync(string filePath, string content, CancellationToken ct = default); IAsyncEnumerable<(string path, string content)> GetFolderContentsAsync( - string folderPath, IEnumerable? extensions, - CancellationToken ct); + CancellationToken ct = default); } } \ No newline at end of file diff --git a/code/DeltaKustoIntegration/Kusto/IKustoManagementGateway.cs b/code/DeltaKustoIntegration/Kusto/IKustoManagementGateway.cs index ce9f45ce..c9d02c3a 100644 --- a/code/DeltaKustoIntegration/Kusto/IKustoManagementGateway.cs +++ b/code/DeltaKustoIntegration/Kusto/IKustoManagementGateway.cs @@ -10,8 +10,10 @@ namespace DeltaKustoIntegration.Kusto { public interface IKustoManagementGateway { - Task GetDatabaseSchemaAsync(CancellationToken ct); - - Task ExecuteCommandsAsync(IEnumerable commands, CancellationToken ct); + Task GetDatabaseSchemaAsync(CancellationToken ct = default); + + Task ExecuteCommandsAsync( + IEnumerable commands, + CancellationToken ct = default); } } \ No newline at end of file diff --git a/code/DeltaKustoIntegration/Kusto/KustoManagementGateway.cs b/code/DeltaKustoIntegration/Kusto/KustoManagementGateway.cs index 50e1546a..13c2db0b 100644 --- a/code/DeltaKustoIntegration/Kusto/KustoManagementGateway.cs +++ b/code/DeltaKustoIntegration/Kusto/KustoManagementGateway.cs @@ -301,6 +301,8 @@ public void Validate(string json, string tableName) } #endregion + private static readonly TimeSpan TIMEOUT = TimeSpan.FromSeconds(10); + private readonly Uri _clusterUri; private readonly string _database; private readonly ITokenProvider _tokenProvider; @@ -450,6 +452,8 @@ private async Task ExecuteCommandAsync( { try { + ct = CancellationTokenHelper.MergeCancellationToken(ct, TIMEOUT); + var token = await _tokenProvider.GetTokenAsync(_clusterUri, ct); // Implementation of https://docs.microsoft.com/en-us/azure/data-explorer/kusto/api/rest/request#examples diff --git a/code/DeltaKustoIntegration/TokenProvider/ITokenProvider.cs b/code/DeltaKustoIntegration/TokenProvider/ITokenProvider.cs index 8679c240..fbad4de3 100644 --- a/code/DeltaKustoIntegration/TokenProvider/ITokenProvider.cs +++ b/code/DeltaKustoIntegration/TokenProvider/ITokenProvider.cs @@ -8,6 +8,6 @@ namespace DeltaKustoIntegration.TokenProvider { public interface ITokenProvider { - Task GetTokenAsync(Uri clusterUri, CancellationToken ct); + Task GetTokenAsync(Uri clusterUri, CancellationToken ct = default); } } \ No newline at end of file diff --git a/code/DeltaKustoIntegration/TokenProvider/LoginTokenProvider.cs b/code/DeltaKustoIntegration/TokenProvider/LoginTokenProvider.cs index 980e8690..e764e96f 100644 --- a/code/DeltaKustoIntegration/TokenProvider/LoginTokenProvider.cs +++ b/code/DeltaKustoIntegration/TokenProvider/LoginTokenProvider.cs @@ -13,6 +13,8 @@ namespace DeltaKustoIntegration.TokenProvider { internal class LoginTokenProvider : ITokenProvider { + private static readonly TimeSpan TIMEOUT = TimeSpan.FromSeconds(5); + private readonly ITracer _tracer; private readonly SimpleHttpClientFactory _httpClientFactory; private readonly string _tenantId; @@ -69,6 +71,7 @@ async Task ITokenProvider.GetTokenAsync( private async Task RetrieveTokenAsync(Uri clusterUri, CancellationToken ct) { + ct = CancellationTokenHelper.MergeCancellationToken(ct, TIMEOUT); _tracer.WriteLine(true, "LoginTokenProvider.RetrieveTokenAsync start"); // Implementation of https://docs.microsoft.com/en-us/azure/data-explorer/kusto/api/rest/request#examples diff --git a/code/delta-kusto/ApiClient.cs b/code/delta-kusto/ApiClient.cs index de71122e..544f1400 100644 --- a/code/delta-kusto/ApiClient.cs +++ b/code/delta-kusto/ApiClient.cs @@ -91,6 +91,7 @@ private class ErrorOutput #endregion private const string DEFAULT_ROOT_URL = "https://delta-kusto.azurefd.net/"; + private static readonly TimeSpan TIMEOUT = TimeSpan.FromSeconds(10); private static readonly string ROOT_URL = ComputeRootUrl(); private static readonly bool _doApiCalls = ComputeDoApiCalls(); @@ -108,7 +109,7 @@ public ApiClient(ITracer tracer, SimpleHttpClientFactory httpClientFactory) { if (_doApiCalls) { - var tokenSource = new CancellationTokenSource(TimeOuts.API); + var tokenSource = new CancellationTokenSource(TIMEOUT); var ct = tokenSource.Token; _tracer.WriteLine(true, "ActivateAsync - Start"); @@ -142,7 +143,7 @@ public ApiClient(ITracer tracer, SimpleHttpClientFactory httpClientFactory) { if (_doApiCalls) { - var tokenSource = new CancellationTokenSource(TimeOuts.API); + var tokenSource = new CancellationTokenSource(TIMEOUT); var ct = tokenSource.Token; _tracer.WriteLine(true, "RegisterExceptionAsync - Start"); diff --git a/code/delta-kusto/DeltaOrchestration.cs b/code/delta-kusto/DeltaOrchestration.cs index ba5571f0..66bc180b 100644 --- a/code/delta-kusto/DeltaOrchestration.cs +++ b/code/delta-kusto/DeltaOrchestration.cs @@ -70,9 +70,11 @@ public async Task ComputeDeltaAsync( var parameters = await LoadParameterizationAsync(parameterFilePath, pathOverrides); + var parameterFolderPath = Path.GetDirectoryName(parameterFilePath); try { + var localFileGateway = _fileGateway.ChangeFolder(parameterFolderPath!); var tokenProvider = _tokenProviderFactory.CreateProvider(parameters.TokenProvider); var orderedJobs = parameters.Jobs.OrderBy(p => p.Value.Priority); var success = true; @@ -84,6 +86,7 @@ public async Task ComputeDeltaAsync( var (jobName, job) = jobPair; var jobSuccess = await ProcessJobAsync( parameters, + localFileGateway, tokenProvider, jobName, job); @@ -109,6 +112,7 @@ public async Task ComputeDeltaAsync( private async Task ProcessJobAsync( MainParameterization parameters, + IFileGateway localFileGateway, ITokenProvider? tokenProvider, string jobName, JobParameterization job) @@ -119,16 +123,16 @@ private async Task ProcessJobAsync( { _tracer.WriteLine(false, "Current DB Provider... "); - var currentDbProvider = CreateDatabaseProvider(job.Current, tokenProvider); + var currentDbProvider = + CreateDatabaseProvider(job.Current, tokenProvider, localFileGateway); _tracer.WriteLine(false, "Target DB Provider... "); - var targetDbProvider = CreateDatabaseProvider(job.Target, tokenProvider); - var tokenSourceRetrieveDb = new CancellationTokenSource(TimeOuts.RETRIEVE_DB); - var ctRetrieveDb = tokenSourceRetrieveDb.Token; + var targetDbProvider = + CreateDatabaseProvider(job.Target, tokenProvider, localFileGateway); - var currentDbTask = RetrieveDatabaseAsync(currentDbProvider, "current", ctRetrieveDb); - var targetDbTask = RetrieveDatabaseAsync(targetDbProvider, "target", ctRetrieveDb); + var currentDbTask = RetrieveDatabaseAsync(currentDbProvider, "current"); + var targetDbTask = RetrieveDatabaseAsync(targetDbProvider, "target"); await Task.WhenAll(currentDbTask, targetDbTask); @@ -142,18 +146,16 @@ private async Task ProcessJobAsync( var jobSuccess = ReportOnDeltaCommands(parameters, actions); var actionProviders = CreateActionProvider( job.Action!, + localFileGateway, tokenProvider, job.Current?.Adx); - var tokenSourceAction = new CancellationTokenSource(TimeOuts.ACTION); - var ctAction = tokenSourceRetrieveDb.Token; _tracer.WriteLine(false, "Processing delta commands..."); foreach (var actionProvider in actionProviders) { await actionProvider.ProcessDeltaCommandsAsync( parameters.FailIfDataLoss, - actions, - ctAction); + actions); } _tracer.WriteLine(false, "Delta processed / Job completed"); _tracer.WriteLine(false, ""); @@ -168,12 +170,11 @@ await actionProvider.ProcessDeltaCommandsAsync( private async Task RetrieveDatabaseAsync( IDatabaseProvider currentDbProvider, - string db, - CancellationToken ct) + string db) { _tracer.WriteLine(false, $"Retrieving {db}..."); - var model = await currentDbProvider.RetrieveDatabaseAsync(ct); + var model = await currentDbProvider.RetrieveDatabaseAsync(); _tracer.WriteLine(false, $"{db} retrieved"); @@ -209,15 +210,12 @@ internal async Task LoadParameterizationAsync( string parameterFilePath, IEnumerable pathOverrides) { - var tokenSource = new CancellationTokenSource(TimeOuts.FILE); - var ct = tokenSource.Token; - try { var deserializer = new DeserializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .Build(); - var parameterText = await _fileGateway.GetFileContentAsync(parameterFilePath, ct); + var parameterText = await _fileGateway.GetFileContentAsync(parameterFilePath); var parameters = deserializer.Deserialize(parameterText); if (parameters == null) @@ -241,6 +239,7 @@ internal async Task LoadParameterizationAsync( private IImmutableList CreateActionProvider( ActionParameterization action, + IFileGateway localFileGateway, ITokenProvider? tokenProvider, AdxSourceParameterization? database) { @@ -251,14 +250,14 @@ private IImmutableList CreateActionProvider( if (action.FilePath != null) { builder.Add(new OneFileActionProvider( - _fileGateway, + localFileGateway, action.FilePath, action.UsePluralForms)); } if (action.FolderPath != null) { builder.Add(new MultiFilesActionProvider( - _fileGateway, + localFileGateway, action.FolderPath, action.UsePluralForms)); } @@ -283,7 +282,8 @@ private IImmutableList CreateActionProvider( private IDatabaseProvider CreateDatabaseProvider( SourceParameterization? source, - ITokenProvider? tokenProvider) + ITokenProvider? tokenProvider, + IFileGateway localFileGateway) { if (source == null) { @@ -316,13 +316,16 @@ private IDatabaseProvider CreateDatabaseProvider( { _tracer.WriteLine(true, "Database scripts"); - return new ScriptDatabaseProvider(_tracer, _fileGateway, source.Scripts); + return new ScriptDatabaseProvider(_tracer, localFileGateway, source.Scripts); } else if (source.JsonFilePath != null) { _tracer.WriteLine(true, "Json file"); - return new JsonDatabaseProvider(_tracer, _fileGateway, source.JsonFilePath); + return new JsonDatabaseProvider( + _tracer, + localFileGateway, + source.JsonFilePath); } else { diff --git a/code/delta-kusto/TimeOuts.cs b/code/delta-kusto/TimeOuts.cs deleted file mode 100644 index 97919ee3..00000000 --- a/code/delta-kusto/TimeOuts.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; - -namespace delta_kusto -{ - internal class TimeOuts - { - public static TimeSpan API = TimeSpan.FromSeconds(10); - - public static TimeSpan FILE = TimeSpan.FromSeconds(.5); - - public static TimeSpan RETRIEVE_DB = TimeSpan.FromSeconds(15); - - public static TimeSpan ACTION = TimeSpan.FromSeconds(40); - } -} \ No newline at end of file diff --git a/code/delta-kusto/delta-kusto.csproj b/code/delta-kusto/delta-kusto.csproj index 42042d70..5539615a 100644 --- a/code/delta-kusto/delta-kusto.csproj +++ b/code/delta-kusto/delta-kusto.csproj @@ -5,7 +5,7 @@ net5.0 delta_kusto enable - 0.3.0 + 0.3.1