Skip to content

Commit

Permalink
Prevent duplicate tag push (#5748)
Browse files Browse the repository at this point in the history 
* we use the generated SHA of the assets repo within the tag name that is uploaded. This means that if a set of recordings has already been pushed, we won't attempt to push again and error out erroneously

* our tests were not cleaning up after themselves appropriately, update the integrationpushtests to actually clean up _all_ of their created tags.

---------

Co-authored-by: Ben Broderick Phillips <[email protected]>
  • Loading branch information
@scbedd @benbp
scbedd and benbp authored Apr 17, 2023
1 parent 188585c commit 7620a2d
Show file tree
Hide file tree
Showing 8 changed files with 157 additions and 57 deletions.
47 changes: 29 additions & 18 deletions ...st-proxy/Azure.Sdk.Tools.TestProxy.Tests/IntegrationTests/GitStoreIntegrationPushTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public GitStoreIntegrationPushTests()
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""language/tables"",
""Tag"": ""language/tables_fc54d0""
""Tag"": ""python/tables_fc54d0""
}")]
[Trait("Category", "Integration")]
public async Task Scenario1(string inputJson)
Expand All @@ -59,8 +59,8 @@ public async Task Scenario1(string inputJson)
var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest:true);
try
{
// Ensure that the TagPrefix was updated
Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
// Ensure that the Tag was updated
Assert.NotEqual(originalTag, assets.TagPrefix);

var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);

Expand Down Expand Up @@ -105,6 +105,7 @@ public async Task Scenario1(string inputJson)
finally
{
DirectoryHelper.DeleteGitDirectory(testFolder);
TestHelpers.CleanupIntegrationTestTag(assets);
TestHelpers.CleanupIntegrationTestTag(updatedAssets);
}
}
Expand Down Expand Up @@ -141,8 +142,8 @@ public async Task Scenario2(string inputJson)
var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
try
{
// Ensure that the TagPrefix was updated
Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
// Ensure that the Tag was updated
Assert.NotEqual(originalTag, assets.Tag);

var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);

Expand Down Expand Up @@ -187,6 +188,7 @@ public async Task Scenario2(string inputJson)
finally
{
DirectoryHelper.DeleteGitDirectory(testFolder);
TestHelpers.CleanupIntegrationTestTag(assets);
TestHelpers.CleanupIntegrationTestTag(updatedAssets);
}
}
Expand All @@ -207,7 +209,7 @@ public async Task Scenario2(string inputJson)
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""language/tables"",
""Tag"": ""language/tables_9e81fb""
""Tag"": ""python/tables_9e81fb""
}")]
[Trait("Category", "Integration")]
public async Task Scenario3(string inputJson)
Expand All @@ -223,8 +225,8 @@ public async Task Scenario3(string inputJson)
var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
try
{
// Ensure that the TagPrefix was updated
Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
// Ensure that the Tag was updated
Assert.NotEqual(originalTag, assets.Tag);

var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);

Expand Down Expand Up @@ -280,6 +282,7 @@ public async Task Scenario3(string inputJson)
finally
{
DirectoryHelper.DeleteGitDirectory(testFolder);
TestHelpers.CleanupIntegrationTestTag(assets);
TestHelpers.CleanupIntegrationTestTag(updatedAssets);
}
}
Expand Down Expand Up @@ -316,8 +319,8 @@ public async Task Scenario4(string inputJson)
var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
try
{
// Ensure that the TagPrefix was updated
Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
// Ensure that the Tag was updated
Assert.NotEqual(originalTag, assets.Tag);

var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);

Expand Down Expand Up @@ -362,6 +365,7 @@ public async Task Scenario4(string inputJson)
finally
{
DirectoryHelper.DeleteGitDirectory(testFolder);
TestHelpers.CleanupIntegrationTestTag(assets);
TestHelpers.CleanupIntegrationTestTag(updatedAssets);
}
}
Expand All @@ -384,7 +388,7 @@ public async Task Scenario4(string inputJson)
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""language/tables"",
""Tag"": ""language/tables_9e81fb""
""Tag"": ""python/tables_9e81fb""
}")]
[Trait("Category", "Integration")]
public async Task Scenario5(string inputJson)
Expand All @@ -401,14 +405,15 @@ public async Task Scenario5(string inputJson)
// The first restore needs to be done with isPushTest set to true so it creates the branch
var testFolder = TestHelpers.DescribeTestFolder(assets, folderStructure, isPushTest: true);
// The second restore needs to use the assets that was updated in the first restore so it
// restores from the branch we're going to push to
// restores from the tag we're going to push to
var testFolder2 = TestHelpers.DescribeTestFolder(assets, folderStructure);
try
{
// Ensure that the TagPrefix was updated
Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
// Ensure that the Tag was updated
Assert.NotEqual(originalTag, assets.Tag);

var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);
assets = TestHelpers.LoadAssetsFromFile(jsonFileLocation);

var parsedConfiguration = await _defaultStore.ParseConfigurationFile(jsonFileLocation);
await _defaultStore.Restore(jsonFileLocation);
Expand Down Expand Up @@ -474,22 +479,27 @@ public async Task Scenario5(string inputJson)

// Ensure that the targeted tag is present on the repo
TestHelpers.CheckExistenceOfTag(updatedAssets, localFilePath);

// Update the second assets file and do another restore
TestHelpers.UpdateAssetsFile(updatedAssets, jsonFileLocation2);
await defaultStore2.Restore(jsonFileLocation2);
updatedAssets = TestHelpers.LoadAssetsFromFile(jsonFileLocation2);

// Verify the files pushes in another directory are restored correctly here
Assert.Equal(3, System.IO.Directory.EnumerateFiles(localFilePath2).Count());
Assert.True(TestHelpers.VerifyFileVersion(localFilePath2, "file1.txt", 2));
Assert.True(TestHelpers.VerifyFileVersion(localFilePath2, "file2.txt", 3));
Assert.True(TestHelpers.VerifyFileVersion(localFilePath2, "file6.txt", 1));
await TestHelpers.CheckBreadcrumbAgainstAssetsJsons(new string[] { jsonFileLocation });

}
catch(Exception)
{
throw;
}
finally
{
DirectoryHelper.DeleteGitDirectory(testFolder);
TestHelpers.CleanupIntegrationTestTag(assets);
TestHelpers.CleanupIntegrationTestTag(updatedAssets);
DirectoryHelper.DeleteGitDirectory(testFolder2);
}
Expand Down Expand Up @@ -530,8 +540,8 @@ public async Task LargePushPerformance(int numberOfFiles, double fileSize)

try
{
// Ensure that the TagPrefix was updated
Assert.NotEqual(originalTagPrefix, assets.TagPrefix);
// Ensure that the Tag was updated
Assert.NotEqual(originalTag, assets.Tag);

var jsonFileLocation = Path.Join(testFolder, GitStoretests.AssetsJson);

Expand Down Expand Up @@ -567,6 +577,7 @@ public async Task LargePushPerformance(int numberOfFiles, double fileSize)
finally
{
DirectoryHelper.DeleteGitDirectory(testFolder);
TestHelpers.CleanupIntegrationTestTag(assets);
TestHelpers.CleanupIntegrationTestTag(updatedAssets);
}
}
Expand Down
18 changes: 9 additions & 9 deletions ...t-proxy/Azure.Sdk.Tools.TestProxy.Tests/IntegrationTests/GitStoreIntegrationResetTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public GitStoreIntegrationResetTests()
}

// Scenario 1 - Changes to existing files only are detected and overridden with Reset response Y
// 1. Restore from Tag language/tables_fc54d0
// 1. Restore from Tag python/tables_fc54d0
// 2. Expect: 3 files with versions they were checked in with
// 3. Update one or more files, incrementing their version
// 4. Expect: files updated should be at version 2
Expand All @@ -50,7 +50,7 @@ public GitStoreIntegrationResetTests()
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""main"",
""Tag"": ""language/tables_fc54d0""
""Tag"": ""python/tables_fc54d0""
}")]
[Trait("Category", "Integration")]
public async Task Scenario1(string inputJson)
Expand Down Expand Up @@ -101,7 +101,7 @@ public async Task Scenario1(string inputJson)
}

// Scenario 2 - Changes to existing files only are detected and retained with Reset response N
// 1. Restore from Tag language/tables_fc54d0
// 1. Restore from Tag python/tables_fc54d0
// 2. Expect: 3 files with versions they were checked in with
// 3. Update one or more files, incrementing their version
// 4. Expect: files updated should be at version 2
Expand All @@ -114,7 +114,7 @@ public async Task Scenario1(string inputJson)
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""main"",
""Tag"": ""language/tables_fc54d0""
""Tag"": ""python/tables_fc54d0""
}")]
[Trait("Category", "Integration")]
public async Task Scenario2(string inputJson)
Expand Down Expand Up @@ -168,7 +168,7 @@ public async Task Scenario2(string inputJson)
}

// Scenario 3 - Restore from Tag, add and remove files, Reset response Y
// 1. Restore from Tag language/tables_9e81fb
// 1. Restore from Tag python/tables_9e81fb
// 2. Expect: 4 files with versions they were checked in with
// 3. Update add/remove files
// 4. Expect: Untouched files are the same versions as step 2, added files are version 1, removed files are gone
Expand All @@ -181,7 +181,7 @@ public async Task Scenario2(string inputJson)
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""main"",
""Tag"": ""language/tables_9e81fb""
""Tag"": ""python/tables_9e81fb""
}")]
[Trait("Category", "Integration")]
public async Task Scenario3(string inputJson)
Expand Down Expand Up @@ -244,7 +244,7 @@ public async Task Scenario3(string inputJson)
}

// Scenario 4 - Restore from Tag, add and remove files, Reset response N
// 1. Restore from Tag language/tables_9e81fb
// 1. Restore from Tag python/tables_9e81fb
// 2. Expect: 4 files with versions they were checked in with
// 3. Update add/remove files
// 4. Expect: Untouched files are the same versions as step 2, added files are version 1, removed files are gone
Expand All @@ -257,7 +257,7 @@ public async Task Scenario3(string inputJson)
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""main"",
""Tag"": ""language/tables_9e81fb""
""Tag"": ""python/tables_9e81fb""
}")]
[Trait("Category", "Integration")]
public async Task Scenario4(string inputJson)
Expand Down Expand Up @@ -319,7 +319,7 @@ public async Task Scenario4(string inputJson)
}

// Scenario 5 - Restore from Tag, add and remove files, Reset response N, then Reset response Y
// 1. Restore from Tag language/tables_9e81fb
// 1. Restore from Tag python/tables_9e81fb
// 2. Expect: 3 files with versions they were checked in with
// 3. Update add/remove files
// 4. Expect: Untouched files are the same versions as step 2, added files are version 1, removed files are gone
Expand Down
8 changes: 4 additions & 4 deletions ...proxy/Azure.Sdk.Tools.TestProxy.Tests/IntegrationTests/GitStoreIntegrationRestoreTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public GitStoreIntegrationRestoreTests()
private GitStore _defaultStore = new GitStore();

// Scenario1
// Tag language/tables_fc54d0
// Tag python/tables_fc54d0
// This was the initial push of the test files:
// Added file1.txt
// Added file2.txt
Expand All @@ -47,7 +47,7 @@ public GitStoreIntegrationRestoreTests()
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""main"",
""Tag"": ""language/tables_fc54d0""
""Tag"": ""python/tables_fc54d0""
}")]
[Trait("Category", "Integration")]
public async Task Scenario1(string inputJson)
Expand Down Expand Up @@ -85,7 +85,7 @@ public async Task Scenario1(string inputJson)


// Scenario2
// Tag language/tables_9e81fb
// Tag python/tables_9e81fb
// This was the second push of the test files.
// Unchanged file1.txt
// Updated file2.txt
Expand All @@ -102,7 +102,7 @@ public async Task Scenario1(string inputJson)
""AssetsRepoPrefixPath"": ""pull/scenarios"",
""AssetsRepoId"": """",
""TagPrefix"": ""main"",
""Tag"": ""language/tables_9e81fb""
""Tag"": ""python/tables_9e81fb""
}")]
[Trait("Category", "Integration")]
public async Task Scenario2(string inputJson)
Expand Down
25 changes: 12 additions & 13 deletions tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ public static string DescribeTestFolder(Assets assets, string[] sampleFiles, str
InitIntegrationTag(assets, adjustedAssetsRepoTag);

// set the TagPrefix to the adjusted test branch
assets.TagPrefix = adjustedAssetsRepoTag;
assets.Tag = adjustedAssetsRepoTag;
localAssetsJsonContent = JsonSerializer.Serialize(assets);
}

Expand Down Expand Up @@ -402,8 +402,8 @@ public static async Task CheckBreadcrumbAgainstAssetsJsons(IEnumerable<string> j
/// This function is only used by the Push scenarios. It'll clone the assets repository
/// </summary>
/// <param name="assets"></param>
/// <param name="adjustedAssetsRepoBranch"></param>
public static void InitIntegrationTag(Assets assets, string adjustedAssetsRepoBranch)
/// <param name="adjustedAssetsRepoTag"></param>
public static void InitIntegrationTag(Assets assets, string adjustedAssetsRepoTag)
{
// generate a test folder root
string tmpPath = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString());
Expand All @@ -419,26 +419,25 @@ public static void InitIntegrationTag(Assets assets, string adjustedAssetsRepoBr
var gitCloneUrl = GitStore.GetCloneUrl(assets.AssetsRepo, tmpPath);

// Clone the original assets repo
GitHandler.Run($"clone {gitCloneUrl} .", tmpPath);
GitHandler.Run($"clone --filter=blob:none {gitCloneUrl} .", tmpPath);

// Check to see if there's already a branch
CommandResult commandResult = GitHandler.Run($"ls-remote --tags {gitCloneUrl} {assets.TagPrefix}", tmpPath);
// Check to see if the tag already exists
CommandResult commandResult = GitHandler.Run($"ls-remote --tags {gitCloneUrl} {assets.Tag}", tmpPath);

// If the commandResult response is empty, there's nothing to do and we can return
if (!String.IsNullOrWhiteSpace(commandResult.StdOut))
{
// If the commandResult response is not empty, the command result will have something
// similar to the following:
// e4a4949a2b6cc2ff75afd0fe0d97cbcabf7b67b7 refs/heads/scenario_clean_push
GitHandler.Run($"checkout {assets.TagPrefix}", tmpPath);

GitHandler.Run($"checkout {assets.Tag}", tmpPath);
}

// Create the adjustedAssetsRepoBranch from the original branch. The reason being is that pushing
// Create the adjustedAssetsRepoTag from the original branch. The reason being is that pushing
// to a branch of a branch is automatic
GitHandler.Run($"tag {adjustedAssetsRepoBranch}", tmpPath);
// Push the contents of the TagPrefix into the adjustedAssetsRepoBranch
GitHandler.Run($"push origin {adjustedAssetsRepoBranch}", tmpPath);
GitHandler.Run($"tag {adjustedAssetsRepoTag}", tmpPath);
// Push the contents of the TagPrefix into the adjustedAssetsRepoTag
GitHandler.Run($"push origin {adjustedAssetsRepoTag}", tmpPath);
}
finally
{
Expand Down Expand Up @@ -473,7 +472,7 @@ public static void CleanupIntegrationTestTag(Assets assets)
Directory.CreateDirectory(tmpPath);
GitProcessHandler GitHandler = new GitProcessHandler();
string gitCloneUrl = GitStore.GetCloneUrl(assets.AssetsRepo, Directory.GetCurrentDirectory());
GitHandler.Run($"clone {gitCloneUrl} .", tmpPath);
GitHandler.Run($"clone --filter=blob:none {gitCloneUrl} .", tmpPath);
GitHandler.Run($"push origin --delete {assets.Tag}", tmpPath);
}
finally
Expand Down
4 changes: 2 additions & 2 deletions tools/test-proxy/Azure.Sdk.Tools.TestProxy/RecordingHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,10 +564,10 @@ public static string GetAssetsJsonLocation(string pathToAssetsJson, string conte

if (!Path.IsPathFullyQualified(pathToAssetsJson))
{
path = Path.Join(contextDirectory, pathToAssetsJson.Replace("\\", "/"));
path = Path.Join(contextDirectory, pathToAssetsJson);
}

return path;
return path.Replace("\\", "/");
}

public async Task Restore(string pathToAssetsJson)
Expand Down
12 changes: 11 additions & 1 deletion tools/test-proxy/Azure.Sdk.Tools.TestProxy/Store/GitStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,17 @@ public async Task Push(string pathToAssetsJson) {
}

GitHandler.Run($"tag {generatedTagName}", config);
GitHandler.Run($"push origin {generatedTagName}", config);

var remoteResult = GitHandler.Run($"ls-remote origin --tags {generatedTagName}", config);

if (string.IsNullOrWhiteSpace(remoteResult.StdOut))
{
GitHandler.Run($"push origin {generatedTagName}", config);
}
else
{
_consoleWrapper.WriteLine($"Not attempting to push tag '{generatedTagName}', as it already exists within the assets repo");
}
}
catch(GitProcessException e)
{
Expand Down
Loading

0 comments on commit 7620a2d

Please sign in to comment.