From 400c259b0af2bcad56af9d028aff526bc40a33fe Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 20 Dec 2019 11:34:53 -0800 Subject: [PATCH 1/2] Changes for unreleased dependencies --- eng/versioning/pom_file_version_scanner.ps1 | 61 ++++++++++------- eng/versioning/set_versions.py | 74 ++++++++++++++------- eng/versioning/utils.py | 21 ++++-- eng/versioning/version_client.txt | 14 ++-- sdk/core/azure-core-amqp/pom.xml | 2 +- 5 files changed, 110 insertions(+), 62 deletions(-) diff --git a/eng/versioning/pom_file_version_scanner.ps1 b/eng/versioning/pom_file_version_scanner.ps1 index 353bb71e508dc..35804f8bc8d01 100644 --- a/eng/versioning/pom_file_version_scanner.ps1 +++ b/eng/versioning/pom_file_version_scanner.ps1 @@ -39,6 +39,11 @@ $DependencyTypeForError = "$($DependencyTypeCurrent)|$($DependencyTypeDependency $UpdateTagFormat = "{x-version-update;:;$($DependencyTypeForError)}" $StartTime = $(get-date) +function Write-Error-With-Color([string]$msg) +{ + Write-Host "$($msg)" -ForegroundColor Red +} + # The expected format for a depenency, as found in the eng\versioning\version_*.txt files, is as follows: # groupId:artifactId;dependency-version;current-version class Dependency { @@ -49,7 +54,7 @@ class Dependency { [string]$inputString ){ $split = $inputString.Split(";") - if ($split.Count -ne 3) + if (($split.Count -ne 3) -and ($split.Count -ne 2)) { # throw and let the caller handle the error since it'll have access to the # filename of the file with the malformed line for reporting @@ -57,7 +62,10 @@ class Dependency { } $this.id = $split[0] $this.depVer = $split[1] - $this.curVer = $split[2] + if ($split.Count -eq 3) + { + $this.curVer = $split[2] + } } } @@ -188,6 +196,11 @@ function Test-Dependency-Tag-And-Version { } elseif ($depType -eq $DependencyTypeCurrent) { + # Verify that none of the 'current' dependencies are using a groupId that starts with 'unreleased_' + if ($depKey.StartsWith('unreleased_')) + { + return "Error: $($versionUpdateString) is using an unreleased_ dependency and trying to set current value. Only dependency versions can be set with an unreleased dependency." + } if ($versionString -ne $libHash[$depKey].curVer) { return "Error: $($depKey)'s is '$($versionString)' but the current version is listed as $($libHash[$depKey].curVer)" @@ -239,11 +252,11 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { return } } - Write-Output "processing pomFile=$($pomFile)" + Write-Host "processing pomFile=$($pomFile)" $dependencyManagement = $xmlPomFile.GetElementsByTagName("dependencyManagement")[0] if ($dependencyManagement) { - Write-Output "Error: is not allowed. Every dependency must have its own version and version update tag" + Write-Error-With-Color "Error: is not allowed. Every dependency must have its own version and version update tag" } # Ensure that the project has a version tag with the exception of projects under the eng directory which @@ -266,7 +279,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { { $script:FoundError = $true # every project string needs to have an update tag and projects version tags are always 'current' - Write-Output "Error: project/version update tag should be " + Write-Error-With-Color "Error: project/version update tag should be " } else { @@ -275,7 +288,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($retVal) { $script:FoundError = $true - Write-Output "$($retVal)" + Write-Host "$($retVal)" } } } @@ -284,7 +297,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { $script:FoundError = $true # # every project string needs to have an update tag and projects version tags are always 'current' - Write-Output "Error: Missing project/version update tag. The tag should be " + Write-Error-With-Color "Error: Missing project/version update tag. The tag should be " } } @@ -292,7 +305,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { { # output an error for missing version element $script:FoundError = $true - Write-Output "Error: Could not find project/version node for $($pomFile)" + Write-Error-With-Color "Error: Could not find project/version node for $($pomFile)" } } @@ -310,7 +323,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($versionNode.NextSibling.Value.Trim() -ne "{x-version-update;$($groupId):$($artifactId);current}") { $script:FoundError = $true - Write-Output "Error: project/parent/version update tag should be " + Write-Error-With-Color "Error: project/parent/version update tag should be " } else { @@ -319,7 +332,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($retVal) { $script:FoundError = $true - Write-Output "$($retVal)" + Write-Host "$($retVal)" } } } @@ -327,14 +340,14 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { { $script:FoundError = $true # every project string needs to have an update tag and projects version tags are always 'current' - Write-Output "Error: Missing project/parent/version update tag. The tag should be " + Write-Error-With-Color "Error: Missing project/parent/version update tag. The tag should be " } } else { # output an error for missing version element $script:FoundError = $true - Write-Output "Error: Could not find project/parent/version node for $($pomFile)" + Write-Error-With-Color "Error: Could not find project/parent/version node for $($pomFile)" } } @@ -348,7 +361,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if (!$versionNode) { $script:FoundError = $true - Write-Output "Error: dependency is missing version element for groupId=$($groupId), artifactId=$($artifactId) should be " + Write-Error-With-Color "Error: dependency is missing version element for groupId=$($groupId), artifactId=$($artifactId) should be " continue } if ($versionNode.NextSibling -and $versionNode.NextSibling.NodeType -eq "Comment") @@ -358,7 +371,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($versionNode.NextSibling.Value.Trim() -notmatch "{x-version-update;(\w+)?$($groupId):$($artifactId);\w+}") { $script:FoundError = $true - Write-Output "Error: dependency version update tag for groupId=$($groupId), artifactId=$($artifactId) should be " + Write-Error-With-Color "Error: dependency version update tag for groupId=$($groupId), artifactId=$($artifactId) should be " } else { @@ -367,14 +380,14 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($retVal) { $script:FoundError = $true - Write-Output "$($retVal)" + Write-Error-With-Color $retVal } } } else { $script:FoundError = $true - Write-Output "Error: Missing dependency version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be " + Write-Error-With-Color "Error: Missing dependency version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be " } } # Verify every plugin has a group, artifact and version @@ -388,14 +401,14 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if (!$groupId) { $script:FoundError = $true - Write-Output "Error: plugin $($artifactId) is missing its groupId tag" + Write-Error-With-Color "Error: plugin $($artifactId) is missing its groupId tag" continue } $versionNode = $pluginNode.GetElementsByTagName("version")[0] if (!$versionNode) { $script:FoundError = $true - Write-Output "Error: plugin is missing version element for groupId=$($groupId), artifactId=$($artifactId) should be " + Write-Error-With-Color "Error: plugin is missing version element for groupId=$($groupId), artifactId=$($artifactId) should be " continue } if ($versionNode.NextSibling -and $versionNode.NextSibling.NodeType -eq "Comment") @@ -405,7 +418,7 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($versionNode.NextSibling.Value.Trim() -notmatch "{x-version-update;(\w+)?$($groupId):$($artifactId);\w+}") { $script:FoundError = $true - Write-Output "Error: plugin version update tag for groupId=$($groupId), artifactId=$($artifactId) should be " + Write-Error-With-Color "Error: plugin version update tag for groupId=$($groupId), artifactId=$($artifactId) should be " } else { @@ -414,25 +427,25 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object { if ($retVal) { $script:FoundError = $true - Write-Output "$($retVal)" + Write-Host "$($retVal)" } } } else { $script:FoundError = $true - Write-Output "Error: Missing plugin version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be " + Write-Error-With-Color "Error: Missing plugin version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be " } } } $ElapsedTime = $(get-date) - $StartTime $TotalRunTime = "{0:HH:mm:ss}" -f ([datetime]$ElapsedTime.Ticks) -Write-Output "Total run time=$($TotalRunTime)" +Write-Host "Total run time=$($TotalRunTime)" if ($script:FoundError) { - Write-Output "There were errors encountered during execution. Please fix any errors and run the script again." - Write-Output "This script can be run locally from the root of the repo. .\eng\pom_file_version_scanner.ps1" + Write-Host "There were errors encountered during execution. Please fix any errors and run the script again." + Write-Host "This script can be run locally from the root of the repo. .\eng\pom_file_version_scanner.ps1" exit(1) } diff --git a/eng/versioning/set_versions.py b/eng/versioning/set_versions.py index af1f0c5c8b937..710cfef47172f 100644 --- a/eng/versioning/set_versions.py +++ b/eng/versioning/set_versions.py @@ -14,19 +14,19 @@ # and the current versions. # # -# python utilities/set_versions.py --ut [library|external_dependency|all] --bt [client|data|management] --bq --ar +# python eng\versioning\set_versions.py --ut [library|external_dependency|all] --bt [client|data|management] --bq --ar # # Use case: increment the version of a given artifact in the approprate version_[client|data|management].txt file # -# python eng/versioning/set_versions.py --bt [client|data|management] --increment-version --artifact-id +# python eng\versioning\set_versions.py --bt [client|data|management] --increment-version --artifact-id # For example: To update increment the version of azure-core -# python eng/versioning/update_versions.py --bt client --iv -ar azure-core +# python eng\versioning\set_versions.py --bt client --iv --ar azure-core # # Use case: verify the version of a given artifact in the approprate version_[client|data|management].txt file # -# python eng/versioning/set_versions.py --bt [client|data|management] --verify-version --artifact-id +# python eng\versioning\set_versions.py --bt [client|data|management] --verify-version --artifact-id # For example: To update increment the version of azure-core -# python eng/versioning/update_versions.py --bt client --vv -ar azure-core +# python eng\versioning\set_versions.py --bt client --vv --ar azure-core # # The script must be run at the root of azure-sdk-for-java. @@ -42,6 +42,9 @@ from utils import version_regex_str_with_names_anchored from utils import prerelease_version_regex_with_name +# some things that should not be updated for devops builds, in the case where everything is being updated in one call +items_we_should_not_update = ['com.azure:azure-sdk-all', 'com.azure:azure-sdk-parent', 'com.azure:azure-client-sdk-parent', 'azure-data-sdk-parent'] + # The regex string we want should be the anchored one since the entire string is what's being matched version_regex_named = re.compile(version_regex_str_with_names_anchored) prerelease_regex_named = re.compile(prerelease_version_regex_with_name) @@ -50,7 +53,7 @@ def update_versions_file_for_nightly_devops(update_type, build_type, build_quali version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt') print('version_file=' + version_file) - + version_map = {} newlines = [] with open(version_file, encoding='utf-8') as f: for raw_line in f: @@ -59,23 +62,44 @@ def update_versions_file_for_nightly_devops(update_type, build_type, build_quali newlines.append(raw_line) else: module = CodeModule(stripped_line) + # basically we don't want to change any of the parent versions here, only + # library versions + if not artifact_id and module.name in items_we_should_not_update: + newlines.append(module.string_for_version_file()) + continue if not artifact_id or module.artifact_id == artifact_id: - if update_type == UpdateType.library or update_type == UpdateType.all: - if '-' in module.current: - module.current += "." + build_qualifier - else: - module.current += '-' + build_qualifier - match = version_regex_named.match(module.current) - if not match: - raise ValueError('{}\'s current version + build qualifier {} is not a valid semver version'.format(module.name, module.current + build_qualifier)) - if update_type == UpdateType.external_dependency or update_type == UpdateType.all: - if '-' in module.dependency: - module.dependency += "." + build_qualifier + if (update_type == UpdateType.library or update_type == UpdateType.all): + if hasattr(module, 'current'): + set_both = False + # In the case where the current and dependency are both equal then both + # need to be updated. In theory, this should only really happen when a + # new library has been added and has not yet been released but can also + # happen if a library has just been released and the devops build kicks + # of before the human submits the PR to increase the current version. Being + # that it's a devops build and both versions are being updated this should + # be OK. + if module.current == module.dependency: + set_both = True + if '-' in module.current: + module.current += "." + build_qualifier + else: + module.current += '-' + build_qualifier + match = version_regex_named.match(module.current) + if not match: + raise ValueError('{}\'s current version + build qualifier {} is not a valid semver version'.format(module.name, module.current + build_qualifier)) + if set_both: + module.dependency = module.current + # we need to handle the unreleased dependency which should have the same version as the "current" dependency + # of the non-unreleased artifact else: - module.dependency += '-' + build_qualifier - match = version_regex_named.match(module.dependency) - if not match: - raise ValueError('{}\'s dependency version + build qualifier {} is not a valid semver version'.format(module.name, module.dependency + build_qualifier)) + if (module.name.startswith('unreleased_')): + # strip off the 'unreleased_' prepend to get the regular module entry + real_name = module.name.replace('unreleased_', '') + real_module = version_map[real_name] + module.dependency = real_module.current + else: + raise ValueError('{}\' does not have a current dependency and its groupId does not the required unreleased_ prepend'.format(module.name)) + version_map[module.name] = module newlines.append(module.string_for_version_file()) with open(version_file, 'w', encoding='utf-8') as f: @@ -97,7 +121,7 @@ def prep_version_file_for_source_testing(build_type): newlines.append(raw_line) else: module = CodeModule(stripped_line) - if not module.current == module.dependency: + if hasattr(module, 'current') and not module.current == module.dependency: module.dependency = module.current file_changed = True newlines.append(module.string_for_version_file()) @@ -133,7 +157,7 @@ def increment_version_for_artifact(build_type, artifact_id): # version then just increment the revision. Otherwise increment the # minor version, zero the patch and add "-beta.1" to the end # https://github.com/Azure/azure-sdk/blob/master/docs/policies/releases.md#java - if module.artifact_id == artifact_id: + if module.artifact_id == artifact_id and hasattr(module, 'current'): artifact_found = True vmatch = version_regex_named.match(module.current) if (vmatch.group('prerelease') is not None): @@ -181,7 +205,7 @@ def verify_current_version_of_artifact(build_type, artifact_id): # of the following: # .. # ..-beta. - if module.artifact_id == artifact_id: + if module.artifact_id == artifact_id and hasattr(module, 'current'): artifact_found = True vmatch = version_regex_named.match(module.current) temp_ver = '{}.{}.{}'.format(vmatch.group('major'), vmatch.group('minor'), vmatch.group('patch')) @@ -207,7 +231,7 @@ def verify_current_version_of_artifact(build_type, artifact_id): if module.current != temp_ver: raise ValueError('artifact ({}) version ({}) in version file ({}) does not match the version constructed from the semver pieces ({})'.format(artifact_id, module.current, version_file, temp_ver)) - print('The version {} for artifact_id {} looks good!'.format(module.name, module.current)) + print('The version {} for {} looks good!'.format(module.current, module.name)) if not artifact_found: diff --git a/eng/versioning/utils.py b/eng/versioning/utils.py index d944275a97a34..8928047f80552 100644 --- a/eng/versioning/utils.py +++ b/eng/versioning/utils.py @@ -65,10 +65,16 @@ def __init__(self, module_str): self.group_id = items[0].split(':')[0] self.artifact_id = items[0].split(':')[1] - if len(items) == 2: - self.external_dependency = items[1].strip() - self.update_type = UpdateType.external_dependency + if len(items) == 2: + if self.group_id.startswith('unreleased_'): + self.dependency = items[1].strip() + self.update_type = UpdateType.library + else: + self.external_dependency = items[1].strip() + self.update_type = UpdateType.external_dependency elif len(items) == 3: + if self.group_id.startswith('unreleased_'): + raise ValueError('Unreleased dependency entries should not have a current version, they should only a dependency version') self.dependency = items[1] self.current = items[2].strip() self.update_type = UpdateType.library @@ -78,11 +84,16 @@ def __init__(self, module_str): # overridden string primarily used for error reporting def __str__(self): # current may or may not exist + if hasattr(self, 'external_dependency'): + return self.name + ': External Dependency version=' + self.external_dependency try: return self.name + ': Dependency version=' + self.dependency + ': Current version=' + self.current except AttributeError: - return self.name + ': External Dependency version=' + self.external_dependency + return self.name + ': Dependency version=' + self.dependency # return the CodeModule string formatted for a version file def string_for_version_file(self): - return self.name + ';' + self.dependency + ';' + self.current + '\n' + try: + return self.name + ';' + self.dependency + ';' + self.current + '\n' + except AttributeError: + return self.name + ';' + self.dependency + '\n' diff --git a/eng/versioning/version_client.txt b/eng/versioning/version_client.txt index f7dc914cb0264..aa4b7138b087d 100644 --- a/eng/versioning/version_client.txt +++ b/eng/versioning/version_client.txt @@ -1,5 +1,5 @@ # Format; -# groupId:artifactId;dependency-version;current-version +# :;dependency-version;current-version com.azure:azure-sdk-all;1.0.0;1.0.0 com.azure:azure-sdk-parent;1.6.0;1.6.0 @@ -30,9 +30,9 @@ com.azure:azure-storage-file-share;12.1.0-beta.1;12.1.0-beta.2 com.azure:azure-storage-file-datalake;12.0.0-beta.8;12.0.0-beta.9 com.azure:azure-storage-queue;12.2.0-beta.1;12.2.0-beta.2 -# This is a temporary fix for libaries that need the unreleased dependency version of core. It's worth -# noting that the 'current' version isn't used, these entries are for the dependency versions -unreleased_com.azure:azure-core;1.2.0-beta.1;1.2.0-beta.1 - -# This is a temporary fix for libaries that need the unreleased dependency version of azure-core-test -unreleased_com.azure:azure-core-test;1.1.0-beta.2;1.1.0-beta.2 +# Unreleased dependencies: Copy the entry from above, prepent "unreleased_" and remove the current +# version. Unreleased dependencies are only valid for dependency versions +# Format; +# unreleased_:;dependency-version +unreleased_com.azure:azure-core;1.2.0-beta.1 +unreleased_com.azure:azure-core-test;1.1.0-beta.2 diff --git a/sdk/core/azure-core-amqp/pom.xml b/sdk/core/azure-core-amqp/pom.xml index b5432a7c16b43..696aa3bbd3af5 100644 --- a/sdk/core/azure-core-amqp/pom.xml +++ b/sdk/core/azure-core-amqp/pom.xml @@ -57,7 +57,7 @@ com.azure azure-core - 1.2.0-beta.1 + 1.2.0-beta.1 com.microsoft.azure From 622324f6715e123c67abf683cd764a083f7c26b2 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 20 Dec 2019 14:19:48 -0800 Subject: [PATCH 2/2] Forward slashes in the use case commands in set_version --- eng/versioning/set_versions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/versioning/set_versions.py b/eng/versioning/set_versions.py index 710cfef47172f..37e1abca5fc06 100644 --- a/eng/versioning/set_versions.py +++ b/eng/versioning/set_versions.py @@ -14,19 +14,19 @@ # and the current versions. # # -# python eng\versioning\set_versions.py --ut [library|external_dependency|all] --bt [client|data|management] --bq --ar +# python eng/versioning/set_versions.py --ut [library|external_dependency|all] --bt [client|data|management] --bq --ar # # Use case: increment the version of a given artifact in the approprate version_[client|data|management].txt file # -# python eng\versioning\set_versions.py --bt [client|data|management] --increment-version --artifact-id +# python eng/versioning/set_versions.py --bt [client|data|management] --increment-version --artifact-id # For example: To update increment the version of azure-core -# python eng\versioning\set_versions.py --bt client --iv --ar azure-core +# python eng/versioning/set_versions.py --bt client --iv --ar azure-core # # Use case: verify the version of a given artifact in the approprate version_[client|data|management].txt file # -# python eng\versioning\set_versions.py --bt [client|data|management] --verify-version --artifact-id +# python eng/versioning/set_versions.py --bt [client|data|management] --verify-version --artifact-id # For example: To update increment the version of azure-core -# python eng\versioning\set_versions.py --bt client --vv --ar azure-core +# python eng/versioning/set_versions.py --bt client --vv --ar azure-core # # The script must be run at the root of azure-sdk-for-java.