diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 1cc84837c20e..18f07c20c500 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -33,6 +33,9 @@ jobs: steps: - template: ../steps/analyze.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Test' condition: and(succeededOrFailed(), ne(variables['Skip.Test'], 'true')) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-nightly.yml b/eng/pipelines/templates/jobs/archetype-sdk-nightly.yml index 9bcdf241c717..e848889aff29 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-nightly.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-nightly.yml @@ -31,6 +31,9 @@ jobs: steps: - template: ../steps/analyze.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Test' variables: diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index d9e243026647..1b06883bce50 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -1,3 +1,7 @@ +parameters: + BuildTargetingString: 'azure-*' + ServiceDirectory: '' + steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -5,7 +9,7 @@ steps: versionSpec: '$(PythonVersion)' - script: | - pip install setuptools wheel Jinja2 + pip install setuptools wheel Jinja2 packaging pip install doc-warden==0.3.0 ward scan -d $(Build.SourcesDirectory) -c $(Build.SourcesDirectory)/.docsettings.yml displayName: 'Verify Readmes' @@ -16,6 +20,12 @@ steps: scriptPath: 'scripts/analyze_deps.py' arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html"' + - task: PythonScript@0 + displayName: 'Verify Change Log' + inputs: + scriptPath: 'scripts/devops_tasks/verify_change_log.py' + arguments: '"${{ parameters.BuildTargetingString }}" --service=${{parameters.ServiceDirectory}}' + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR # builds should be sufficient. diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 2cf43dac9114..9d6f2c9fc8f9 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -76,7 +76,8 @@ def clean_coverage(coverage_dir): else: raise -def parse_setup_requires(setup_path): + +def parse_setup(setup_path): setup_filename = os.path.join(setup_path, 'setup.py') mock_setup = textwrap.dedent('''\ def setup(*args, **kwargs): @@ -113,8 +114,18 @@ def setup(*args, **kwargs): except KeyError as e: python_requires = ">=2.7" + version = kwargs['version'] + name = kwargs['name'] + + return name, version, python_requires + + +def parse_setup_requires(setup_path): + _, _, python_requires = parse_setup(setup_path) + return python_requires + def filter_for_compatibility(package_set): collected_packages = [] v = sys.version_info diff --git a/scripts/devops_tasks/find_change_log.ps1 b/scripts/devops_tasks/find_change_log.ps1 new file mode 100644 index 000000000000..f83961b1c06b --- /dev/null +++ b/scripts/devops_tasks/find_change_log.ps1 @@ -0,0 +1,105 @@ + +param ( + $workingDir, + $version +) +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?))" + + +function ExtractReleaseNotes($changeLogLocation) +{ + $releaseNotes = @{} + $contentArrays = @{} + if ($changeLogLocation.Length -eq 0) + { + return $releaseNotes + } + + try { + $contents = Get-Content $changeLogLocation + + # walk the document, finding where the version specifiers are and creating lists + $version = "" + Write-Host "Versions in change log $changeLogLocation" + foreach($line in $contents){ + if ($line -match $RELEASE_TITLE_REGEX) + { + Write-Host $line + $version = $matches["version"] + $contentArrays[$version] = @() + } + + $contentArrays[$version] += $line + } + + # resolve each of discovered version specifier string arrays into real content + foreach($key in $contentArrays.Keys) + { + $releaseNotes[$key] = New-Object PSObject -Property @{ + ReleaseVersion = $key + ReleaseContent = $contentArrays[$key] -join [Environment]::NewLine + } + } + } + catch + { + Write-Host "Error parsing $changeLogLocation." + Write-Host $_.Exception.Message + } + + return $releaseNotes +} + + +function VerifyPackages($rootDirectory) +{ + #This function verifies version in history.md for a given package + try + { + $historyFiles = Get-ChildItem -Path $rootDirectory -Recurse -Include "history.md" + if ($historyFiles -eq $null) + { + exit(0) + } + + #Find current version of package from _version.py and package name from setup.py + $changeFile = @($historyFiles)[0] + #Get Version and release notes in each change log files + $releaseNotes = ExtractReleaseNotes -changeLogLocation $changeFile + if ($releaseNotes.Count -gt 0) + { + #Log package if it doesn't have current version in history.md + if ( $releaseNotes.Contains($version)) + { + $content = $releaseNotes[$version] + Write-Host "Change log [$changeFile] is updated with current version $version" + Write-Host "Release notes for version $version" + Write-Host "****************************************************************************************************" + Write-Host $content.ReleaseContent + Write-Host "****************************************************************************************************" + } + else + { + Write-Host "Change log [$changeFile] does not have current version $version" + exit(1) + } + } + } + catch + { + Write-Host "Error verifying version in change log" + Write-Host $_.Exception.Message + exit(1) + } + +} + + +if (($workingDir -eq $null) -or ($version -eq $null)) +{ + Write-Host "Invalid arguements. workingDir and version are mandatory arguements" + exit(1) +} + +VerifyPackages -rootDirectory $workingDir + diff --git a/scripts/devops_tasks/verify_change_log.py b/scripts/devops_tasks/verify_change_log.py new file mode 100644 index 000000000000..84a03a2bcf5f --- /dev/null +++ b/scripts/devops_tasks/verify_change_log.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# Normally, this module will be executed as referenced as part of the devops build definitions. +# An enterprising user can easily glance over this and leverage for their own purposes. + +import argparse +import sys +import os +import logging + +from common_tasks import process_glob_string, parse_setup, run_check_call + +excluded_packages = ["azure"] + +logging.getLogger().setLevel(logging.INFO) + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +psscript = os.path.join(root_dir, "scripts", "devops_tasks", "find_change_log.ps1") + + +def find_change_log(targeted_package, version): + # Execute powershell script to find a matching version in history.md + command_array = ["pwsh"] + command_array.append("-File {}".format(psscript)) + command_array.append("-workingDir {}".format(targeted_package)) + command_array.append("-version {}".format(version)) + command_array.append("set-ExecutionPolicy Unrestricted") + + allowed_return_codes = [] + + # Execute powershell script to verify version + er_result = run_check_call( + command_array, root_dir, allowed_return_codes, True, False + ) + + if er_result: + logging.error( + "Failed to find version in change log for package {}".format( + targeted_package + ) + ) + return False + + return True + + +def verify_packages(targeted_packages): + # run the build and distribution + change_log_missing = {} + + for package in targeted_packages: + # Parse setup.py using common helper method to get version and package name + pkg_name, version, _ = parse_setup(package) + + # Skip management packages + if "-mgmt" in pkg_name or pkg_name in excluded_packages: + continue + + if not find_change_log(package, version): + logging.error( + "Change log is not updated for package {0}, version {1}".format( + pkg_name, version + ) + ) + change_log_missing[pkg_name] = version + + return change_log_missing + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Verifies latest version is updated in history.md file, Called from DevOps YAML Pipeline" + ) + + parser.add_argument( + "glob_string", + nargs="?", + help=( + "A comma separated list of glob strings that will target the top level directories that contain packages. " + 'Examples: All == "azure-*", Single = "azure-keyvault"' + ), + ) + + parser.add_argument( + "--service", + help=( + "Name of service directory (under sdk/) to build." + "Example: --service applicationinsights" + ), + ) + + parser.add_argument( + "--pkgfilter", + default="", + dest="package_filter_string", + help=( + "An additional string used to filter the set of artifacts by a simple CONTAINS clause." + ), + ) + + args = parser.parse_args() + + # We need to support both CI builds of everything and individual service + # folders. This logic allows us to do both. + if args.service: + service_dir = os.path.join("sdk", args.service) + target_dir = os.path.join(root_dir, service_dir) + else: + target_dir = root_dir + + targeted_packages = process_glob_string( + args.glob_string, target_dir, args.package_filter_string + ) + change_missing = verify_packages(targeted_packages) + if len(change_missing) > 0: + logging.error("Below packages do not have change log in history.md") + logging.error("***************************************************") + for pkg_name in change_missing.keys(): + logging.error("{0} - {1}".format(pkg_name, change_missing[pkg_name])) + + sys.exit(1) diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 8b52ac11404e..b90c3caa146b 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -6,4 +6,4 @@ typing_extensions>=3.7.2 opencensus>=0.6.0 opencensus-ext-azure>=0.3.1 opencensus-ext-threading -mock +mock \ No newline at end of file