From 613697e5b43104d6316d909759e08fe17745b04b Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 21 Feb 2019 12:29:43 -0800 Subject: [PATCH 1/5] Update Java build pipeline yaml files to match guidelines (#2957) - Clean-up variable names based on new guidelines - Publish shipping assets alone into a "packages" artifact - Explicitly call PUblishTestResults to workaround https://github.com/Microsoft/azure-pipelines-tasks/issues/9651 - Turn off maven.wagon.http.pool to help with package download issues. --- .azure-pipelines/client.test.live.yml | 57 ++++++-------- .azure-pipelines/client.yml | 108 ++++++++++++++------------ 2 files changed, 82 insertions(+), 83 deletions(-) diff --git a/.azure-pipelines/client.test.live.yml b/.azure-pipelines/client.test.live.yml index 4367d626bb0ea..f165948db356c 100644 --- a/.azure-pipelines/client.test.live.yml +++ b/.azure-pipelines/client.test.live.yml @@ -1,21 +1,4 @@ -variables: - - group: KeyVault Test Secrets - - name: ARM_CLIENTID - value: $(java-keyvault-test-arm-client-id) - - name: ARM_CLIENTKEY - value: $(java-keyvault-test-arm-client-key) - - name: AZURE_BATCH_ACCOUNT - value: $(java-batch-test-account-name) - - name: AZURE_BATCH_ACCESS_KEY - value: $(java-batch-test-account-access-key) - - name: STORAGE_ACCOUNT_NAME - value: $(java-batch-test-storage-account-name) - - name: STORAGE_ACCOUNT_KEY - value: $(java-batch-test-storage-account-key) - - jobs: - - job: 'LiveTest' strategy: @@ -24,29 +7,35 @@ jobs: pomFilePath: './batch/data-plane/pom.xml' AzureKeyVault: pomFilePath: './keyvault/data-plane/pom.xml' - + pool: vmImage: 'vs2017-win2016' - # Adding -Dmaven.wagon.http.pool=false to the mvn command as a workaround to prevent build failures at maven artifacts downloading stage. steps: - - bash: | - LOG_PARAMS='-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' - mvn test -Dmaven.wagon.http.pool=false -Dsurefire.rerunFailingTestsCount=3 $LOG_PARAMS -f $(pomFilePath) - displayName: 'Run Live Tests' + # Adding -Dmaven.wagon.http.pool=false to the mvn command as a workaround to prevent build failures at maven artifacts downloading stage. + - task: Maven@3 + displayName: 'Run Live tests' + inputs: + mavenPomFile: $(pomFilePath) + options: '--batch-mode -Dmaven.wagon.http.pool=false -Dsurefire.rerunFailingTestsCount=3' + mavenOptions: '-Xmx3072m -Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + javaHomeOption: 'JDKVersion' + jdkVersionOption: '1.8' + jdkArchitectureOption: 'x64' + publishJUnitResults: false + goals: 'test' env: - ARM_CLIENTID: $(ARM_CLIENTID) - ARM_CLIENTKEY: $(ARM_CLIENTKEY) + ARM_CLIENTID: $(java-keyvault-test-arm-client-id) + ARM_CLIENTKEY: $(java-keyvault-test-arm-client-key) AZURE_TEST_MODE: RECORD - AZURE_BATCH_ACCOUNT: $(AZURE_BATCH_ACCOUNT) - AZURE_BATCH_ACCESS_KEY: $(AZURE_BATCH_ACCESS_KEY) - STORAGE_ACCOUNT_KEY: $(STORAGE_ACCOUNT_KEY) - STORAGE_ACCOUNT_NAME: $(STORAGE_ACCOUNT_NAME) + AZURE_BATCH_ACCOUNT: $(java-batch-test-account-name) + AZURE_BATCH_ACCESS_KEY: $(java-batch-test-account-access-key) + STORAGE_ACCOUNT_KEY: $(java-batch-test-storage-account-key) + STORAGE_ACCOUNT_NAME: $(java-batch-test-storage-account-name) AZURE_BATCH_ENDPOINT: https://servbatch.centralus.batch.azure.com - + - task: PublishTestResults@2 + condition: succeededOrFailed() inputs: - testResultsFormat: 'JUnit' - testResultsFiles: '**/TEST-*.xml' - - + mergeTestResults: true + testRunTitle: 'Live tests for $(pomFilePath)' diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index f484468ac122e..e100cdcf8ad33 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -4,6 +4,10 @@ trigger: - master +variables: + DefaultOptions: '--batch-mode -Dmaven.wagon.http.pool=false' + LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + jobs: - job: 'Build' @@ -13,13 +17,13 @@ jobs: strategy: matrix: Java 8: - artifactDir: 'java-8' - mvnFlags: '' - javaVer: '1.8' + ArtifactName: 'packages' + ProfileFlag: '' + JavaVersion: '1.8' Java LTS: - artifactDir: 'java-lts' - mvnFlags: '-Djava-lts' - javaVer: '1.11' + ArtifactName: 'java-lts-packages' + ProfileFlag: '-Djava-lts' + JavaVersion: '1.11' steps: - task: Maven@3 @@ -27,23 +31,24 @@ jobs: inputs: mavenPomFile: $(pomFile) goals: 'javadoc:jar source:jar package' - options: '--batch-mode -Dmaven.test.skip=true -Dinclude-template $(mvnFlags)' # We include template-module so we ensure it always builds in CI - mavenOptions: '-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + options: '$(DefaultOptions) $(ProfileFlag) -Dmaven.test.skip=true -Dinclude-template' # We include template-module so we ensure it always builds in CI + mavenOptions: '$(LoggingOptions)' javaHomeOption: 'JDKVersion' - jdkVersionOption: $(javaVer) + jdkVersionOption: $(JavaVersion) jdkArchitectureOption: 'x64' publishJUnitResults: false - powershell: | # use sources files as a marker to find POM files so we can copy them to the output. The sources.jar file is generally # in a target folder next to the pom.xml file so $_.Directory.Parent should get us the folder that contains the POM. - mkdir (Join-Path $(Build.ArtifactStagingDirectory) $(artifactDir)) - dir -r -i *-sources.jar | % { copy (Join-Path $_.Directory.Parent.FullName "pom.xml") (Join-Path $(Build.ArtifactStagingDirectory) $(artifactDir) $_.Name.Replace("-sources.jar",".pom")) } - dir -r -i *.jar | % { copy $_ (Join-Path $(Build.ArtifactStagingDirectory) $(artifactDir)) } + dir -r -i *-sources.jar | % { copy (Join-Path $_.Directory.Parent.FullName "pom.xml") (Join-Path $(Build.ArtifactStagingDirectory) $_.Name.Replace("-sources.jar",".pom")) } + dir -r -i *.jar | % { copy $_ $(Build.ArtifactStagingDirectory) } displayName: 'Copy JAR and POM files to artifact staging' - task: PublishBuildArtifacts@1 displayName: 'Publish' + inputs: + artifactName: '$(ArtifactName)' - job: 'Analyze' @@ -70,8 +75,8 @@ jobs: displayName: 'Install reporting tools' inputs: mavenPomFile: $(pomFile) - options: '--batch-mode -Dinclude-non-shipping-modules -DskipTests -Dgpg.skip' - mavenOptions: '-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + options: '$(DefaultOptions) -Dinclude-non-shipping-modules -DskipTests -Dgpg.skip' + mavenOptions: '$(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: '1.11' jdkArchitectureOption: 'x64' @@ -82,8 +87,8 @@ jobs: displayName: 'Generate Maven project site, including JavaDocs, SpotBugs, and CheckStyle reports' inputs: mavenPomFile: $(pomFile) - options: '--batch-mode -DskipTests -Dgpg.skip' - mavenOptions: '-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + options: '$(DefaultOptions) -DskipTests -Dgpg.skip' + mavenOptions: '$(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: '1.11' jdkArchitectureOption: 'x64' @@ -106,48 +111,48 @@ jobs: strategy: matrix: Linux - Java 8: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - javaVer: '1.8' - mvnFlags: '' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + JavaVersion: '1.8' + ProfileFlag: '' macOS - Java 8: - os.name: 'macOS' - os.vmImage: 'macOS-10.13' - javaVer: '1.8' - mvnFlags: '' + OSName: 'macOS' + OSVmImage: 'macOS-10.13' + JavaVersion: '1.8' + ProfileFlag: '' Windows - Java 8: - os.name: 'Windows' - os.vmImage: 'vs2017-win2016' - javaVer: '1.8' - mvnFlags: '' + OSName: 'Windows' + OSVmImage: 'vs2017-win2016' + JavaVersion: '1.8' + ProfileFlag: '' Linux - Java LTS: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - javaVer: '1.11' - mvnFlags: '-Djava-lts' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + JavaVersion: '1.11' + ProfileFlag: '-Djava-lts' macOS - Java LTS: - os.name: 'macOS' - os.vmImage: 'macOS-10.13' - javaVer: '1.11' - mvnFlags: '-Djava-lts' + OSName: 'macOS' + OSVmImage: 'macOS-10.13' + JavaVersion: '1.11' + ProfileFlag: '-Djava-lts' Windows - Java LTS: - os.name: 'Windows' - os.vmImage: 'vs2017-win2016' - javaVer: '1.11' - mvnFlags: '-Djava-lts' + OSName: 'Windows' + OSVmImage: 'vs2017-win2016' + JavaVersion: '1.11' + ProfileFlag: '-Djava-lts' pool: - vmImage: $(os.vmImage) + vmImage: $(OSVmImage) steps: - task: Maven@3 displayName: 'Start Jetty' inputs: mavenPomFile: $(pomFile) - options: '--batch-mode $(mvnFlags)' - mavenOptions: '-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + options: '$(DefaultOptions) $(ProfileFlag)' + mavenOptions: '$(LoggingOptions)' javaHomeOption: 'JDKVersion' - jdkVersionOption: $(javaVer) + jdkVersionOption: $(JavaVersion) jdkArchitectureOption: 'x64' publishJUnitResults: false goals: 'jetty:run-forked' @@ -156,11 +161,16 @@ jobs: displayName: 'Run tests' inputs: mavenPomFile: $(pomFile) - options: '--batch-mode $(mvnFlags)' - mavenOptions: '-Xmx3072m -Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + options: '$(DefaultOptions) $(ProfileFlag)' + mavenOptions: '-Xmx3072m $(LoggingOptions)' javaHomeOption: 'JDKVersion' - jdkVersionOption: $(javaVer) + jdkVersionOption: $(JavaVersion) jdkArchitectureOption: 'x64' - publishJUnitResults: true - testResultsFiles: '**/TEST-*.xml' + publishJUnitResults: false goals: 'test' + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + mergeTestResults: true + testRunTitle: '$(OSName) on Java $(JavaVersion)' From c086bd8a08c9aa8ef4c0c4bee7ce5cf242ead506 Mon Sep 17 00:00:00 2001 From: Jonathan Giles Date: Fri, 22 Feb 2019 09:57:57 +1300 Subject: [PATCH 2/5] Update index.html (#2965) Make more mobile friendly and add some icons --- .../src/main/resources/index.html | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/eng/code-quality-reports/src/main/resources/index.html b/eng/code-quality-reports/src/main/resources/index.html index 582c6d8c8ea3a..e5c073f6edc9f 100644 --- a/eng/code-quality-reports/src/main/resources/index.html +++ b/eng/code-quality-reports/src/main/resources/index.html @@ -352,7 +352,8 @@ background-color: #fff; border: 1px solid #e8e8e8; font-size: 1.1em; - padding: 15px + padding: 15px; + overflow: hidden; } body.status-page .list-group .list-group-item a { @@ -413,7 +414,6 @@ } body.status-page footer { - position: absolute; width: 100%; bottom: 0; height: 60px; @@ -480,9 +480,11 @@

Azure Java SDK

  • + GitHub
  • + Azure Java Developers Documentation Home
    • @@ -494,15 +496,19 @@

    Azure Java SDK

  • + Maven Site
  • + SpotBugs (Aggregate)
  • + CheckStyle (Aggregate)
  • + JavaDoc (Aggregate)
    • @@ -514,6 +520,7 @@

    Azure Java SDK

  • + Azure Client SDK for Java
    @@ -539,4 +546,4 @@

    Azure Java SDK

    - \ No newline at end of file + From 3eb32a415c5e1555a431d726e017adcd8321220b Mon Sep 17 00:00:00 2001 From: Jonathan Giles Date: Fri, 22 Feb 2019 14:52:45 +1300 Subject: [PATCH 3/5] Fix issue where spotbugs can't find the excludes filter file after the reporting refactoring. (#2966) --- pom.client.xml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pom.client.xml b/pom.client.xml index 9c557c6704624..0030bddc99f60 100644 --- a/pom.client.xml +++ b/pom.client.xml @@ -291,6 +291,23 @@ com.github.spotbugs spotbugs-maven-plugin ${spotbugs.version} + + + com.azure + sdk-build-tools + ${maven-build-tools.version} + + + + Max + Low + true + ${project.build.directory}/spotbugs + spotbugs/spotbugs-exclude.xml + + + false + verify @@ -318,7 +335,7 @@ ${project.build.directory}/spotbugs - *.xml + spotbugsXml.xml ${project.build.directory}/spotbugs fancy-hist.xsl From 88f42268e6509c87faddb8ea5cdae8b77d8504cd Mon Sep 17 00:00:00 2001 From: Connie Yau Date: Fri, 22 Feb 2019 10:21:13 -0800 Subject: [PATCH 4/5] P3 Bugs (#2967) * Adding ConstantName suppression for BatchErrorCodeStrings and TaskFailureInformationCodes * Fix internal methods on SignatureEncoding * Add spotbug excludes for shipped cryptographic algorithms * Adding exclusion for null variables * Adding spotbug-exclude for content-type localisation --- .../checkstyle/checkstyle-suppressions.xml | 2 ++ .../resources/spotbugs/spotbugs-exclude.xml | 20 +++++++++++++++++-- .../cryptography/SignatureEncoding.java | 8 ++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml index 3a2339fc6058a..70aa4a322e7a5 100755 --- a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml +++ b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml @@ -30,4 +30,6 @@ + + diff --git a/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml b/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml index e50fea05fbf4d..906a06dd54c64 100755 --- a/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml +++ b/eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml @@ -8,9 +8,12 @@ - + - + + + + @@ -45,4 +48,17 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/keyvault/data-plane/azure-keyvault-cryptography/src/main/java/com/microsoft/azure/keyvault/cryptography/SignatureEncoding.java b/keyvault/data-plane/azure-keyvault-cryptography/src/main/java/com/microsoft/azure/keyvault/cryptography/SignatureEncoding.java index 90ec2cc625f91..5912e38411d61 100644 --- a/keyvault/data-plane/azure-keyvault-cryptography/src/main/java/com/microsoft/azure/keyvault/cryptography/SignatureEncoding.java +++ b/keyvault/data-plane/azure-keyvault-cryptography/src/main/java/com/microsoft/azure/keyvault/cryptography/SignatureEncoding.java @@ -47,7 +47,7 @@ public static byte[] fromAsn1Der(byte[] asn1DerSignature, String algorithm) thro public static byte[] fromAsn1Der(byte[] asn1DerSignature, Ecdsa algorithm) { try { - return Asn1DerSignatureEncoding.Decode(asn1DerSignature, algorithm); + return Asn1DerSignatureEncoding.decode(asn1DerSignature, algorithm); } catch (IllegalArgumentException ex) { throw (IllegalArgumentException) new IllegalArgumentException( ex.getMessage() + " " + Hex.encodeHexString(asn1DerSignature)).initCause(ex); @@ -84,7 +84,7 @@ public static byte[] toAsn1Der(byte[] signature, String algorithm) throws NoSuch */ public static byte[] toAsn1Der(byte[] signature, Ecdsa algorithm) { try { - return Asn1DerSignatureEncoding.Encode(signature, algorithm); + return Asn1DerSignatureEncoding.encode(signature, algorithm); } catch (IllegalArgumentException ex) { throw (IllegalArgumentException) new IllegalArgumentException( ex.getMessage() + " " + Hex.encodeHexString(signature)).initCause(ex); @@ -110,7 +110,7 @@ private Asn1DerSignatureEncoding() { } - public static byte[] Encode(byte[] signature, Ecdsa algorithm) { + public static byte[] encode(byte[] signature, Ecdsa algorithm) { int coordLength = algorithm.getCoordLength(); // verify that the signature is the correct length for the given algorithm @@ -144,7 +144,7 @@ public static byte[] Encode(byte[] signature, Ecdsa algorithm) { return asn1DerSignature.toByteArray(); } - public static byte[] Decode(byte[] bytes, Ecdsa algorithm) { + public static byte[] decode(byte[] bytes, Ecdsa algorithm) { int coordLength = algorithm.getCoordLength(); ByteArrayInputStream asn1DerSignature = new ByteArrayInputStream(bytes); From c5c54973bb272520452627387f02425664b92237 Mon Sep 17 00:00:00 2001 From: Vinay Gera Date: Fri, 22 Feb 2019 19:58:02 -0500 Subject: [PATCH 5/5] Batch unit tests time out update. (#2968) --- .../java/com/microsoft/azure/batch/BatchTestBase.java | 4 ++-- .../test/java/com/microsoft/azure/batch/PoolTests.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/batch/data-plane/src/test/java/com/microsoft/azure/batch/BatchTestBase.java b/batch/data-plane/src/test/java/com/microsoft/azure/batch/BatchTestBase.java index 2159edfbe90f3..1732ac0d90e09 100644 --- a/batch/data-plane/src/test/java/com/microsoft/azure/batch/BatchTestBase.java +++ b/batch/data-plane/src/test/java/com/microsoft/azure/batch/BatchTestBase.java @@ -54,8 +54,8 @@ static CloudPool createIfNotExistPaaSPool(String poolId) throws Exception { String POOL_OS_FAMILY = "4"; String POOL_OS_VERSION = "*"; - // 5 minutes - long POOL_STEADY_TIMEOUT_IN_SECONDS = 5 * 60 * 1000; + // 10 minutes + long POOL_STEADY_TIMEOUT_IN_SECONDS = 10 * 60 * 1000; // Check if pool exists if (!batchClient.poolOperations().existsPool(poolId)) { diff --git a/batch/data-plane/src/test/java/com/microsoft/azure/batch/PoolTests.java b/batch/data-plane/src/test/java/com/microsoft/azure/batch/PoolTests.java index e96eea19f1355..ef6aa9774a5d9 100644 --- a/batch/data-plane/src/test/java/com/microsoft/azure/batch/PoolTests.java +++ b/batch/data-plane/src/test/java/com/microsoft/azure/batch/PoolTests.java @@ -58,8 +58,8 @@ public void canCRUDLowPriIaaSPool() throws Exception { int POOL_VM_COUNT = 0; int POOL_LOW_PRI_VM_COUNT = 2; - // 5 minutes - long POOL_STEADY_TIMEOUT_IN_SECONDS = 5 * 60 * 1000; + // 10 minutes + long POOL_STEADY_TIMEOUT_IN_SECONDS = 10 * 60 * 1000; // Check if pool exists if (!batchClient.poolOperations().existsPool(poolId)) { @@ -366,8 +366,8 @@ public void canCRUDLowPriPaaSPool() throws Exception { String POOL_OS_FAMILY = "4"; String POOL_OS_VERSION = "*"; - // 5 minutes - long POOL_STEADY_TIMEOUT_IN_SECONDS = 5 * 60 * 1000; + // 10 minutes + long POOL_STEADY_TIMEOUT_IN_SECONDS = 10 * 60 * 1000; // Check if pool exists if (!batchClient.poolOperations().existsPool(poolId)) {