Merge branch 'master' into release/identity-service

Azure · Jul 7, 2020 · e94f359 · e94f359
2 parents d295415 + 1961116
commit e94f359
Show file tree

Hide file tree

Showing 260 changed files with 2,807 additions and 1,773 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -17,7 +17,7 @@
 /sdk/core/                                          @alzimmermsft @jianghaolu @srnagar @hemanttanwar @anuchandy
 /sdk/core/azure-core-tracing-opentelemetry/         @samvaity @alzimmermsft
 /sdk/cosmos/                                        @moderakh @kushagraThapar @David-Noble-at-work @kirankumarkolli @mbhaskar
-/sdk/cosmos/azure-spring-data-cosmosdb/             @kushagraThapar
+/sdk/cosmos/azure-spring-data-cosmosdb/             @kushagraThapar @saragluna @yiliuTo @chenrujun @zhoufenqin @jialindai
 /sdk/eventhubs/                                     @conniey @srnagar @mssfang
 /sdk/formrecognizer/                                @samvaity @mssfang @sima-zhu
 /sdk/identity/                                      @schaabs @g2vinay @jianghaolu
@@ -27,7 +27,7 @@
 /sdk/servicebus/                                    @yvgopal @nemakam @hemanttanwar @conniey
 /sdk/storage/                                       @amishra-dev @rickle-msft @jaschrep-msft @gapra-msft @alzimmermsft @sima-zhu
 /sdk/textanalytics/                                 @samvaity @mssfang @sima-zhu
-/sdk/spring/                                        @saragluna @yiliuTo @chenrujun @jialindai
+/sdk/spring/                                        @saragluna @yiliuTo @chenrujun @zhoufenqin @jialindai
 
 # end to end tests
 /sdk/e2e/                                           @jianghaolu @g2vinay

diff --git a/.gitignore b/.gitignore
@@ -68,3 +68,4 @@ package-lock.json
 # VS Code
 .vscode/
 .factorypath
+jacoco.exec
diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml
@@ -417,6 +417,8 @@
   <suppress checks="LineLength" files="com.azure.ai.formrecognizer.implementation.models"/>
   <suppress checks="LineLength" files="com.azure.ai.formrecognizer.Transforms.java"/>
 
+  <suppress checks="GoodLoggingCheck" files="com.azure.ai.formrecognizer.models.FieldValueType.java" />
+
   <!-- TODO: Fix with https://github.com/Azure/azure-sdk-for-java/issues#6716 Method name should follow a common vocabulary. -->
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files=".*[/\\]textanalytics[/\\].*"/>
 

diff --git a/eng/common/pipelines/templates/steps/verify-changelog.yml b/eng/common/pipelines/templates/steps/verify-changelog.yml
@@ -21,5 +21,5 @@ steps:
         -ForRelease $${{ parameters.ForRelease }}
       pwsh: true
       workingDirectory: $(Pipeline.Workspace)
-    displayName: Verify ChangeLog / Release Notes
+    displayName: Verify ChangeLogEntry for ${{ parameters.PackageName }}
     continueOnError: false
diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1
@@ -6,7 +6,6 @@ param (
   # used by VerifyPackages
   $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory)
   $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary)
-  [ValidateSet("Nuget","NPM","PyPI","Maven")]
   $packageRepository, # used to indicate destination against which we will check the existing version.
   # valid options: PyPI, Nuget, NPM, Maven, C
   # used by CreateTags

diff --git a/eng/common/scripts/modules/ChangeLog-Operations.psm1 b/eng/common/scripts/modules/ChangeLog-Operations.psm1
@@ -94,9 +94,8 @@ function Confirm-ChangeLogEntry {
   }
 
   if ($ForRelease -eq $True) {
-    $CurrentDate = Get-Date -Format "yyyy-MM-dd"
-    if ($changeLogEntry.ReleaseStatus -ne "($CurrentDate)") {
-      Write-Host ("##[warning]Incorrect Date: Please use the current date in the Changelog '{0}' before releasing the package" -f $ChangeLogLocation)
+    if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") {
+      Write-Host ("##[error]No release date set. Please set a release date with format 'yyyy-MM-dd' in the heading for version '{0}' in the changelog '{1}'." -f $VersionString, $ChangelogLocation)
       exit 1
     }
 

diff --git a/eng/common/scripts/modules/Package-Properties.psm1 b/eng/common/scripts/modules/Package-Properties.psm1
@@ -106,7 +106,7 @@ function Extract-PythonPkgProps ($pkgPath, $serviceName, $pkgName)
     {
         $setupLocation = $pkgPath.Replace('\','/')
         pushd $RepoRoot
-        $setupProps = (python -c "import scripts.devops_tasks.common_tasks; obj=scripts.devops_tasks.common_tasks.parse_setup('$setupLocation'); print('{0},{1}'.format(obj[0], obj[1]));") -split ","
+        $setupProps = (python -c "import sys; import os; sys.path.append(os.path.join('scripts', 'devops_tasks')); from common_tasks import parse_setup; obj=parse_setup('$setupLocation'); print('{0},{1}'.format(obj[0], obj[1]));") -split ","
         popd
         if (($setupProps -ne $null) -and ($setupProps[0] -eq $pkgName))
         {

diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml
@@ -94,17 +94,25 @@ stages:
             DisplayName: 'Emulator only Integration Tests'
             PROTOCOLS: '["Tcp"]'
             DESIRED_CONSISTENCIES: '["Strong", "Session"]'
+            AdditionalArgs: '-DargLine="-DACCOUNT_HOST=https://localhost:8081/"'
           Https_Integration_Tests_Java8:
             OSVmImage: 'windows-2019'
             JavaTestVersion: '1.8'
             ProfileFlag: '-Pemulator'
             DisplayName: 'Emulator only Integration Tests'
             PROTOCOLS: '["Https"]'
             DESIRED_CONSISTENCIES: '["Strong", "Session"]'
+            AdditionalArgs: '-DargLine="-DACCOUNT_HOST=https://localhost:8081/"'
+          Spring_Integration_Tests_Java8:
+            OSVmImage: 'windows-2019'
+            JavaTestVersion: '1.8'
+            ProfileFlag: '-P integration-test-emulator'
+            DisplayName: 'Spring Emulator only Integration Tests'
+            AdditionalArgs: '-DargLine="-DACCOUNT_HOST=https://localhost:8081/ -DACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== -DSECONDARY_ACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="'
 
         TestStepMavenInputs:
           goals: 'verify'
-          options: '$(ProfileFlag) -Dgpg.skip -DargLine="-DACCOUNT_HOST=https://localhost:8081/" -pl $(ProjectList)'
+          options: '$(ProfileFlag) -Dgpg.skip $(AdditionalArgs) -pl $(ProjectList)'
           mavenAuthenticateFeed: true
           jdkVersionOption: $(JavaTestVersion)
 

diff --git a/sdk/appservice/mgmt/src/test/java/com/azure/resourcemanager/appservice/AppServiceTest.java b/sdk/appservice/mgmt/src/test/java/com/azure/resourcemanager/appservice/AppServiceTest.java
@@ -34,11 +34,11 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
-import java.net.URL;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.temporal.ChronoUnit;
 
+import com.azure.resourcemanager.resources.fluentcore.utils.Utils;
 import org.apache.commons.net.ftp.FTP;
 import org.apache.commons.net.ftp.FTPClient;
 import org.junit.jupiter.api.Assertions;
@@ -171,7 +171,7 @@ public static void uploadFileToWebApp(PublishingProfile profile, String fileName
 
     protected Response<String> curl(String urlString) throws IOException {
         try {
-            return stringResponse(httpClient.getString(getHost(urlString), getPathAndQuery(urlString))).block();
+            return stringResponse(httpClient.getString(Utils.getHost(urlString), Utils.getPathAndQuery(urlString))).block();
         } catch (MalformedURLException e) {
             Assertions.fail();
             return null;
@@ -180,7 +180,7 @@ protected Response<String> curl(String urlString) throws IOException {
 
     protected String post(String urlString, String body) {
         try {
-            return stringResponse(httpClient.postString(getHost(urlString), getPathAndQuery(urlString), body))
+            return stringResponse(httpClient.postString(Utils.getHost(urlString), Utils.getPathAndQuery(urlString), body))
                 .block()
                 .getValue();
         } catch (Exception e) {
@@ -201,23 +201,6 @@ private static Mono<SimpleResponse<String>> stringResponse(Mono<SimpleResponse<F
                                     response.getRequest(), response.getStatusCode(), response.getHeaders(), str)));
     }
 
-    private static String getHost(String urlString) throws MalformedURLException {
-        URL url = new URL(urlString);
-        String protocol = url.getProtocol();
-        String host = url.getAuthority();
-        return protocol + "://" + host;
-    }
-
-    private static String getPathAndQuery(String urlString) throws MalformedURLException {
-        URL url = new URL(urlString);
-        String path = url.getPath();
-        String query = url.getQuery();
-        if (query != null && !query.isEmpty()) {
-            path = path + "?" + query;
-        }
-        return path;
-    }
-
     protected WebAppTestClient httpClient =
         RestProxy
             .create(

diff --git a/...osmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/BackoffRetryUtility.java b/...osmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/BackoffRetryUtility.java
@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 package com.azure.cosmos.implementation;
 
+import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 import java.time.Duration;
@@ -38,6 +39,17 @@ static public <T> Mono<T> executeRetry(Callable<Mono<T>> callbackMethod,
         }).retryWhen(RetryUtils.toRetryWhenFunc(retryPolicy));
     }
 
+    static public <T> Flux<T> fluxExecuteRetry(Callable<Flux<T>> callbackMethod, IRetryPolicy retryPolicy) {
+
+        return Flux.defer(() -> {
+            try {
+                return callbackMethod.call();
+            } catch (Exception e) {
+                return Flux.error(e);
+            }
+        }).retryWhen(RetryUtils.toRetryWhenFunc(retryPolicy));
+    }
+
     static public <T> Mono<T> executeAsync(
             Function<Quadruple<Boolean, Boolean, Duration, Integer>, Mono<T>> callbackMethod, IRetryPolicy retryPolicy,
             Function<Quadruple<Boolean, Boolean, Duration, Integer>, Mono<T>> inBackoffAlternateCallbackMethod,

diff --git a/...s/src/main/java/com/azure/cosmos/implementation/InvalidPartitionExceptionRetryPolicy.java b/...s/src/main/java/com/azure/cosmos/implementation/InvalidPartitionExceptionRetryPolicy.java
@@ -41,7 +41,9 @@ public InvalidPartitionExceptionRetryPolicy(RxCollectionCache collectionCache,
     @Override
     public void onBeforeSendRequest(RxDocumentServiceRequest request) {
         this.request = request;
-        this.nextPolicy.onBeforeSendRequest(request);
+        if (this.nextPolicy != null) {
+            this.nextPolicy.onBeforeSendRequest(request);
+        }
     }
 
     @Override
@@ -56,7 +58,7 @@ public Mono<ShouldRetryResult> shouldRetry(Exception e) {
                 // TODO: this is blocking. is that fine?
                 if(this.cosmosQueryRequestOptions != null) {
                     this.clientCollectionCache.refresh(
-                        BridgeInternal.getMetaDataDiagnosticContext(this.request.requestContext.cosmosDiagnostics),
+                        this.request != null ? BridgeInternal.getMetaDataDiagnosticContext(this.request.requestContext.cosmosDiagnostics) : null,
                         collectionLink,
                         ModelBridgeInternal.getPropertiesFromQueryRequestOptions(this.cosmosQueryRequestOptions));
                 } else {
@@ -73,6 +75,9 @@ public Mono<ShouldRetryResult> shouldRetry(Exception e) {
             }
         }
 
-        return this.nextPolicy.shouldRetry(e);
+        if (this.nextPolicy != null) {
+            return this.nextPolicy.shouldRetry(e);
+        }
+        return Mono.just(ShouldRetryResult.error(e));
     }
 }
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ObservableHelper.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ObservableHelper.java
@@ -44,4 +44,21 @@ static public <T> Mono<T> inlineIfPossibleAsObs(Callable<Mono<T>> function, IRet
             return BackoffRetryUtility.executeRetry(() -> function.call(), retryPolicy);
         }
     }
+
+    static public <T> Flux<T> fluxInlineIfPossibleAsObs(Callable<Flux<T>> function, IRetryPolicy retryPolicy) {
+
+        if (retryPolicy == null) {
+            // shortcut
+            return Flux.defer(() -> {
+                try {
+                    return function.call();
+                } catch (Exception e) {
+                    return Flux.error(e);
+                }
+            });
+
+        } else {
+            return BackoffRetryUtility.fluxExecuteRetry(() -> function.call(), retryPolicy);
+        }
+    }
 }
diff --git a/...smos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/...smos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java
@@ -560,18 +560,42 @@ private String parentResourceLinkToQueryLink(String parentResouceLink, ResourceT
     }
 
     private <T extends Resource> Flux<FeedResponse<T>> createQuery(
-            String parentResourceLink,
+        String parentResourceLink,
+        SqlQuerySpec sqlQuery,
+        CosmosQueryRequestOptions options,
+        Class<T> klass,
+        ResourceType resourceTypeEnum) {
+
+        String resourceLink = parentResourceLinkToQueryLink(parentResourceLink, resourceTypeEnum);
+        UUID activityId = Utils.randomUUID();
+        IDocumentQueryClient queryClient = documentQueryClientImpl(RxDocumentClientImpl.this);
+
+        // Trying to put this logic as low as the query pipeline
+        // Since for parallelQuery, each partition will have its own request, so at this point, there will be no request associate with this retry policy.
+        // For default document context, it already wired up InvalidPartitionExceptionRetry, but there is no harm to wire it again here
+        InvalidPartitionExceptionRetryPolicy invalidPartitionExceptionRetryPolicy = new InvalidPartitionExceptionRetryPolicy(
+            this.collectionCache,
+            null,
+            resourceLink,
+            options);
+
+        return ObservableHelper.fluxInlineIfPossibleAsObs(
+            () -> createQueryInternal(resourceLink, sqlQuery, options, klass, resourceTypeEnum, queryClient, activityId),
+            invalidPartitionExceptionRetryPolicy);
+    }
+
+    private <T extends Resource> Flux<FeedResponse<T>> createQueryInternal(
+            String resourceLink,
             SqlQuerySpec sqlQuery,
             CosmosQueryRequestOptions options,
             Class<T> klass,
-            ResourceType resourceTypeEnum) {
+            ResourceType resourceTypeEnum,
+            IDocumentQueryClient queryClient,
+            UUID activityId) {
 
-        String queryResourceLink = parentResourceLinkToQueryLink(parentResourceLink, resourceTypeEnum);
-
-        UUID activityId = Utils.randomUUID();
-        IDocumentQueryClient queryClient = documentQueryClientImpl(RxDocumentClientImpl.this);
         Flux<? extends IDocumentQueryExecutionContext<T>> executionContext =
-                DocumentQueryExecutionContextFactory.createDocumentQueryExecutionContextAsync(queryClient, resourceTypeEnum, klass, sqlQuery , options, queryResourceLink, false, activityId);
+                DocumentQueryExecutionContextFactory.createDocumentQueryExecutionContextAsync(queryClient, resourceTypeEnum, klass, sqlQuery , options, resourceLink, false, activityId);
+
         return executionContext.flatMap(iDocumentQueryExecutionContext -> {
             QueryInfo queryInfo = null;
             if (iDocumentQueryExecutionContext instanceof PipelinedDocumentQueryExecutionContext) {

diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/AzureKeyCredentialTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/AzureKeyCredentialTest.java
@@ -1,25 +1,24 @@
 package com.azure.cosmos;
 
-import com.azure.cosmos.implementation.InternalObjectNode;
 import com.azure.cosmos.implementation.FailureValidator;
+import com.azure.cosmos.implementation.InternalObjectNode;
 import com.azure.cosmos.implementation.RetryAnalyzer;
-import com.azure.cosmos.models.CosmosContainerResponse;
-import com.azure.cosmos.models.CosmosDatabaseResponse;
-import com.azure.cosmos.models.CosmosItemResponse;
+import com.azure.cosmos.implementation.TestConfigurations;
 import com.azure.cosmos.models.CosmosContainerProperties;
 import com.azure.cosmos.models.CosmosContainerRequestOptions;
+import com.azure.cosmos.models.CosmosContainerResponse;
 import com.azure.cosmos.models.CosmosDatabaseProperties;
 import com.azure.cosmos.models.CosmosDatabaseRequestOptions;
+import com.azure.cosmos.models.CosmosDatabaseResponse;
 import com.azure.cosmos.models.CosmosItemRequestOptions;
+import com.azure.cosmos.models.CosmosItemResponse;
 import com.azure.cosmos.models.CosmosResponse;
 import com.azure.cosmos.models.IndexingMode;
 import com.azure.cosmos.models.IndexingPolicy;
 import com.azure.cosmos.models.ModelBridgeInternal;
 import com.azure.cosmos.models.PartitionKey;
-import com.azure.cosmos.models.PartitionKeyDefinition;
 import com.azure.cosmos.rx.CosmosItemResponseValidator;
 import com.azure.cosmos.rx.TestSuiteBase;
-import com.azure.cosmos.implementation.TestConfigurations;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
@@ -65,17 +64,6 @@ public Object[][] crudArgProvider() {
         };
     }
 
-    private CosmosContainerProperties getCollectionDefinition(String collectionName) {
-        PartitionKeyDefinition partitionKeyDef = new PartitionKeyDefinition();
-        ArrayList<String> paths = new ArrayList<>();
-        paths.add("/mypk");
-        partitionKeyDef.setPaths(paths);
-
-        return new CosmosContainerProperties(
-            collectionName,
-            partitionKeyDef);
-    }
-
     private InternalObjectNode getDocumentDefinition(String documentId) {
         final String uuid = UUID.randomUUID().toString();
         return new InternalObjectNode(String.format("{ "

diff --git a/...zure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerContentResponseOnWriteTest.java b/...zure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerContentResponseOnWriteTest.java
@@ -11,14 +11,12 @@
 import com.azure.cosmos.models.CosmosContainerResponse;
 import com.azure.cosmos.models.IndexingMode;
 import com.azure.cosmos.models.IndexingPolicy;
-import com.azure.cosmos.models.PartitionKeyDefinition;
 import com.azure.cosmos.rx.TestSuiteBase;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Factory;
 import org.testng.annotations.Test;
 
-import java.util.ArrayList;
 import java.util.UUID;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -102,17 +100,6 @@ public void replaceContainer_withContentResponseOnWriteDisabled() throws Excepti
 
     }
 
-    private CosmosContainerProperties getCollectionDefinition(String collectionName) {
-        PartitionKeyDefinition partitionKeyDef = new PartitionKeyDefinition();
-        ArrayList<String> paths = new ArrayList<String>();
-        paths.add("/mypk");
-        partitionKeyDef.setPaths(paths);
-
-        return new CosmosContainerProperties(
-            collectionName,
-            partitionKeyDef);
-    }
-
     private void validateContainerResponse(CosmosContainerProperties containerProperties,
                                            CosmosContainerResponse createResponse) {
         // Basic validation