From bc570709fce496fb8a78cad0dcdfa07b8bae0b4a Mon Sep 17 00:00:00 2001 From: Jianghao Lu Date: Tue, 14 Jun 2016 17:03:29 -0700 Subject: [PATCH 01/16] Basic prototyping of async in fluent --- .../java/com/microsoft/azure/TaskGroup.java | 16 +++++ .../com/microsoft/azure/TaskGroupBase.java | 71 ++++++++++++++----- .../java/com/microsoft/azure/TaskItem.java | 9 ++- 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java index 94b4628ff70ae..29507425d4439 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java @@ -7,6 +7,9 @@ package com.microsoft.azure; +import com.microsoft.rest.ServiceCall; +import com.microsoft.rest.ServiceCallback; + /** * Represents a group of related tasks. *

@@ -39,6 +42,11 @@ public interface TaskGroup> { */ void merge(TaskGroup parentTaskGroup); + /** + * Prepare the graph for execution. + */ + void prepare(); + /** * Executes the tasks in the group. *

@@ -48,6 +56,14 @@ public interface TaskGroup> { */ void execute() throws Exception; + /** + * Executes the tasks in the group asynchronously. + * + * @param callback the callback to call on failure or success + * @return the handle to the REST call + */ + ServiceCall executeAsync(ServiceCallback callback); + /** * Gets the result of execution of a task in the group. *

diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java index d0dc430edc3a6..a4a93ae1f281c 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -7,15 +7,17 @@ package com.microsoft.azure; +import com.microsoft.rest.ServiceCall; +import com.microsoft.rest.ServiceCallback; + /** * The base implementation of TaskGroup interface. * * @param the result type of the tasks in the group - * @param type representing task in the group */ -public abstract class TaskGroupBase> - implements TaskGroup { - private DAGraph> dag; +public abstract class TaskGroupBase + implements TaskGroup> { + private DAGraph, DAGNode>> dag; /** * Creates TaskGroupBase. @@ -23,12 +25,12 @@ public abstract class TaskGroupBase> * @param rootTaskItemId the id of the root task in this task group * @param rootTaskItem the root task */ - public TaskGroupBase(String rootTaskItemId, U rootTaskItem) { + public TaskGroupBase(String rootTaskItemId, TaskItem rootTaskItem) { this.dag = new DAGraph<>(new DAGNode<>(rootTaskItemId, rootTaskItem)); } @Override - public DAGraph> dag() { + public DAGraph, DAGNode>> dag() { return dag; } @@ -38,24 +40,42 @@ public boolean isRoot() { } @Override - public void merge(TaskGroup parentTaskGroup) { + public void merge(TaskGroup> parentTaskGroup) { dag.merge(parentTaskGroup.dag()); } @Override - public void execute() throws Exception { + public void prepare() { if (isRoot()) { dag.prepare(); - DAGNode nextNode = dag.getNext(); - while (nextNode != null) { - if (dag.isRootNode(nextNode)) { - executeRootTask(nextNode.data()); - } else { - nextNode.data().execute(); - } - dag.reportedCompleted(nextNode); - nextNode = dag.getNext(); - } + } + } + + @Override + public void execute() throws Exception { + DAGNode> nextNode = dag.getNext(); + if (nextNode == null) { + return; + } + + if (dag.isRootNode(nextNode)) { + executeRootTask(nextNode.data()); + } else { + nextNode.data().execute(this, nextNode); + } + } + + @Override + public ServiceCall executeAsync(final ServiceCallback callback) { + final DAGNode> nextNode = dag.getNext(); + if (nextNode == null) { + return null; + } + + if (dag.isRootNode(nextNode)) { + return executeRootTaskAsync(nextNode.data(), callback); + } else { + return nextNode.data().executeAsync(this, nextNode, callback); } } @@ -74,5 +94,18 @@ public T taskResult(String taskId) { * @param task the root task in this group * @throws Exception the exception */ - public abstract void executeRootTask(U task) throws Exception; + public abstract void executeRootTask(TaskItem task) throws Exception; + + /** + * executes the root task in this group asynchronously. + *

+ * this method will be invoked when all the task dependencies of the root task are finished + * executing, at this point root task can be executed by consuming the result of tasks it + * depends on. + * + * @param task the root task in this group + * @param callback the callback when the task fails or succeeds + * @return the handle to the REST call + */ + public abstract ServiceCall executeRootTaskAsync(TaskItem task, ServiceCallback callback); } diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java index 7df830f69bb9e..870105917457b 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java @@ -7,6 +7,9 @@ package com.microsoft.azure; +import com.microsoft.rest.ServiceCall; +import com.microsoft.rest.ServiceCallback; + /** * Type representing a task in a task group {@link TaskGroup}. * @@ -22,7 +25,11 @@ public interface TaskItem { * Executes the task. *

* once executed the result will be available through result getter + * + * @param taskGroup the task group dispatching tasks * @throws Exception exception */ - void execute() throws Exception; + void execute(TaskGroup> taskGroup, DAGNode> node) throws Exception; + + ServiceCall executeAsync(TaskGroup> taskGroup, DAGNode> node, ServiceCallback callback); } From 5aac1faca68b57ac728769e169bee72c3822ba85 Mon Sep 17 00:00:00 2001 From: Jianghao Lu Date: Wed, 15 Jun 2016 00:08:26 -0700 Subject: [PATCH 02/16] Add applyAsync() --- .../src/main/java/com/microsoft/azure/TaskItem.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java index 870105917457b..fb74c23845c3b 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java @@ -27,9 +27,20 @@ public interface TaskItem { * once executed the result will be available through result getter * * @param taskGroup the task group dispatching tasks + * @param node the node the task item is associated with * @throws Exception exception */ void execute(TaskGroup> taskGroup, DAGNode> node) throws Exception; + /** + * Executes the task asynchronously. + *

+ * once executed the result will be available through result getter + + * @param taskGroup the task group dispatching tasks + * @param node the node the task item is associated with + * @param callback callback to call on success or failure + * @return the handle of the REST call + */ ServiceCall executeAsync(TaskGroup> taskGroup, DAGNode> node, ServiceCallback callback); } From 0261a6e8f4ef014f32ed2b46f9901ea9d9c83028 Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 15 Jun 2016 10:03:50 -0700 Subject: [PATCH 03/16] Adding few more Azure environments --- .../com/microsoft/azure/AzureEnvironment.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java index 1721e2ff97bb8..1012a1fc09ac9 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java @@ -71,7 +71,25 @@ public AzureEnvironment( "https://login.chinacloudapi.cn/", "https://management.core.chinacloudapi.cn/", true, - "https://management.chinacloudapi.cn"); + "https://management.chinacloudapi.cn/"); + + /** + * Provides the settings for authentication with Azure US Government. + */ + public static final AzureEnvironment AZURE_US_GOVERNMENT = new AzureEnvironment( + "https://login.microsoftonline.com/", + "https://management.core.usgovcloudapi.net/", + true, + "https://management.usgovcloudapi.net/"); + + /** + * Provides the settings for authentication with Azure German Government. + */ + public static final AzureEnvironment AZURE_GERMAN_GOVERNMENT = new AzureEnvironment( + "https://login.microsoftonline.de/", + "https://management.core.cloudapi.de/", + true, + "https://management.microsoftazure.de"); /** * Gets the base URL of the management service. From c8db8d3f72d2d99437849b7d99359bdbb330d09f Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 15 Jun 2016 10:34:31 -0700 Subject: [PATCH 04/16] correcting environment name --- .../src/main/java/com/microsoft/azure/AzureEnvironment.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java index 1012a1fc09ac9..8c01b6e1e7af9 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java @@ -83,9 +83,9 @@ public AzureEnvironment( "https://management.usgovcloudapi.net/"); /** - * Provides the settings for authentication with Azure German Government. + * Provides the settings for authentication with Azure German. */ - public static final AzureEnvironment AZURE_GERMAN_GOVERNMENT = new AzureEnvironment( + public static final AzureEnvironment AZURE_GERMAN = new AzureEnvironment( "https://login.microsoftonline.de/", "https://management.core.cloudapi.de/", true, From 6c8f63a106131b9b070bafccd7abd2a1a54ad9ac Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 15 Jun 2016 10:36:13 -0700 Subject: [PATCH 05/16] ensuring trailing slash for urls --- .../src/main/java/com/microsoft/azure/AzureEnvironment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java index 8c01b6e1e7af9..d4374a9f21ffc 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java @@ -89,7 +89,7 @@ public AzureEnvironment( "https://login.microsoftonline.de/", "https://management.core.cloudapi.de/", true, - "https://management.microsoftazure.de"); + "https://management.microsoftazure.de/"); /** * Gets the base URL of the management service. From 738d121b2c84dbbcf1964ab6bb216cf69797933a Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 15 Jun 2016 16:23:11 -0700 Subject: [PATCH 06/16] use AZURE_GERMANY insteadof AZURE_GERMAN --- .../src/main/java/com/microsoft/azure/AzureEnvironment.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java index d4374a9f21ffc..1b4ec58f12c7c 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java @@ -83,9 +83,9 @@ public AzureEnvironment( "https://management.usgovcloudapi.net/"); /** - * Provides the settings for authentication with Azure German. + * Provides the settings for authentication with Azure Germany. */ - public static final AzureEnvironment AZURE_GERMAN = new AzureEnvironment( + public static final AzureEnvironment AZURE_GERMANY = new AzureEnvironment( "https://login.microsoftonline.de/", "https://management.core.cloudapi.de/", true, From bb4a917c6aff4eedc535ca128c7e2471e86b0495 Mon Sep 17 00:00:00 2001 From: anuchan Date: Thu, 16 Jun 2016 13:09:47 -0700 Subject: [PATCH 07/16] Fixing the bug of trying to create created resources during update, adding better way to name gen resource names --- .../src/main/java/com/microsoft/azure/TaskGroupBase.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java index d0dc430edc3a6..1a43baebf0004 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -50,10 +50,15 @@ public void execute() throws Exception { while (nextNode != null) { if (dag.isRootNode(nextNode)) { executeRootTask(nextNode.data()); + dag.reportedCompleted(nextNode); } else { - nextNode.data().execute(); + // TaskGroupBase::execute will be called both in update and create + // scenarios, so run the task only if it not not executed already. + if (nextNode.data().result() == null) { + nextNode.data().execute(); + dag.reportedCompleted(nextNode); + } } - dag.reportedCompleted(nextNode); nextNode = dag.getNext(); } } From f84d8eb783dbc0fe9721c89ca78bb0add61c0372 Mon Sep 17 00:00:00 2001 From: anuchan Date: Thu, 16 Jun 2016 14:18:18 -0700 Subject: [PATCH 08/16] making dag::prepare to work for update --- .../src/main/java/com/microsoft/azure/DAGNode.java | 8 +++++++- .../src/main/java/com/microsoft/azure/DAGraph.java | 4 ++++ .../src/main/java/com/microsoft/azure/TaskGroupBase.java | 3 +-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java index ccc700618e58f..7de859e25f1f6 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java @@ -59,7 +59,6 @@ public List dependencyKeys() { * @param dependencyKey the id of the dependency node */ public void addDependency(String dependencyKey) { - toBeResolved++; super.addChild(dependencyKey); } @@ -70,6 +69,13 @@ public boolean hasDependencies() { return this.hasChildren(); } + /** + * prepare the node for traversal. + */ + public void prepare() { + this.toBeResolved = this.dependencyKeys().size(); + } + /** * @return true if all dependencies of this node are ready to be consumed */ diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java index 8b10bde8b3e3c..cfd50d9ca5b40 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -77,6 +77,10 @@ public void merge(DAGraph parent) { * in the DAG with no dependencies. */ public void prepare() { + for (Map.Entry entry: graph.entrySet()) { + entry.getValue().prepare(); + } + initializeQueue(); if (queue.isEmpty()) { throw new RuntimeException("Found circular dependency"); diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java index 1a43baebf0004..5cf4369142ea3 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -50,15 +50,14 @@ public void execute() throws Exception { while (nextNode != null) { if (dag.isRootNode(nextNode)) { executeRootTask(nextNode.data()); - dag.reportedCompleted(nextNode); } else { // TaskGroupBase::execute will be called both in update and create // scenarios, so run the task only if it not not executed already. if (nextNode.data().result() == null) { nextNode.data().execute(); - dag.reportedCompleted(nextNode); } } + dag.reportedCompleted(nextNode); nextNode = dag.getNext(); } } From 9c048fc3b4d6a4afbac71dcb5aa9046b2d5f9399 Mon Sep 17 00:00:00 2001 From: Jianghao Lu Date: Fri, 17 Jun 2016 10:15:46 -0700 Subject: [PATCH 09/16] Add a bunch of configs to RestClient --- .../java/com/microsoft/azure/RestClient.java | 70 +++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java b/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java index 2517f890a52ca..532ba2e392809 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java @@ -14,17 +14,20 @@ import com.microsoft.rest.credentials.ServiceClientCredentials; import com.microsoft.rest.retry.RetryHandler; import com.microsoft.rest.serializer.JacksonMapperAdapter; - -import java.lang.reflect.Field; -import java.net.CookieManager; -import java.net.CookiePolicy; - +import okhttp3.ConnectionPool; import okhttp3.Interceptor; import okhttp3.JavaNetCookieJar; import okhttp3.OkHttpClient; import okhttp3.logging.HttpLoggingInterceptor; import retrofit2.Retrofit; +import java.lang.reflect.Field; +import java.net.CookieManager; +import java.net.CookiePolicy; +import java.net.Proxy; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + /** * An instance of this class stores the client information for making REST calls. */ @@ -260,6 +263,63 @@ public Buildable withInterceptor(Interceptor interceptor) { return this; } + /** + * Set the read timeout on the HTTP client. Default is 10 seconds. + * + * @param timeout the timeout numeric value + * @param unit the time unit for the numeric value + * @return the builder itself for chaining + */ + public Buildable withReadTimeout(long timeout, TimeUnit unit) { + httpClientBuilder.readTimeout(timeout, unit); + return this; + } + + /** + * Set the connection timeout on the HTTP client. Default is 10 seconds. + * + * @param timeout the timeout numeric value + * @param unit the time unit for the numeric value + * @return the builder itself for chaining + */ + public Buildable withConnectionTimeout(long timeout, TimeUnit unit) { + httpClientBuilder.connectTimeout(timeout, unit); + return this; + } + + /** + * Set the maximum idle connections for the HTTP client. Default is 5. + * + * @param maxIdleConnections the maximum idle connections + * @return the builder itself for chaining + */ + public Buildable withMaxIdleConnections(int maxIdleConnections) { + httpClientBuilder.connectionPool(new ConnectionPool(maxIdleConnections, 5, TimeUnit.MINUTES)); + return this; + } + + /** + * Sets the executor for async callbacks to run on. + * + * @param executor the executor to execute the callbacks. + * @return the builder itself for chaining + */ + public Buildable withCallbackExecutor(Executor executor) { + retrofitBuilder.callbackExecutor(executor); + return this; + } + + /** + * Sets the proxy for the HTTP client. + * + * @param proxy the proxy to use + * @return the builder itself for chaining + */ + public Buildable withProxy(Proxy proxy) { + httpClientBuilder.proxy(proxy); + return this; + } + /** * Build a RestClient with all the current configurations. * From 999fb18b2ea092f1f7d831c66d7fbcd9da46c97c Mon Sep 17 00:00:00 2001 From: Jianghao Lu Date: Fri, 17 Jun 2016 11:19:27 -0700 Subject: [PATCH 10/16] Add retry for 404 GET --- ...rceGetExponentialBackoffRetryStrategy.java | 48 +++++++++++++++++++ .../java/com/microsoft/azure/RestClient.java | 1 + 2 files changed, 49 insertions(+) create mode 100644 azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java b/azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java new file mode 100644 index 0000000000000..a145fecc6795b --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java @@ -0,0 +1,48 @@ +/** + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + */ + +package com.microsoft.azure; + +import com.microsoft.rest.retry.RetryStrategy; +import okhttp3.Response; + +/** + * A retry strategy with backoff parameters for calculating the exponential + * delay between retries for 404s from GET calls. + */ +public class ResourceGetExponentialBackoffRetryStrategy extends RetryStrategy { + /** + * Represents the default number of retries. + */ + private static final int DEFAULT_NUMBER_OF_ATTEMPTS = 3; + + /** + * Creates an instance of the retry strategy. + */ + public ResourceGetExponentialBackoffRetryStrategy() { + this(null, DEFAULT_FIRST_FAST_RETRY); + } + + /** + * Initializes a new instance of the {@link RetryStrategy} class. + * + * @param name The name of the retry strategy. + * @param firstFastRetry true to immediately retry in the first attempt; otherwise, false. + */ + private ResourceGetExponentialBackoffRetryStrategy(String name, boolean firstFastRetry) { + super(name, firstFastRetry); + } + + @Override + public boolean shouldRetry(int retryCount, Response response) { + int code = response.code(); + //CHECKSTYLE IGNORE MagicNumber FOR NEXT 2 LINES + return retryCount < DEFAULT_NUMBER_OF_ATTEMPTS + && code == 404 + && response.request().method().equalsIgnoreCase("GET"); + } +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java b/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java index 532ba2e392809..0ace881f0ee51 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java @@ -330,6 +330,7 @@ public RestClient build() { OkHttpClient httpClient = httpClientBuilder .addInterceptor(baseUrlHandler) .addInterceptor(customHeadersInterceptor) + .addInterceptor(new RetryHandler(new ResourceGetExponentialBackoffRetryStrategy())) .addInterceptor(new RetryHandler()) .build(); return new RestClient(httpClient, From 45c3d887e58c2c9535007c11fa03c03840c25954 Mon Sep 17 00:00:00 2001 From: anuchan Date: Tue, 21 Jun 2016 12:46:20 -0700 Subject: [PATCH 11/16] couple of javadoc fixes --- .../credentials/ApplicationTokenCredentials.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java b/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java index 163d10f60a1d5..384ab3f7671af 100644 --- a/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java +++ b/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java @@ -113,13 +113,14 @@ public ApplicationTokenCredentials withDefaultSubscriptionId(String subscription * * @param credentialsFile A file with credentials, using the standard Java properties format. * and the following keys: - * subscription= - * tenant= - * client= - * key= - * managementURI= - * baseURL= - * authURL= + * subscription=<subscription-id> + * tenant=<tenant-id> + * client=<client-id> + * key=<client-key> + * managementURI=<management-URI> + * baseURL=<base-URL> + * authURL=<authentication-URL> + * * @return The credentials based on the file. * @throws IOException exception thrown from file access errors. */ From ac633da64004a935e9ae8af61c5b98df8c20d0d3 Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 22 Jun 2016 11:00:39 -0700 Subject: [PATCH 12/16] Making DAG to work correctly to handle more cases --- .../java/com/microsoft/azure/DAGNode.java | 25 ++++++++++-- .../java/com/microsoft/azure/DAGraph.java | 40 ++++++++++++------- .../main/java/com/microsoft/azure/Node.java | 3 +- .../java/com/microsoft/azure/TaskGroup.java | 11 ++--- .../com/microsoft/azure/TaskGroupBase.java | 20 +++------- .../java/com/microsoft/azure/DAGraphTest.java | 2 +- .../com/microsoft/azure/DAGraphTests.java | 2 +- 7 files changed, 63 insertions(+), 40 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java index 7de859e25f1f6..890228fff654c 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java @@ -8,6 +8,7 @@ package com.microsoft.azure; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -18,6 +19,7 @@ public class DAGNode extends Node { private List dependentKeys; private int toBeResolved; + private boolean isPreparer; /** * Creates a DAG node. @@ -34,7 +36,7 @@ public DAGNode(String key, T data) { * @return a list of keys of nodes in {@link DAGraph} those are dependents on this node */ List dependentKeys() { - return this.dependentKeys; + return Collections.unmodifiableList(this.dependentKeys); } /** @@ -70,10 +72,27 @@ public boolean hasDependencies() { } /** - * prepare the node for traversal. + * Mark or un-mark this node as being preparer. + * + * @param isPreparer true if this node needs to be marked as preparer, false otherwise. + */ + public void setPreparer(boolean isPreparer) { + this.isPreparer = isPreparer; + } + + /** + * @return true if this node is marked as preparer + */ + public boolean isPreparer() { + return isPreparer; + } + + /** + * initialize the node so that traversal can be performed on the parent DAG. */ - public void prepare() { + public void initialize() { this.toBeResolved = this.dependencyKeys().size(); + this.dependentKeys.clear(); } /** diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java index cfd50d9ca5b40..6bb33b0e0b2c7 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -32,6 +32,7 @@ public class DAGraph> extends Graph { public DAGraph(U rootNode) { this.rootNode = rootNode; this.queue = new ArrayDeque<>(); + this.rootNode.setPreparer(true); this.addNode(rootNode); } @@ -52,6 +53,14 @@ public boolean isRootNode(U node) { return this.rootNode == node; } + /** + * @return true if this dag is the preparer responsible for + * preparing the DAG for traversal. + */ + public boolean isPreparer() { + return this.rootNode.isPreparer(); + } + /** * Merge this DAG with another DAG. *

@@ -63,7 +72,6 @@ public boolean isRootNode(U node) { public void merge(DAGraph parent) { this.hasParent = true; parent.rootNode.addDependency(this.rootNode.key()); - this.rootNode.addDependent(parent.rootNode.key()); for (Map.Entry entry: graph.entrySet()) { String key = entry.getKey(); if (!parent.graph.containsKey(key)) { @@ -77,13 +85,15 @@ public void merge(DAGraph parent) { * in the DAG with no dependencies. */ public void prepare() { - for (Map.Entry entry: graph.entrySet()) { - entry.getValue().prepare(); - } - - initializeQueue(); - if (queue.isEmpty()) { - throw new RuntimeException("Found circular dependency"); + if (isPreparer()) { + for (Map.Entry entry : graph.entrySet()) { + // Prepare each node for traversal + entry.getValue().initialize(); + // Mark other sub-DAGs are non-preparer + entry.getValue().setPreparer(false); + } + initializeDependentKeys(); + initializeQueue(); } } @@ -115,6 +125,7 @@ public T getNodeData(String key) { * @param completed the node ready to be consumed */ public void reportedCompleted(U completed) { + completed.setPreparer(true); String dependency = completed.key(); for (String dependentKey : graph.get(dependency).dependentKeys()) { DAGNode dependent = graph.get(dependentKey); @@ -126,27 +137,25 @@ public void reportedCompleted(U completed) { } /** - * populate dependents of all nodes. + * Initializes dependents of all nodes. *

* the DAG will be explored in DFS order and all node's dependents will be identified, * this prepares the DAG for traversal using getNext method, each call to getNext returns next node * in the DAG with no dependencies. */ - public void populateDependentKeys() { - this.queue.clear(); + private void initializeDependentKeys() { visit(new Visitor() { + // This 'visit' will be called only once per each node. @Override public void visit(U node) { if (node.dependencyKeys().isEmpty()) { - queue.add(node.key()); return; } String dependentKey = node.key(); for (String dependencyKey : node.dependencyKeys()) { graph.get(dependencyKey) - .dependentKeys() - .add(dependentKey); + .addDependent(dependentKey); } } }); @@ -163,5 +172,8 @@ private void initializeQueue() { this.queue.add(entry.getKey()); } } + if (queue.isEmpty()) { + throw new RuntimeException("Found circular dependency"); + } } } diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java index 3dc61d8065f6e..cbb83f1d2a58f 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java @@ -8,6 +8,7 @@ package com.microsoft.azure; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -57,7 +58,7 @@ public boolean hasChildren() { * @return children (neighbours) of this node */ public List children() { - return this.children; + return Collections.unmodifiableList(this.children); } /** diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java index 29507425d4439..26bbbece2606b 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java @@ -27,11 +27,6 @@ public interface TaskGroup> { */ DAGraph> dag(); - /** - * @return true if this is a root (parent) task group composing other task groups. - */ - boolean isRoot(); - /** * Merges this task group with parent task group. *

@@ -42,6 +37,12 @@ public interface TaskGroup> { */ void merge(TaskGroup parentTaskGroup); + /** + * @return true if the group is responsible for preparing execution of original task in + * this group and all tasks belong other task group it composes. + */ + boolean isPreparer(); + /** * Prepare the graph for execution. */ diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java index 48978f12ca11b..4144a42af39af 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -35,8 +35,8 @@ public DAGraph, DAGNode>> dag() { } @Override - public boolean isRoot() { - return !dag.hasParent(); + public boolean isPreparer() { + return dag.isPreparer(); } @Override @@ -46,7 +46,7 @@ public void merge(TaskGroup> parentTaskGroup) { @Override public void prepare() { - if (isRoot()) { + if (isPreparer()) { dag.prepare(); } } @@ -61,11 +61,7 @@ public void execute() throws Exception { if (dag.isRootNode(nextNode)) { executeRootTask(nextNode.data()); } else { - // TaskGroupBase::execute will be called both in update and create - // scenarios, so run the task only if it not not executed already. - if (nextNode.data().result() == null) { - nextNode.data().execute(this, nextNode); - } + nextNode.data().execute(this, nextNode); } } @@ -79,13 +75,7 @@ public ServiceCall executeAsync(final ServiceCallback callback) { if (dag.isRootNode(nextNode)) { return executeRootTaskAsync(nextNode.data(), callback); } else { - // TaskGroupBase::execute will be called both in update and create - // scenarios, so run the task only if it not not executed already. - if (nextNode.data().result() == null) { - return nextNode.data().executeAsync(this, nextNode, callback); - } else { - return null; - } + return nextNode.data().executeAsync(this, nextNode, callback); } } diff --git a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java index 8c1e12dd7011d..f3b278797e2dd 100644 --- a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java +++ b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java @@ -64,7 +64,7 @@ public void testDAGraphGetNext() { dag.addNode(nodeH); dag.addNode(nodeI); - dag.populateDependentKeys(); + dag.prepare(); DAGNode nextNode = dag.getNext(); int i = 0; while (nextNode != null) { diff --git a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java index e368b04c89db4..985fe306f0f52 100644 --- a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java +++ b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java @@ -71,7 +71,7 @@ public void testDAGraphGetNext() { dag.addNode(nodeH); dag.addNode(nodeI); - dag.populateDependentKeys(); + dag.prepare(); DAGNode nextNode = dag.getNext(); int i = 0; while (nextNode != null) { From cfbafd00f42de46a9aa0c413cb06d4e5ef530da8 Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 22 Jun 2016 11:19:36 -0700 Subject: [PATCH 13/16] comment corrections --- .../src/main/java/com/microsoft/azure/DAGNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java index 890228fff654c..efaf8a8c8cf95 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java @@ -72,7 +72,7 @@ public boolean hasDependencies() { } /** - * Mark or un-mark this node as being preparer. + * Mark or un-mark this node as preparer. * * @param isPreparer true if this node needs to be marked as preparer, false otherwise. */ From 25beef0032b536e2ce686e2290f03931ae45ceb8 Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 22 Jun 2016 16:26:44 -0700 Subject: [PATCH 14/16] Addressing review comments [javadoc, simplfying the map iteration] --- .../src/main/java/com/microsoft/azure/DAGNode.java | 6 +++--- .../src/main/java/com/microsoft/azure/DAGraph.java | 12 ++++++------ .../src/main/java/com/microsoft/azure/Graph.java | 2 +- .../main/java/com/microsoft/azure/TaskGroupBase.java | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java index efaf8a8c8cf95..112130413fd87 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java @@ -40,7 +40,7 @@ List dependentKeys() { } /** - * mark the node identified by the given key as dependent of this node. + * Mark the node identified by the given key as dependent of this node. * * @param key the id of the dependent node */ @@ -56,7 +56,7 @@ public List dependencyKeys() { } /** - * mark the node identified by the given key as this node's dependency. + * Mark the node identified by the given key as this node's dependency. * * @param dependencyKey the id of the dependency node */ @@ -88,7 +88,7 @@ public boolean isPreparer() { } /** - * initialize the node so that traversal can be performed on the parent DAG. + * Initialize the node so that traversal can be performed on the parent DAG. */ public void initialize() { this.toBeResolved = this.dependencyKeys().size(); diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java index 6bb33b0e0b2c7..343e7129e82e6 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -64,7 +64,7 @@ public boolean isPreparer() { /** * Merge this DAG with another DAG. *

- * this will mark this DAG as a child DAG, the dependencies of nodes in this DAG will be merged + * This will mark this DAG as a child DAG, the dependencies of nodes in this DAG will be merged * with (copied to) the parent DAG * * @param parent the parent DAG @@ -86,11 +86,11 @@ public void merge(DAGraph parent) { */ public void prepare() { if (isPreparer()) { - for (Map.Entry entry : graph.entrySet()) { + for (U node : graph.values()) { // Prepare each node for traversal - entry.getValue().initialize(); + node.initialize(); // Mark other sub-DAGs are non-preparer - entry.getValue().setPreparer(false); + node.setPreparer(false); } initializeDependentKeys(); initializeQueue(); @@ -101,7 +101,7 @@ public void prepare() { * Gets next node in the DAG which has no dependency or all of it's dependencies are resolved and * ready to be consumed. *

- * null will be returned when all the nodes are explored + * Null will be returned when all the nodes are explored * * @return next node */ @@ -139,7 +139,7 @@ public void reportedCompleted(U completed) { /** * Initializes dependents of all nodes. *

- * the DAG will be explored in DFS order and all node's dependents will be identified, + * The DAG will be explored in DFS order and all node's dependents will be identified, * this prepares the DAG for traversal using getNext method, each call to getNext returns next node * in the DAG with no dependencies. */ diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java index 0cf8e2a7d231b..40ceebaa50b2b 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java @@ -15,7 +15,7 @@ /** * Type representing a directed graph data structure. *

- * each node in a graph is represented by {@link Node} + * Each node in a graph is represented by {@link Node} * * @param the type of the data stored in the graph's nodes * @param the type of the nodes in the graph diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java index 4144a42af39af..efc8d30e491bc 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -85,9 +85,9 @@ public T taskResult(String taskId) { } /** - * executes the root task in this group. + * Executes the root task in this group. *

- * this method will be invoked when all the task dependencies of the root task are finished + * This method will be invoked when all the task dependencies of the root task are finished * executing, at this point root task can be executed by consuming the result of tasks it * depends on. * @@ -97,9 +97,9 @@ public T taskResult(String taskId) { public abstract void executeRootTask(TaskItem task) throws Exception; /** - * executes the root task in this group asynchronously. + * Executes the root task in this group asynchronously. *

- * this method will be invoked when all the task dependencies of the root task are finished + * This method will be invoked when all the task dependencies of the root task are finished * executing, at this point root task can be executed by consuming the result of tasks it * depends on. * From 1d5e9c1901cd167376bbe93f3db06363a859091a Mon Sep 17 00:00:00 2001 From: anuchan Date: Wed, 22 Jun 2016 16:45:06 -0700 Subject: [PATCH 15/16] improving comment as per review --- .../src/main/java/com/microsoft/azure/DAGraph.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java index 343e7129e82e6..9ec33a692f1aa 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -100,10 +100,8 @@ public void prepare() { /** * Gets next node in the DAG which has no dependency or all of it's dependencies are resolved and * ready to be consumed. - *

- * Null will be returned when all the nodes are explored * - * @return next node + * @return next node or null if all the nodes have been explored */ public U getNext() { return graph.get(queue.poll()); From cf2fdb582fb504ea5f772ed03cffb07662c8c626 Mon Sep 17 00:00:00 2001 From: anuchan Date: Sat, 25 Jun 2016 16:02:07 -0700 Subject: [PATCH 16/16] Adding publicIp sample, only mark sub-DAGs as non-prepare - the root DAG stay as preparer --- .../src/main/java/com/microsoft/azure/DAGraph.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java index 9ec33a692f1aa..58179e0152ed4 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -89,8 +89,10 @@ public void prepare() { for (U node : graph.values()) { // Prepare each node for traversal node.initialize(); - // Mark other sub-DAGs are non-preparer - node.setPreparer(false); + if (!this.isRootNode(node)) { + // Mark other sub-DAGs as non-preparer + node.setPreparer(false); + } } initializeDependentKeys(); initializeQueue();