diff --git a/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java b/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java index 163d10f60a1d..384ab3f7671a 100644 --- a/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java +++ b/azure-client-authentication/src/main/java/com/microsoft/azure/credentials/ApplicationTokenCredentials.java @@ -113,13 +113,14 @@ public ApplicationTokenCredentials withDefaultSubscriptionId(String subscription * * @param credentialsFile A file with credentials, using the standard Java properties format. * and the following keys: - * subscription= - * tenant= - * client= - * key= - * managementURI= - * baseURL= - * authURL= + * subscription=<subscription-id> + * tenant=<tenant-id> + * client=<client-id> + * key=<client-key> + * managementURI=<management-URI> + * baseURL=<base-URL> + * authURL=<authentication-URL> + * * @return The credentials based on the file. * @throws IOException exception thrown from file access errors. */ diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java index 517a2c3205ac..04a6af99d93d 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/AzureEnvironment.java @@ -68,7 +68,25 @@ public AzureEnvironment( "https://login.chinacloudapi.cn/", "https://management.core.chinacloudapi.cn/", true, - "https://management.chinacloudapi.cn"); + "https://management.chinacloudapi.cn/"); + + /** + * Provides the settings for authentication with Azure US Government. + */ + public static final AzureEnvironment AZURE_US_GOVERNMENT = new AzureEnvironment( + "https://login.microsoftonline.com/", + "https://management.core.usgovcloudapi.net/", + true, + "https://management.usgovcloudapi.net/"); + + /** + * Provides the settings for authentication with Azure Germany. + */ + public static final AzureEnvironment AZURE_GERMANY = new AzureEnvironment( + "https://login.microsoftonline.de/", + "https://management.core.cloudapi.de/", + true, + "https://management.microsoftazure.de/"); /** * Gets the base URL of the management service. diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java index ccc700618e58..112130413fd8 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGNode.java @@ -8,6 +8,7 @@ package com.microsoft.azure; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -18,6 +19,7 @@ public class DAGNode extends Node { private List dependentKeys; private int toBeResolved; + private boolean isPreparer; /** * Creates a DAG node. @@ -34,11 +36,11 @@ public DAGNode(String key, T data) { * @return a list of keys of nodes in {@link DAGraph} those are dependents on this node */ List dependentKeys() { - return this.dependentKeys; + return Collections.unmodifiableList(this.dependentKeys); } /** - * mark the node identified by the given key as dependent of this node. + * Mark the node identified by the given key as dependent of this node. * * @param key the id of the dependent node */ @@ -54,12 +56,11 @@ public List dependencyKeys() { } /** - * mark the node identified by the given key as this node's dependency. + * Mark the node identified by the given key as this node's dependency. * * @param dependencyKey the id of the dependency node */ public void addDependency(String dependencyKey) { - toBeResolved++; super.addChild(dependencyKey); } @@ -70,6 +71,30 @@ public boolean hasDependencies() { return this.hasChildren(); } + /** + * Mark or un-mark this node as preparer. + * + * @param isPreparer true if this node needs to be marked as preparer, false otherwise. + */ + public void setPreparer(boolean isPreparer) { + this.isPreparer = isPreparer; + } + + /** + * @return true if this node is marked as preparer + */ + public boolean isPreparer() { + return isPreparer; + } + + /** + * Initialize the node so that traversal can be performed on the parent DAG. + */ + public void initialize() { + this.toBeResolved = this.dependencyKeys().size(); + this.dependentKeys.clear(); + } + /** * @return true if all dependencies of this node are ready to be consumed */ diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java index 8b10bde8b3e3..58179e0152ed 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/DAGraph.java @@ -32,6 +32,7 @@ public class DAGraph> extends Graph { public DAGraph(U rootNode) { this.rootNode = rootNode; this.queue = new ArrayDeque<>(); + this.rootNode.setPreparer(true); this.addNode(rootNode); } @@ -52,10 +53,18 @@ public boolean isRootNode(U node) { return this.rootNode == node; } + /** + * @return true if this dag is the preparer responsible for + * preparing the DAG for traversal. + */ + public boolean isPreparer() { + return this.rootNode.isPreparer(); + } + /** * Merge this DAG with another DAG. *

- * this will mark this DAG as a child DAG, the dependencies of nodes in this DAG will be merged + * This will mark this DAG as a child DAG, the dependencies of nodes in this DAG will be merged * with (copied to) the parent DAG * * @param parent the parent DAG @@ -63,7 +72,6 @@ public boolean isRootNode(U node) { public void merge(DAGraph parent) { this.hasParent = true; parent.rootNode.addDependency(this.rootNode.key()); - this.rootNode.addDependent(parent.rootNode.key()); for (Map.Entry entry: graph.entrySet()) { String key = entry.getKey(); if (!parent.graph.containsKey(key)) { @@ -77,19 +85,25 @@ public void merge(DAGraph parent) { * in the DAG with no dependencies. */ public void prepare() { - initializeQueue(); - if (queue.isEmpty()) { - throw new RuntimeException("Found circular dependency"); + if (isPreparer()) { + for (U node : graph.values()) { + // Prepare each node for traversal + node.initialize(); + if (!this.isRootNode(node)) { + // Mark other sub-DAGs as non-preparer + node.setPreparer(false); + } + } + initializeDependentKeys(); + initializeQueue(); } } /** * Gets next node in the DAG which has no dependency or all of it's dependencies are resolved and * ready to be consumed. - *

- * null will be returned when all the nodes are explored * - * @return next node + * @return next node or null if all the nodes have been explored */ public U getNext() { return graph.get(queue.poll()); @@ -111,6 +125,7 @@ public T getNodeData(String key) { * @param completed the node ready to be consumed */ public void reportedCompleted(U completed) { + completed.setPreparer(true); String dependency = completed.key(); for (String dependentKey : graph.get(dependency).dependentKeys()) { DAGNode dependent = graph.get(dependentKey); @@ -122,27 +137,25 @@ public void reportedCompleted(U completed) { } /** - * populate dependents of all nodes. + * Initializes dependents of all nodes. *

- * the DAG will be explored in DFS order and all node's dependents will be identified, + * The DAG will be explored in DFS order and all node's dependents will be identified, * this prepares the DAG for traversal using getNext method, each call to getNext returns next node * in the DAG with no dependencies. */ - public void populateDependentKeys() { - this.queue.clear(); + private void initializeDependentKeys() { visit(new Visitor() { + // This 'visit' will be called only once per each node. @Override public void visit(U node) { if (node.dependencyKeys().isEmpty()) { - queue.add(node.key()); return; } String dependentKey = node.key(); for (String dependencyKey : node.dependencyKeys()) { graph.get(dependencyKey) - .dependentKeys() - .add(dependentKey); + .addDependent(dependentKey); } } }); @@ -159,5 +172,8 @@ private void initializeQueue() { this.queue.add(entry.getKey()); } } + if (queue.isEmpty()) { + throw new RuntimeException("Found circular dependency"); + } } } diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java index 0cf8e2a7d231..40ceebaa50b2 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Graph.java @@ -15,7 +15,7 @@ /** * Type representing a directed graph data structure. *

- * each node in a graph is represented by {@link Node} + * Each node in a graph is represented by {@link Node} * * @param the type of the data stored in the graph's nodes * @param the type of the nodes in the graph diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java b/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java index 3dc61d8065f6..cbb83f1d2a58 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/Node.java @@ -8,6 +8,7 @@ package com.microsoft.azure; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -57,7 +58,7 @@ public boolean hasChildren() { * @return children (neighbours) of this node */ public List children() { - return this.children; + return Collections.unmodifiableList(this.children); } /** diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java b/azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java new file mode 100644 index 000000000000..a145fecc6795 --- /dev/null +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/ResourceGetExponentialBackoffRetryStrategy.java @@ -0,0 +1,48 @@ +/** + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + * + */ + +package com.microsoft.azure; + +import com.microsoft.rest.retry.RetryStrategy; +import okhttp3.Response; + +/** + * A retry strategy with backoff parameters for calculating the exponential + * delay between retries for 404s from GET calls. + */ +public class ResourceGetExponentialBackoffRetryStrategy extends RetryStrategy { + /** + * Represents the default number of retries. + */ + private static final int DEFAULT_NUMBER_OF_ATTEMPTS = 3; + + /** + * Creates an instance of the retry strategy. + */ + public ResourceGetExponentialBackoffRetryStrategy() { + this(null, DEFAULT_FIRST_FAST_RETRY); + } + + /** + * Initializes a new instance of the {@link RetryStrategy} class. + * + * @param name The name of the retry strategy. + * @param firstFastRetry true to immediately retry in the first attempt; otherwise, false. + */ + private ResourceGetExponentialBackoffRetryStrategy(String name, boolean firstFastRetry) { + super(name, firstFastRetry); + } + + @Override + public boolean shouldRetry(int retryCount, Response response) { + int code = response.code(); + //CHECKSTYLE IGNORE MagicNumber FOR NEXT 2 LINES + return retryCount < DEFAULT_NUMBER_OF_ATTEMPTS + && code == 404 + && response.request().method().equalsIgnoreCase("GET"); + } +} diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java b/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java index 2517f890a52c..0ace881f0ee5 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/RestClient.java @@ -14,17 +14,20 @@ import com.microsoft.rest.credentials.ServiceClientCredentials; import com.microsoft.rest.retry.RetryHandler; import com.microsoft.rest.serializer.JacksonMapperAdapter; - -import java.lang.reflect.Field; -import java.net.CookieManager; -import java.net.CookiePolicy; - +import okhttp3.ConnectionPool; import okhttp3.Interceptor; import okhttp3.JavaNetCookieJar; import okhttp3.OkHttpClient; import okhttp3.logging.HttpLoggingInterceptor; import retrofit2.Retrofit; +import java.lang.reflect.Field; +import java.net.CookieManager; +import java.net.CookiePolicy; +import java.net.Proxy; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + /** * An instance of this class stores the client information for making REST calls. */ @@ -260,6 +263,63 @@ public Buildable withInterceptor(Interceptor interceptor) { return this; } + /** + * Set the read timeout on the HTTP client. Default is 10 seconds. + * + * @param timeout the timeout numeric value + * @param unit the time unit for the numeric value + * @return the builder itself for chaining + */ + public Buildable withReadTimeout(long timeout, TimeUnit unit) { + httpClientBuilder.readTimeout(timeout, unit); + return this; + } + + /** + * Set the connection timeout on the HTTP client. Default is 10 seconds. + * + * @param timeout the timeout numeric value + * @param unit the time unit for the numeric value + * @return the builder itself for chaining + */ + public Buildable withConnectionTimeout(long timeout, TimeUnit unit) { + httpClientBuilder.connectTimeout(timeout, unit); + return this; + } + + /** + * Set the maximum idle connections for the HTTP client. Default is 5. + * + * @param maxIdleConnections the maximum idle connections + * @return the builder itself for chaining + */ + public Buildable withMaxIdleConnections(int maxIdleConnections) { + httpClientBuilder.connectionPool(new ConnectionPool(maxIdleConnections, 5, TimeUnit.MINUTES)); + return this; + } + + /** + * Sets the executor for async callbacks to run on. + * + * @param executor the executor to execute the callbacks. + * @return the builder itself for chaining + */ + public Buildable withCallbackExecutor(Executor executor) { + retrofitBuilder.callbackExecutor(executor); + return this; + } + + /** + * Sets the proxy for the HTTP client. + * + * @param proxy the proxy to use + * @return the builder itself for chaining + */ + public Buildable withProxy(Proxy proxy) { + httpClientBuilder.proxy(proxy); + return this; + } + /** * Build a RestClient with all the current configurations. * @@ -270,6 +330,7 @@ public RestClient build() { OkHttpClient httpClient = httpClientBuilder .addInterceptor(baseUrlHandler) .addInterceptor(customHeadersInterceptor) + .addInterceptor(new RetryHandler(new ResourceGetExponentialBackoffRetryStrategy())) .addInterceptor(new RetryHandler()) .build(); return new RestClient(httpClient, diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java index 94b4628ff70a..26bbbece2606 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroup.java @@ -7,6 +7,9 @@ package com.microsoft.azure; +import com.microsoft.rest.ServiceCall; +import com.microsoft.rest.ServiceCallback; + /** * Represents a group of related tasks. *

@@ -24,11 +27,6 @@ public interface TaskGroup> { */ DAGraph> dag(); - /** - * @return true if this is a root (parent) task group composing other task groups. - */ - boolean isRoot(); - /** * Merges this task group with parent task group. *

@@ -39,6 +37,17 @@ public interface TaskGroup> { */ void merge(TaskGroup parentTaskGroup); + /** + * @return true if the group is responsible for preparing execution of original task in + * this group and all tasks belong other task group it composes. + */ + boolean isPreparer(); + + /** + * Prepare the graph for execution. + */ + void prepare(); + /** * Executes the tasks in the group. *

@@ -48,6 +57,14 @@ public interface TaskGroup> { */ void execute() throws Exception; + /** + * Executes the tasks in the group asynchronously. + * + * @param callback the callback to call on failure or success + * @return the handle to the REST call + */ + ServiceCall executeAsync(ServiceCallback callback); + /** * Gets the result of execution of a task in the group. *

diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java index d0dc430edc3a..efc8d30e491b 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskGroupBase.java @@ -7,15 +7,17 @@ package com.microsoft.azure; +import com.microsoft.rest.ServiceCall; +import com.microsoft.rest.ServiceCallback; + /** * The base implementation of TaskGroup interface. * * @param the result type of the tasks in the group - * @param type representing task in the group */ -public abstract class TaskGroupBase> - implements TaskGroup { - private DAGraph> dag; +public abstract class TaskGroupBase + implements TaskGroup> { + private DAGraph, DAGNode>> dag; /** * Creates TaskGroupBase. @@ -23,39 +25,57 @@ public abstract class TaskGroupBase> * @param rootTaskItemId the id of the root task in this task group * @param rootTaskItem the root task */ - public TaskGroupBase(String rootTaskItemId, U rootTaskItem) { + public TaskGroupBase(String rootTaskItemId, TaskItem rootTaskItem) { this.dag = new DAGraph<>(new DAGNode<>(rootTaskItemId, rootTaskItem)); } @Override - public DAGraph> dag() { + public DAGraph, DAGNode>> dag() { return dag; } @Override - public boolean isRoot() { - return !dag.hasParent(); + public boolean isPreparer() { + return dag.isPreparer(); } @Override - public void merge(TaskGroup parentTaskGroup) { + public void merge(TaskGroup> parentTaskGroup) { dag.merge(parentTaskGroup.dag()); } @Override - public void execute() throws Exception { - if (isRoot()) { + public void prepare() { + if (isPreparer()) { dag.prepare(); - DAGNode nextNode = dag.getNext(); - while (nextNode != null) { - if (dag.isRootNode(nextNode)) { - executeRootTask(nextNode.data()); - } else { - nextNode.data().execute(); - } - dag.reportedCompleted(nextNode); - nextNode = dag.getNext(); - } + } + } + + @Override + public void execute() throws Exception { + DAGNode> nextNode = dag.getNext(); + if (nextNode == null) { + return; + } + + if (dag.isRootNode(nextNode)) { + executeRootTask(nextNode.data()); + } else { + nextNode.data().execute(this, nextNode); + } + } + + @Override + public ServiceCall executeAsync(final ServiceCallback callback) { + final DAGNode> nextNode = dag.getNext(); + if (nextNode == null) { + return null; + } + + if (dag.isRootNode(nextNode)) { + return executeRootTaskAsync(nextNode.data(), callback); + } else { + return nextNode.data().executeAsync(this, nextNode, callback); } } @@ -65,14 +85,27 @@ public T taskResult(String taskId) { } /** - * executes the root task in this group. + * Executes the root task in this group. *

- * this method will be invoked when all the task dependencies of the root task are finished + * This method will be invoked when all the task dependencies of the root task are finished * executing, at this point root task can be executed by consuming the result of tasks it * depends on. * * @param task the root task in this group * @throws Exception the exception */ - public abstract void executeRootTask(U task) throws Exception; + public abstract void executeRootTask(TaskItem task) throws Exception; + + /** + * Executes the root task in this group asynchronously. + *

+ * This method will be invoked when all the task dependencies of the root task are finished + * executing, at this point root task can be executed by consuming the result of tasks it + * depends on. + * + * @param task the root task in this group + * @param callback the callback when the task fails or succeeds + * @return the handle to the REST call + */ + public abstract ServiceCall executeRootTaskAsync(TaskItem task, ServiceCallback callback); } diff --git a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java index 7df830f69bb9..fb74c23845c3 100644 --- a/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java +++ b/azure-client-runtime/src/main/java/com/microsoft/azure/TaskItem.java @@ -7,6 +7,9 @@ package com.microsoft.azure; +import com.microsoft.rest.ServiceCall; +import com.microsoft.rest.ServiceCallback; + /** * Type representing a task in a task group {@link TaskGroup}. * @@ -22,7 +25,22 @@ public interface TaskItem { * Executes the task. *

* once executed the result will be available through result getter + * + * @param taskGroup the task group dispatching tasks + * @param node the node the task item is associated with * @throws Exception exception */ - void execute() throws Exception; + void execute(TaskGroup> taskGroup, DAGNode> node) throws Exception; + + /** + * Executes the task asynchronously. + *

+ * once executed the result will be available through result getter + + * @param taskGroup the task group dispatching tasks + * @param node the node the task item is associated with + * @param callback callback to call on success or failure + * @return the handle of the REST call + */ + ServiceCall executeAsync(TaskGroup> taskGroup, DAGNode> node, ServiceCallback callback); } diff --git a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java index 8c1e12dd7011..f3b278797e2d 100644 --- a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java +++ b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTest.java @@ -64,7 +64,7 @@ public void testDAGraphGetNext() { dag.addNode(nodeH); dag.addNode(nodeI); - dag.populateDependentKeys(); + dag.prepare(); DAGNode nextNode = dag.getNext(); int i = 0; while (nextNode != null) { diff --git a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java index e368b04c89db..985fe306f0f5 100644 --- a/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java +++ b/azure-client-runtime/src/test/java/com/microsoft/azure/DAGraphTests.java @@ -71,7 +71,7 @@ public void testDAGraphGetNext() { dag.addNode(nodeH); dag.addNode(nodeI); - dag.populateDependentKeys(); + dag.prepare(); DAGNode nextNode = dag.getNext(); int i = 0; while (nextNode != null) {