From e4d32d549b3bf29a896ae0ec31f2d88cbcb32377 Mon Sep 17 00:00:00 2001 From: Nikolas Falco Date: Thu, 2 Jan 2025 21:16:34 +0100 Subject: [PATCH] [JENKINS-64418] Add exponential backoff to BitBucket rate limit retry loop (#927) Configure Apache HTTP client to use an exponential backoff retry strategy. Move methods common to both server and cloud client to the abstract class. --- pom.xml | 12 + .../bitbucket/BitbucketDefaultBranch.java | 3 +- .../bitbucket/BitbucketGitSCMBuilder.java | 4 +- .../bitbucket/api/BitbucketProject.java | 3 - .../client/BitbucketCloudApiClient.java | 246 ++---------- .../impl/client/AbstractBitbucketApi.java | 209 ++++++++++- ...ackOffServiceUnavailableRetryStrategy.java | 114 ++++++ .../client/BitbucketServerAPIClient.java | 352 ++++-------------- .../BitbucketIntegrationClientFactory.java | 17 +- .../hooks/WebhookConfigurationTest.java | 23 ++ ...ffServiceUnavailableRetryStrategyTest.java | 96 +++++ .../client/BitbucketServerAPIClientTest.java | 59 ++- ...wse-Jenkinsfile_at_feature_2Fpipeline.json | 3 + ...der-Jenkinsfile_at_feature_2Fpipeline.json | 3 + 14 files changed, 594 insertions(+), 550 deletions(-) create mode 100644 src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java create mode 100644 src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java create mode 100644 src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json create mode 100644 src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json diff --git a/pom.xml b/pom.xml index 3ffbfaa62..e2f9e4e99 100644 --- a/pom.xml +++ b/pom.xml @@ -167,6 +167,18 @@ scribejava-core 8.3.3 + + org.mock-server + mockserver-junit-jupiter + 5.15.0 + test + + + javax.servlet + javax.servlet-api + + + diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketDefaultBranch.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketDefaultBranch.java index c678c4e40..f9d1510c5 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketDefaultBranch.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketDefaultBranch.java @@ -33,7 +33,8 @@ * Represents the default branch of a specific repository */ public class BitbucketDefaultBranch extends InvisibleAction implements Serializable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1826270778226063782L; + @NonNull private final String repoOwner; @NonNull diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketGitSCMBuilder.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketGitSCMBuilder.java index ae532997e..babed5d58 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketGitSCMBuilder.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/BitbucketGitSCMBuilder.java @@ -214,8 +214,8 @@ private void withPullRequestRemote(PullRequestSCMHead head, String headName) { String scmSourceRepository = scmSource.getRepository(); String pullRequestRepoOwner = head.getRepoOwner(); String pullRequestRepository = head.getRepository(); - boolean prFromTargetRepository = pullRequestRepoOwner.equals(scmSourceRepoOwner) - && pullRequestRepository.equals(scmSourceRepository); + boolean prFromTargetRepository = pullRequestRepoOwner.equalsIgnoreCase(scmSourceRepoOwner) + && pullRequestRepository.equalsIgnoreCase(scmSourceRepository); SCMRevision revision = revision(); ChangeRequestCheckoutStrategy checkoutStrategy = head.getCheckoutStrategy(); // PullRequestSCMHead should be refactored to add references to target and source commit hashes. diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketProject.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketProject.java index ae151cef0..a5def38fe 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketProject.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketProject.java @@ -1,8 +1,5 @@ package com.cloudbees.jenkins.plugins.bitbucket.api; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -@JsonIgnoreProperties(ignoreUnknown = true) public class BitbucketProject { private String key; diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java index 09e363b26..d1d5a7752 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java @@ -32,7 +32,6 @@ import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketException; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketPullRequest; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRepository; -import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketTeam; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketWebHook; import com.cloudbees.jenkins.plugins.bitbucket.avatars.AvatarCacheSource.AvatarImage; @@ -64,9 +63,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -76,33 +72,14 @@ import java.util.logging.Level; import javax.imageio.ImageIO; import jenkins.scm.api.SCMFile; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.SocketConfig; import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.ProtectedExternally; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; @@ -116,13 +93,13 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit // Limit images to 16k private static final int MAX_AVATAR_LENGTH = 16384; private static final int MAX_PAGE_LENGTH = 100; - private static final HttpClientConnectionManager connectionManager = getConnectionManager(); - private CloseableHttpClient client; + protected static final HttpClientConnectionManager connectionManager = connectionManager(); + + private final CloseableHttpClient client; private final String owner; private final String projectKey; private final String repositoryName; private final boolean enableCache; - private final BitbucketAuthenticator authenticator; private static final Cache cachedTeam = new Cache<>(6, HOURS); private static final Cache cachedAvatar = new Cache<>(6, HOURS); private static final Cache> cachedRepositories = new Cache<>(3, HOURS); @@ -130,7 +107,7 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit private transient BitbucketRepository cachedRepository; private transient String cachedDefaultBranch; - private static HttpClientConnectionManager getConnectionManager() { + private static HttpClientConnectionManager connectionManager() { try { PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); // NOSONAR connManager.setDefaultMaxPerRoute(20); @@ -166,7 +143,7 @@ public BitbucketCloudApiClient(boolean enableCache, int teamCacheDuration, int r public BitbucketCloudApiClient(boolean enableCache, int teamCacheDuration, int repositoriesCacheDuration, String owner, String projectKey, String repositoryName, BitbucketAuthenticator authenticator) { - this.authenticator = authenticator; + super(authenticator); this.owner = owner; this.projectKey = projectKey; this.repositoryName = repositoryName; @@ -175,32 +152,7 @@ public BitbucketCloudApiClient(boolean enableCache, int teamCacheDuration, int r cachedTeam.setExpireDuration(teamCacheDuration, MINUTES); cachedRepositories.setExpireDuration(repositoriesCacheDuration, MINUTES); } - - // Create Http client - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() - .setConnectionManager(connectionManager) - .setConnectionManagerShared(connectionManager != null) - .setRetryHandler(new StandardHttpRequestRetryHandler()); - - if (authenticator != null) { - authenticator.configureBuilder(httpClientBuilder); - - context = HttpClientContext.create(); - authenticator.configureContext(context, API_HOST); - } - - setClientProxyParams("bitbucket.org", httpClientBuilder); - - this.client = httpClientBuilder.build(); - } - - @Override - protected void finalize() throws Throwable { - if (client != null) { - client.close(); - } - - super.finalize(); + this.client = super.setupClientBuilder("bitbucket.org").build(); } /** @@ -311,8 +263,8 @@ private void setupClosureForPRBranch(BitbucketPullRequestValue pullRequest) { @Deprecated @CheckForNull public String getLogin() { - if (authenticator != null) { - return authenticator.getId(); + if (getAuthenticator() != null) { + return getAuthenticator().getId(); } return null; } @@ -757,7 +709,7 @@ public AvatarImage getTeamAvatar() throws IOException, InterruptedException { /** * The role parameter only makes sense when the request is authenticated, so - * if there is no auth information ({@link #authenticator}) the role will be omitted. + * if there is no auth information ({@link #getAuthenticator()}) the role will be omitted. */ @NonNull @Override @@ -766,8 +718,8 @@ public List getRepositories(@CheckForNull UserRoleInRe StringBuilder cacheKey = new StringBuilder(); cacheKey.append(owner); - if (authenticator != null) { - cacheKey.append("::").append(authenticator.getId()); + if (getAuthenticator() != null) { + cacheKey.append("::").append(getAuthenticator().getId()); } else { cacheKey.append("::"); } @@ -781,7 +733,7 @@ public List getRepositories(@CheckForNull UserRoleInRe } else { cacheKey.append("::"); } - if (role != null && authenticator != null) { + if (role != null && getAuthenticator() != null) { template.set("role", role.getId()); cacheKey.append("::").append(role.getId()); } else { @@ -824,94 +776,7 @@ public List getRepositories() throws IOException, Inte return getRepositories(null); } - @Restricted(ProtectedExternally.class) - protected CloseableHttpResponse executeMethod(HttpRequestBase httpMethod) throws InterruptedException, IOException { - return executeMethod(API_HOST, httpMethod); - } - - @Restricted(ProtectedExternally.class) - protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws InterruptedException, IOException { - HttpClientContext requestContext = null; - if (API_HOST.equals(host)) { - requestContext = context; - if (authenticator != null) { - authenticator.configureRequest(httpMethod); - } - } - - RequestConfig.Builder requestConfig = RequestConfig.custom(); - String connectTimeout = System.getProperty("http.connect.timeout", "10"); - requestConfig.setConnectTimeout(Integer.parseInt(connectTimeout) * 1000); - String connectionRequestTimeout = System.getProperty("http.connect.request.timeout", "60"); - requestConfig.setConnectionRequestTimeout(Integer.parseInt(connectionRequestTimeout) * 1000); - String socketTimeout = System.getProperty("http.socket.timeout", "60"); - requestConfig.setSocketTimeout(Integer.parseInt(socketTimeout) * 1000); - httpMethod.setConfig(requestConfig.build()); - - CloseableHttpResponse response = client.execute(host, httpMethod, requestContext); - while (response.getStatusLine().getStatusCode() == API_RATE_LIMIT_STATUS_CODE) { - release(httpMethod); - if (Thread.interrupted()) { - throw new InterruptedException(); - } - /* - TODO: When bitbucket starts supporting rate limit expiration time, remove 5 sec wait and put code - to wait till expiration time is over. It should also fix the wait for ever loop. - */ - logger.fine("Bitbucket Cloud API rate limit reached, sleeping for 5 sec then retry..."); - Thread.sleep(5000); - response = client.execute(host, httpMethod, requestContext); - } - return response; - } - - /** - * Caller's responsible to close the InputStream. - */ - private InputStream getRequestAsInputStream(String path) throws IOException, InterruptedException { - HttpGet httpget = new HttpGet(path); - HttpHost host = null; - - // Extract host from URL, if present - try { - URI uri = new URI(path); - if (uri.isAbsolute() && ! uri.isOpaque()) { - host = HttpHost.create(""+uri.getScheme()+"://"+uri.getAuthority()); - } - } catch (URISyntaxException ex) { - } - // Use default API Host otherwise - if (host == null) { - host = API_HOST; - } - - try { - CloseableHttpResponse response = executeMethod(host, httpget); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NOT_FOUND) { - EntityUtils.consume(response.getEntity()); - response.close(); - throw new FileNotFoundException("URL: " + path); - } - if (statusCode != HttpStatus.SC_OK) { - String content = getResponseContent(response); - throw buildResponseException(response, content); - } - return new ClosingConnectionInputStream(response, httpget, connectionManager); - } catch (BitbucketRequestException | FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } - } - - private String getRequest(String path) throws IOException, InterruptedException { - try (InputStream inputStream = getRequestAsInputStream(path)){ - return IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - } - - private BufferedImage getImageRequest(String path) throws IOException, InterruptedException { + private BufferedImage getImageRequest(String path) throws IOException { try (InputStream inputStream = getRequestAsInputStream(path)) { int length = MAX_AVATAR_LENGTH; BufferedInputStream bis = new BufferedInputStream(inputStream, length); @@ -919,85 +784,24 @@ private BufferedImage getImageRequest(String path) throws IOException, Interrupt } } - private int headRequestStatus(String path) throws IOException, InterruptedException { - HttpHead httpHead = new HttpHead(path); - try(CloseableHttpResponse response = executeMethod(httpHead)) { - EntityUtils.consume(response.getEntity()); - return response.getStatusLine().getStatusCode(); - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } finally { - release(httpHead); - } - } - - private void deleteRequest(String path) throws IOException, InterruptedException { - HttpDelete httppost = new HttpDelete(path); - try(CloseableHttpResponse response = executeMethod(httppost)) { - EntityUtils.consume(response.getEntity()); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NOT_FOUND) { - throw new FileNotFoundException("URL: " + path); - } - if (statusCode != HttpStatus.SC_NO_CONTENT) { - throw buildResponseException(response, getResponseContent(response)); - } - } catch (BitbucketRequestException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } finally { - release(httppost); - } - } - - private String doRequest(HttpRequestBase request) throws IOException, InterruptedException { - try(CloseableHttpResponse response = executeMethod(request)) { - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NO_CONTENT) { - EntityUtils.consume(response.getEntity()); - // 204, no content - return ""; - } - String content = getResponseContent(response); - EntityUtils.consume(response.getEntity()); - if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { - throw buildResponseException(response, content); - } - return content; - } catch (BitbucketRequestException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + request, e); - } finally { - release(request); - } - } - - private void release(HttpRequestBase method) { - method.releaseConnection(); - connectionManager.closeExpiredConnections(); - } - - private String putRequest(String path, String content) throws IOException, InterruptedException { - HttpPut request = new HttpPut(path); - request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return doRequest(request); + @Override + protected HttpClientConnectionManager getConnectionManager() { + return connectionManager; } - private String postRequest(String path, String content) throws IOException, InterruptedException { - HttpPost httppost = new HttpPost(path); - httppost.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return doRequest(httppost); + @NonNull + @Override + protected HttpHost getHost() { + return API_HOST; } - private String postRequest(String path, List params) throws IOException, InterruptedException { - HttpPost httppost = new HttpPost(path); - httppost.setEntity(new UrlEncodedFormEntity(params)); - return doRequest(httppost); + @NonNull + @Override + protected CloseableHttpClient getClient() { + return client; } - private List getAllBranches(String response) throws IOException, InterruptedException { + private List getAllBranches(String response) throws IOException { List branches = new ArrayList<>(); BitbucketCloudPage page = JsonParser.mapper.readValue(response, new TypeReference>(){}); diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java index a61e13806..8285328f9 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java @@ -23,40 +23,69 @@ */ package com.cloudbees.jenkins.plugins.bitbucket.impl.client; +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticator; import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketRequestException; +import com.cloudbees.jenkins.plugins.bitbucket.client.ClosingConnectionInputStream; import edu.umd.cs.findbugs.annotations.CheckForNull; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import hudson.ProxyConfiguration; import hudson.util.Secret; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.net.Proxy; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import jenkins.model.Jenkins; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.util.EntityUtils; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.ProtectedExternally; @Restricted(ProtectedExternally.class) -public abstract class AbstractBitbucketApi { - protected static final int API_RATE_LIMIT_STATUS_CODE = 429; - +public abstract class AbstractBitbucketApi implements AutoCloseable { protected final Logger logger = Logger.getLogger(this.getClass().getName()); - protected HttpClientContext context; + private final BitbucketAuthenticator authenticator; + private HttpClientContext context; + + protected AbstractBitbucketApi(BitbucketAuthenticator authenticator) { + this.authenticator = authenticator; + } protected String truncateMiddle(@CheckForNull String value, int maxLength) { int length = StringUtils.length(value); @@ -109,7 +138,39 @@ private long getLenghtFromHeader(CloseableHttpResponse response) { return len; } - protected void setClientProxyParams(String host, HttpClientBuilder builder) { + protected HttpClientBuilder setupClientBuilder(@Nullable String host) { + int connectTimeout = Integer.getInteger("http.connect.timeout", 10); + int connectionRequestTimeout = Integer.getInteger("http.connect.request.timeout", 60); + int socketTimeout = Integer.getInteger("http.socket.timeout", 60); + + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(connectTimeout * 1000) + .setConnectionRequestTimeout(connectionRequestTimeout * 1000) + .setSocketTimeout(socketTimeout * 1000) + .build(); + + HttpClientConnectionManager connectionManager = getConnectionManager(); + ServiceUnavailableRetryStrategy serviceUnavailableStrategy = new ExponentialBackOffServiceUnavailableRetryStrategy(2, TimeUnit.SECONDS.toMillis(5), TimeUnit.HOURS.toMillis(1)); + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .useSystemProperties() + .setConnectionManager(connectionManager) + .setConnectionManagerShared(connectionManager != null) + .setServiceUnavailableRetryStrategy(serviceUnavailableStrategy) + .setRetryHandler(new StandardHttpRequestRetryHandler()) + .setDefaultRequestConfig(config) + .disableCookieManagement(); + + if (authenticator != null) { + authenticator.configureBuilder(httpClientBuilder); + + context = HttpClientContext.create(); + authenticator.configureContext(context, getHost()); + } + setClientProxyParams(host, httpClientBuilder); + return httpClientBuilder; + } + + private void setClientProxyParams(String host, HttpClientBuilder builder) { Jenkins jenkins = Jenkins.getInstanceOrNull(); // because unit test ProxyConfiguration proxyConfig = jenkins != null ? jenkins.proxy : null; @@ -150,4 +211,142 @@ protected void setClientProxyParams(String host, HttpClientBuilder builder) { } } + @CheckForNull + protected abstract HttpClientConnectionManager getConnectionManager(); + + @NonNull + protected abstract HttpHost getHost(); + + @NonNull + protected abstract CloseableHttpClient getClient(); + + /* for test purpose */ + protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { + if (authenticator != null) { + authenticator.configureRequest(httpMethod); + } + return getClient().execute(host, httpMethod, context); + } + + protected String doRequest(HttpRequestBase request) throws IOException { + try (CloseableHttpResponse response = executeMethod(getHost(), request)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_NOT_FOUND) { + throw new FileNotFoundException("URL: " + request.getURI()); + } + if (statusCode == HttpStatus.SC_NO_CONTENT) { + EntityUtils.consume(response.getEntity()); + // 204, no content + return ""; + } + String content = getResponseContent(response); + EntityUtils.consume(response.getEntity()); + if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { + throw buildResponseException(response, content); + } + return content; + } catch (BitbucketRequestException e) { + throw e; + } catch (IOException e) { + throw new IOException("Communication error for url: " + request, e); + } finally { + release(request); + } + } + + private void release(HttpRequestBase method) { + method.releaseConnection(); + HttpClientConnectionManager connectionManager = getConnectionManager(); + if (connectionManager != null) { + connectionManager.closeExpiredConnections(); + } + } + + /* + * Caller's responsible to close the InputStream. + */ + protected InputStream getRequestAsInputStream(String path) throws IOException { + HttpGet httpget = new HttpGet(path); + HttpHost host = getHost(); + + // Extract host from URL, if present + try { + URI uri = new URI(host.toURI()); + if (uri.isAbsolute() && ! uri.isOpaque()) { + host = HttpHost.create(uri.getScheme() + "://" + uri.getAuthority()); + } + } catch (URISyntaxException ex) { + // use default + } + + try (CloseableHttpResponse response = executeMethod(host, httpget)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_NOT_FOUND) { + EntityUtils.consume(response.getEntity()); + throw new FileNotFoundException("URL: " + path); + } + if (statusCode != HttpStatus.SC_OK) { + String content = getResponseContent(response); + throw buildResponseException(response, content); + } + return new ClosingConnectionInputStream(response, httpget, getConnectionManager()); + } catch (BitbucketRequestException | FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new IOException("Communication error for url: " + path, e); + } finally { + release(httpget); + } + } + + protected int headRequestStatus(String path) throws IOException { + HttpHead httpHead = new HttpHead(path); + try (CloseableHttpResponse response = executeMethod(getHost(), httpHead)) { + EntityUtils.consume(response.getEntity()); + return response.getStatusLine().getStatusCode(); + } catch (IOException e) { + throw new IOException("Communication error for url: " + path, e); + } finally { + release(httpHead); + } + } + + protected String getRequest(String path) throws IOException { + HttpGet httpget = new HttpGet(path); + return doRequest(httpget); + } + + protected String postRequest(String path, List params) throws IOException { + HttpPost request = new HttpPost(path); + request.setEntity(new UrlEncodedFormEntity(params)); + return doRequest(request); + } + + protected String postRequest(String path, String content) throws IOException { + HttpPost request = new HttpPost(path); + request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); + return doRequest(request); + } + + protected String putRequest(String path, String content) throws IOException { + HttpPut request = new HttpPut(path); + request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); + return doRequest(request); + } + + protected String deleteRequest(String path) throws IOException { + HttpDelete request = new HttpDelete(path); + return doRequest(request); + } + + @Override + public void close() throws Exception { + if (getClient() != null) { + getClient().close(); + } + } + + protected BitbucketAuthenticator getAuthenticator() { + return authenticator; + } } diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java new file mode 100644 index 000000000..78dfb868c --- /dev/null +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java @@ -0,0 +1,114 @@ +package com.cloudbees.jenkins.plugins.bitbucket.impl.client; + +import java.util.concurrent.TimeUnit; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.annotation.Contract; +import org.apache.http.annotation.ThreadingBehavior; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.impl.client.cache.AsynchronousValidationRequest; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.Args; + +/** + * An implementation that backs off exponentially based on the number of + * consecutive failed attempts. It uses the following defaults: + * 
+ *         no delay in case it was never tried or didn't fail so far
+ *     6 secs delay for one failed attempt (= {@link #getInitialExpiryInMillis()})
+ *    60 secs delay for two failed attempts
+ *    10 mins delay for three failed attempts
+ *   100 mins delay for four failed attempts
+ *  ~16 hours delay for five failed attempts
+ *   24 hours delay for six or more failed attempts (= {@link #getMaxExpiryInMillis()})
+ *
+ * + * The following equation is used to calculate the delay for a specific revalidation request: + * 
+ *     delay = {@link #getInitialExpiryInMillis()} * Math.pow({@link #getBackOffRate()},
+ *     {@link AsynchronousValidationRequest#getConsecutiveFailedAttempts()} - 1))
+ *
+ * The resulting delay won't exceed {@link #getMaxExpiryInMillis()}. + */ +@Contract(threading = ThreadingBehavior.SAFE) +public class ExponentialBackOffServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy { + + public static final long DEFAULT_BACK_OFF_RATE = 10; + public static final long DEFAULT_INITIAL_EXPIRY_IN_MILLIS = TimeUnit.SECONDS.toMillis(6); + public static final long DEFAULT_MAX_EXPIRY_IN_MILLIS = TimeUnit.SECONDS.toMillis(86400); + + private final long backOffRate; + private final long initialExpiryInMillis; + private final long maxExpiryInMillis; + private ThreadLocal consecutiveFailedAttempts; // TODO call ThreadLocal#remove method is not possible using the lifecycle of apache client 4.x. Move to http client 5.x ASAP + + /** + * Create a new strategy using a fixed pool of worker threads. + */ + public ExponentialBackOffServiceUnavailableRetryStrategy() { + this(DEFAULT_BACK_OFF_RATE, + DEFAULT_INITIAL_EXPIRY_IN_MILLIS, + DEFAULT_MAX_EXPIRY_IN_MILLIS); + } + + /** + * Create a new strategy by using a fixed pool of worker threads and the + * given parameters to calculated the delay. + * + * @param backOffRate the back off rate to be used; not negative + * @param initialExpiryInMillis the initial expiry in milli seconds; not negative + * @param maxExpiryInMillis the upper limit of the delay in milli seconds; not negative + */ + public ExponentialBackOffServiceUnavailableRetryStrategy( + final long backOffRate, + final long initialExpiryInMillis, + final long maxExpiryInMillis) { + this.backOffRate = Args.notNegative(backOffRate, "BackOffRate"); + this.initialExpiryInMillis = Args.notNegative(initialExpiryInMillis, "InitialExpiryInMillis"); + this.maxExpiryInMillis = Args.notNegative(maxExpiryInMillis, "MaxExpiryInMillis"); + this.consecutiveFailedAttempts = new ThreadLocal<>(); + } + + public long getBackOffRate() { + return backOffRate; + } + + public long getInitialExpiryInMillis() { + return initialExpiryInMillis; + } + + public long getMaxExpiryInMillis() { + return maxExpiryInMillis; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { + int statusCode = response.getStatusLine().getStatusCode(); + consecutiveFailedAttempts.set(executionCount); + return getRetryInterval(executionCount) < maxExpiryInMillis + && (statusCode == HttpStatus.SC_TOO_MANY_REQUESTS + || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE); + } + + private long getRetryInterval(int failedAttempts) { + if (failedAttempts > 0) { + final long delayInSeconds = (long) (initialExpiryInMillis * Math.pow(backOffRate, failedAttempts - 1)); + return Math.min(delayInSeconds, maxExpiryInMillis); + } else { + return 0; + } + } + + /** + * {@inheritDoc} + */ + @Override + public long getRetryInterval() { + Integer attempts = consecutiveFailedAttempts.get(); + return getRetryInterval(attempts == null ? 0 : attempts.intValue()); + } + +} diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java index f407badb1..230cc2ecd 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java @@ -66,17 +66,15 @@ import edu.umd.cs.findbugs.annotations.CheckForNull; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import hudson.Main; import hudson.Util; import java.awt.image.BufferedImage; import java.io.BufferedInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URI; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -91,25 +89,10 @@ import org.apache.commons.lang.StringUtils; import org.apache.http.HttpHost; import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; +import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.ProtectedExternally; import static java.util.Objects.requireNonNull; @@ -120,7 +103,7 @@ public class BitbucketServerAPIClient extends AbstractBitbucketApi implements BitbucketApi { // Max avatar image length in bytes - private static final int MAX_AVATAR_SIZE = 16384; + private static final int MAX_AVATAR_LENGTH = 16384; private static final String API_BASE_PATH = "/rest/api/1.0"; private static final String API_REPOSITORIES_PATH = API_BASE_PATH + "/projects/{owner}/repos{?start,limit}"; @@ -134,7 +117,7 @@ public class BitbucketServerAPIClient extends AbstractBitbucketApi implements Bi private static final String API_PULL_REQUEST_PATH = API_REPOSITORY_PATH + "/pull-requests/{id}"; private static final String API_PULL_REQUEST_MERGE_PATH = API_REPOSITORY_PATH + "/pull-requests/{id}/merge"; private static final String API_PULL_REQUEST_CHANGES_PATH = API_REPOSITORY_PATH + "/pull-requests/{id}/changes{?start,limit}"; - static final String API_BROWSE_PATH = API_REPOSITORY_PATH + "/browse{/path*}{?at}"; + private static final String API_BROWSE_PATH = API_REPOSITORY_PATH + "/browse{/path*}{?at}"; private static final String API_COMMITS_PATH = API_REPOSITORY_PATH + "/commits{/hash}"; private static final String API_PROJECT_PATH = API_BASE_PATH + "/projects/{owner}"; private static final String AVATAR_PATH = API_BASE_PATH + "/projects/{owner}/avatar.png"; @@ -149,35 +132,37 @@ public class BitbucketServerAPIClient extends AbstractBitbucketApi implements Bi private static final String API_MIRRORS_FOR_REPO_PATH = "/rest/mirroring/1.0/repos/{id}/mirrors"; private static final String API_MIRRORS_PATH = "/rest/mirroring/1.0/mirrorServers"; - private static final Integer DEFAULT_PAGE_LIMIT = 200; - private static final Duration API_RATE_LIMIT_INITIAL_SLEEP = Main.isUnitTest ? Duration.ofMillis(100) : Duration.ofSeconds(5); - private static final Duration API_RATE_LIMIT_MAX_SLEEP = Duration.ofMinutes(30); + + protected static final HttpClientConnectionManager connectionManager = connectionManager(); + + private static HttpClientConnectionManager connectionManager() { + try { + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); // NOSONAR + connManager.setDefaultMaxPerRoute(20); + connManager.setMaxTotal(22); + return connManager; + } catch (Exception e) { + // in case of exception this avoids ClassNotFoundError which prevents the classloader from loading this class again + return null; + } + } /** * Repository owner. */ private final String owner; - /** * The repository that this object is managing. */ private final String repositoryName; - /** * Indicates if the client is using user-centric API endpoints or project API otherwise. */ private final boolean userCentric; - - /** - * Credentials to access API services. - * Almost @NonNull (but null is accepted for anonymous access). - */ - private final BitbucketAuthenticator authenticator; - private final String baseURL; - private final BitbucketServerWebhookImplementation webhookImplementation; + private final CloseableHttpClient client; @Deprecated public BitbucketServerAPIClient(@NonNull String baseURL, @NonNull String owner, @CheckForNull String repositoryName, @@ -194,12 +179,13 @@ public BitbucketServerAPIClient(@NonNull String baseURL, @NonNull String owner, public BitbucketServerAPIClient(@NonNull String baseURL, @NonNull String owner, @CheckForNull String repositoryName, @CheckForNull BitbucketAuthenticator authenticator, boolean userCentric, @NonNull BitbucketServerWebhookImplementation webhookImplementation) { - this.authenticator = authenticator; + super(authenticator); this.userCentric = userCentric; this.owner = owner; this.repositoryName = repositoryName; this.baseURL = Util.removeTrailingSlash(baseURL); this.webhookImplementation = requireNonNull(webhookImplementation); + this.client = setupClientBuilder(baseURL).build(); } /** @@ -241,7 +227,7 @@ public String getRepositoryName() { @Override public List getPullRequests() throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_PULL_REQUESTS_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUESTS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName); return getPullRequests(template); @@ -250,7 +236,7 @@ public List getPullRequests() throws IOException, In @NonNull public List getOutgoingOpenPullRequests(String fromRef) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_PULL_REQUESTS_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUESTS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("at", fromRef) @@ -262,7 +248,7 @@ public List getOutgoingOpenPullRequests(String fromR @NonNull public List getIncomingOpenPullRequests(String toRef) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_PULL_REQUESTS_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUESTS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("at", toRef) @@ -369,7 +355,7 @@ private void setupClosureForPRBranch(BitbucketServerPullRequest pr) { private void callPullRequestChangesById(@NonNull String id) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_PULL_REQUEST_CHANGES_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUEST_CHANGES_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", id).set("limit", 1) @@ -379,7 +365,7 @@ private void callPullRequestChangesById(@NonNull String id) throws IOException, private boolean getPullRequestCanMergeById(@NonNull String id) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_PULL_REQUEST_MERGE_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUEST_MERGE_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", id) @@ -399,7 +385,7 @@ private boolean getPullRequestCanMergeById(@NonNull String id) throws IOExceptio @NonNull public BitbucketPullRequest getPullRequestById(@NonNull Integer id) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_PULL_REQUEST_PATH) + .fromTemplate(this.baseURL + API_PULL_REQUEST_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", id) @@ -427,7 +413,7 @@ public BitbucketRepository getRepository() throws IOException, InterruptedExcept "Cannot get a repository from an API instance that is not associated with a repository"); } String url = UriTemplate - .fromTemplate(API_REPOSITORY_PATH) + .fromTemplate(this.baseURL + API_REPOSITORY_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(); @@ -449,7 +435,7 @@ public BitbucketRepository getRepository() throws IOException, InterruptedExcept @NonNull public List getMirrors() throws IOException, InterruptedException { UriTemplate uriTemplate = UriTemplate - .fromTemplate(API_MIRRORS_PATH); + .fromTemplate(this.baseURL + API_MIRRORS_PATH); return getResources(uriTemplate, BitbucketMirrorServerDescriptors.class); } @@ -463,7 +449,7 @@ public List getMirrors() throws IOException, InterruptedE @NonNull public List getMirrors(@NonNull Long repositoryId) throws IOException, InterruptedException { UriTemplate uriTemplate = UriTemplate - .fromTemplate(API_MIRRORS_FOR_REPO_PATH) + .fromTemplate(this.baseURL + API_MIRRORS_FOR_REPO_PATH) .set("id", repositoryId); return getResources(uriTemplate, BitbucketMirroredRepositoryDescriptors.class); } @@ -478,8 +464,7 @@ public List getMirrors(@NonNull Long repo */ @NonNull public BitbucketMirroredRepository getMirroredRepository(@NonNull String url) throws IOException, InterruptedException { - HttpGet httpget = new HttpGet(url); - String response = getRequest(httpget); + String response = getRequest(url); try { return JsonParser.toJava(response, BitbucketMirroredRepository.class); } catch (IOException e) { @@ -494,7 +479,7 @@ public BitbucketMirroredRepository getMirroredRepository(@NonNull String url) th public void postCommitComment(@NonNull String hash, @NonNull String comment) throws IOException, InterruptedException { postRequest( UriTemplate - .fromTemplate(API_COMMIT_COMMENT_PATH) + .fromTemplate(this.baseURL + API_COMMIT_COMMENT_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("hash", hash) @@ -513,7 +498,7 @@ public void postBuildStatus(@NonNull BitbucketBuildStatus status) throws IOExcep BitbucketServerBuildStatus newStatus = new BitbucketServerBuildStatus(status); newStatus.setName(truncateMiddle(newStatus.getName(), 255)); - String url = UriTemplate.fromTemplate(API_COMMIT_STATUS_PATH) + String url = UriTemplate.fromTemplate(this.baseURL + API_COMMIT_STATUS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("hash", newStatus.getHash()) @@ -527,13 +512,13 @@ public void postBuildStatus(@NonNull BitbucketBuildStatus status) throws IOExcep @Override public boolean checkPathExists(@NonNull String branchOrHash, @NonNull String path) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_BROWSE_PATH) + .fromTemplate(this.baseURL + API_BROWSE_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("path", path.split(Operator.PATH.getSeparator())) .set("at", branchOrHash) .expand(); - int status = getRequestStatus(url); + int status = headRequestStatus(url); if (HttpStatus.SC_OK == status) { return true; // Bitbucket returns UNAUTHORIZED when no credentials are provided @@ -549,7 +534,7 @@ public boolean checkPathExists(@NonNull String branchOrHash, @NonNull String pat @Override public String getDefaultBranch() throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_DEFAULT_BRANCH_PATH) + .fromTemplate(this.baseURL + API_DEFAULT_BRANCH_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(); @@ -601,7 +586,7 @@ public List getBranches() throws IOException, Interrupted private List getServerBranches(String apiPath) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(apiPath) + .fromTemplate(this.baseURL + apiPath) .set("owner", getUserCentricOwner()) .set("repo", repositoryName); @@ -617,7 +602,7 @@ private List getServerBranches(String apiPath) throws IOE private BitbucketServerBranch getSingleTag(String tagName) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_TAGS_FILTERED_PATH) + .fromTemplate(this.baseURL + API_TAGS_FILTERED_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("filterText", tagName); @@ -632,7 +617,7 @@ private BitbucketServerBranch getSingleTag(String tagName) throws IOException, I private BitbucketServerBranch getSingleBranch(String branchName) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_BRANCHES_FILTERED_PATH) + .fromTemplate(this.baseURL + API_BRANCHES_FILTERED_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("filterText", branchName); @@ -650,7 +635,7 @@ private BitbucketServerBranch getSingleBranch(String branchName) throws IOExcept @Override public BitbucketCommit resolveCommit(@NonNull String hash) throws IOException, InterruptedException { String url = UriTemplate - .fromTemplate(API_COMMITS_PATH) + .fromTemplate(this.baseURL + API_COMMITS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("hash", hash) @@ -682,7 +667,7 @@ public void registerCommitWebHook(BitbucketWebHook hook) throws IOException, Int case PLUGIN: putRequest( UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(), @@ -693,7 +678,7 @@ public void registerCommitWebHook(BitbucketWebHook hook) throws IOException, Int case NATIVE: postRequest( UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(), @@ -713,7 +698,7 @@ public void updateCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case PLUGIN: postRequest( UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_CONFIG_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_CONFIG_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -724,7 +709,7 @@ public void updateCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case NATIVE: putRequest( UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -744,7 +729,7 @@ public void removeCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case PLUGIN: deleteRequest( UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_CONFIG_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_CONFIG_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -755,7 +740,7 @@ public void removeCommitWebHook(BitbucketWebHook hook) throws IOException, Inter case NATIVE: deleteRequest( UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("id", hook.getUuid()) @@ -775,7 +760,7 @@ public List getWebHooks() throws IOException, Interr switch (webhookImplementation) { case PLUGIN: String url = UriTemplate - .fromTemplate(WEBHOOK_REPOSITORY_PATH) + .fromTemplate(this.baseURL + WEBHOOK_REPOSITORY_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .expand(); @@ -783,7 +768,7 @@ public List getWebHooks() throws IOException, Interr return JsonParser.toJava(response, BitbucketServerWebhooks.class); case NATIVE: UriTemplate urlTemplate = UriTemplate - .fromTemplate(API_WEBHOOKS_PATH) + .fromTemplate(this.baseURL + API_WEBHOOKS_PATH) .set("owner", getUserCentricOwner()) .set("repo", repositoryName); return getResources(urlTemplate, NativeBitbucketServerWebhooks.class); @@ -800,7 +785,9 @@ public BitbucketTeam getTeam() throws IOException, InterruptedException { if (userCentric) { return null; } else { - String url = UriTemplate.fromTemplate(API_PROJECT_PATH).set("owner", getOwner()).expand(); + String url = UriTemplate.fromTemplate(this.baseURL + API_PROJECT_PATH) + .set("owner", getOwner()) + .expand(); try { String response = getRequest(url); return JsonParser.toJava(response, BitbucketServerProject.class); @@ -820,7 +807,9 @@ public AvatarImage getTeamAvatar() throws IOException { if (userCentric) { return null; } else { - String url = UriTemplate.fromTemplate(AVATAR_PATH).set("owner", getOwner()).expand(); + String url = UriTemplate.fromTemplate(this.baseURL + AVATAR_PATH) + .set("owner", getOwner()) + .expand(); try { BufferedImage response = getImageRequest(url); return new AvatarImage(response, System.currentTimeMillis()); @@ -842,7 +831,7 @@ public AvatarImage getTeamAvatar() throws IOException { public List getRepositories(@CheckForNull UserRoleInRepository role) throws IOException, InterruptedException { UriTemplate template = UriTemplate - .fromTemplate(API_REPOSITORIES_PATH) + .fromTemplate(this.baseURL + API_REPOSITORIES_PATH) .set("owner", getUserCentricOwner()); List repositories; @@ -912,7 +901,7 @@ private V getResource(UriTemplate template, Class V getResource(UriTemplate template, Class 0 ? Math.min(MAX_AVATAR_SIZE, length) : MAX_AVATAR_SIZE; - BufferedInputStream bis = new BufferedInputStream(is, length); - content = ImageIO.read(bis); - } - } - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND) { - throw new FileNotFoundException("URL: " + path); - } - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new BitbucketRequestException(response.getStatusLine().getStatusCode(), - "HTTP request error. Status: " + response.getStatusLine().getStatusCode() + ": " - + response.getStatusLine().getReasonPhrase() + ".\n" + response); - } - return content; - } catch (BitbucketRequestException | FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw new IOException("Communication error for url: " + path, e); - } finally { - httpget.releaseConnection(); - } - - } - - /** - * Create HttpClient from given host/port - * @param request the {@link HttpRequestBase} for which an HttpClient will be created - * @return CloseableHttpClient - */ - private CloseableHttpClient getHttpClient(final HttpRequestBase request) { - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); - httpClientBuilder.useSystemProperties(); - httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); - httpClientBuilder.disableCookieManagement(); - - RequestConfig.Builder requestConfig = RequestConfig.custom(); - String connectTimeout = System.getProperty("http.connect.timeout", "10"); - requestConfig.setConnectTimeout(Integer.parseInt(connectTimeout) * 1000); - String connectionRequestTimeout = System.getProperty("http.connect.request.timeout", "60"); - requestConfig.setConnectionRequestTimeout(Integer.parseInt(connectionRequestTimeout) * 1000); - String socketTimeout = System.getProperty("http.socket.timeout", "60"); - requestConfig.setSocketTimeout(Integer.parseInt(socketTimeout) * 1000); - request.setConfig(requestConfig.build()); - - final String host = getMethodHost(request); - - if (authenticator != null) { - authenticator.configureBuilder(httpClientBuilder); - - context = HttpClientContext.create(); - authenticator.configureContext(context, HttpHost.create(host)); - } - - setClientProxyParams(host, httpClientBuilder); - - return httpClientBuilder.build(); - } - - private int getRequestStatus(String path) throws IOException, InterruptedException { - HttpGet httpget = new HttpGet(this.baseURL + path); - if (authenticator != null) { - authenticator.configureRequest(httpget); - } - - try(CloseableHttpClient client = getHttpClient(httpget); - CloseableHttpResponse response = executeMethod(client, httpget)) { - EntityUtils.consume(response.getEntity()); - return response.getStatusLine().getStatusCode(); - } finally { - httpget.releaseConnection(); + try (InputStream inputStream = getRequestAsInputStream(path)) { + int length = MAX_AVATAR_LENGTH; + BufferedInputStream bis = new BufferedInputStream(inputStream, length); + return ImageIO.read(bis); } } - private static String getMethodHost(HttpRequestBase method) { - URI uri = method.getURI(); - String scheme = uri.getScheme() == null ? "http" : uri.getScheme(); - return scheme + "://" + uri.getAuthority(); - } - - private String postRequest(String path, List params) throws IOException, InterruptedException { - HttpPost request = new HttpPost(this.baseURL + path); - request.setEntity(new UrlEncodedFormEntity(params)); - return postRequest(request); - } - - private String postRequest(String path, String content) throws IOException, InterruptedException { - HttpPost request = new HttpPost(this.baseURL + path); - request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return postRequest(request); + @Override + protected HttpClientConnectionManager getConnectionManager() { + return connectionManager; } - private String postRequest(HttpPost httppost) throws IOException, InterruptedException { - return doRequest(httppost); + @NonNull + @Override + protected CloseableHttpClient getClient() { + return client; } - private String doRequest(HttpRequestBase request) throws IOException, InterruptedException { - if (authenticator != null) { - authenticator.configureRequest(request); - } - - try (CloseableHttpClient client = getHttpClient(request); - CloseableHttpResponse response = executeMethod(client, request)) { - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.SC_NO_CONTENT) { - EntityUtils.consume(response.getEntity()); - // 204, no content - return ""; - } - String content = getResponseContent(response); - EntityUtils.consume(response.getEntity()); - if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { - throw new BitbucketRequestException(statusCode, "HTTP request error. Status: " + statusCode + ": " + response.getStatusLine().getReasonPhrase() + ".\n" + response); + @NonNull + @Override + protected HttpHost getHost() { + String url = baseURL; + try { + // it's really needed? + URL tmp = new URL(baseURL); + if (tmp.getProtocol() == null) { + url = new URL("http", tmp.getHost(), tmp.getPort(), tmp.getFile()).toString(); } - return content; - } finally { - request.releaseConnection(); + } catch (MalformedURLException e) { } - } - - private String putRequest(String path, String content) throws IOException, InterruptedException { - HttpPut request = new HttpPut(this.baseURL + path); - request.setEntity(new StringEntity(content, ContentType.create("application/json", "UTF-8"))); - return doRequest(request); - } - - private String deleteRequest(String path) throws IOException, InterruptedException { - HttpDelete request = new HttpDelete(this.baseURL + path); - return doRequest(request); + return HttpHost.create(url); } @Override @@ -1119,7 +962,7 @@ public Iterable getDirectoryContent(BitbucketSCMFile directory) throws int start=0; String branchOrHash = directory.getHash().contains("+") ? directory.getRef() : directory.getHash(); UriTemplate template = UriTemplate - .fromTemplate(API_BROWSE_PATH + "{&start,limit}") + .fromTemplate(this.baseURL + API_BROWSE_PATH + "{&start,limit}") .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("path", directory.getPath().split(Operator.PATH.getSeparator())) @@ -1168,7 +1011,7 @@ public InputStream getFileContent(BitbucketSCMFile file) throws IOException, Int int start=0; String branchOrHash = file.getHash().contains("+") ? file.getRef() : file.getHash(); UriTemplate template = UriTemplate - .fromTemplate(API_BROWSE_PATH + "{&start,limit}") + .fromTemplate(this.baseURL + API_BROWSE_PATH + "{&start,limit}") .set("owner", getUserCentricOwner()) .set("repo", repositoryName) .set("path", file.getPath().split(Operator.PATH.getSeparator())) @@ -1202,35 +1045,4 @@ private Map collectLines(String response, final List line return content; } - private CloseableHttpResponse executeMethod(CloseableHttpClient client, HttpRequestBase httpMethod) throws IOException, InterruptedException { - CloseableHttpResponse response = executeMethodNoRetry(client, httpMethod, context); - Instant start = Instant.now(); - Instant forcedEnd = start.plus(API_RATE_LIMIT_MAX_SLEEP); - Duration sleepDuration = API_RATE_LIMIT_INITIAL_SLEEP; - while (response.getStatusLine().getStatusCode() == API_RATE_LIMIT_STATUS_CODE - && Instant.now().plus(sleepDuration).isBefore(forcedEnd)) { - response.close(); - httpMethod.releaseConnection(); - /* - * TODO: If The Bitbucket Server API ever starts sending rate limit expiration time, we should - * change this to a more precise sleep. - * TODO: It would be better to log this to a context-appropriate TaskListener, e.g. an org/repo scan log. - */ - logger.log(Level.FINE, "Bitbucket server API rate limit reached, sleeping for {0} before retrying", - sleepDuration); - Thread.sleep(sleepDuration.toMillis()); - // Duration increases exponentially: 5s, 7s, 10s, 15s, 22s, ... 6m6s, 9m9s. - // We will retry at most 13 times and sleep for roughly 27 minutes. - sleepDuration = Duration.ofSeconds((int)(sleepDuration.getSeconds() * 1.5)); - response = executeMethodNoRetry(client, httpMethod, context); - } - return response; - } - - // Exists just so it can be mocked in BitbucketIntegrationClientFactory. - @Restricted(ProtectedExternally.class) - protected CloseableHttpResponse executeMethodNoRetry(CloseableHttpClient client, HttpRequestBase httpMethod, HttpClientContext context) throws IOException, InterruptedException { - return client.execute(httpMethod, context); - } - } diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java index e07d39091..20cb051a1 100644 --- a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketIntegrationClientFactory.java @@ -36,8 +36,6 @@ import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.impl.client.CloseableHttpClient; import org.apache.tools.ant.filters.StringInputStream; import static org.mockito.Mockito.mock; @@ -92,10 +90,9 @@ public static class BitbucketServerIntegrationClient extends BitbucketServerAPIC private final String payloadRootPath; private final IRequestAudit audit; - private boolean rateLimitNextRequest; // TODO: Would be nice to have a better way to mock non-200 responses. private BitbucketServerIntegrationClient(String payloadRootPath, String baseURL, String owner, String repositoryName) { - super(baseURL, owner, repositoryName, (BitbucketAuthenticator) null, false); + super(baseURL, owner, repositoryName, mock(BitbucketAuthenticator.class), false); if (payloadRootPath == null) { this.payloadRootPath = PAYLOAD_RESOURCE_ROOTPATH; @@ -107,16 +104,8 @@ private BitbucketServerIntegrationClient(String payloadRootPath, String baseURL, this.audit = mock(IRequestAudit.class); } - public void rateLimitNextRequest() { - rateLimitNextRequest = true; - } - @Override - protected CloseableHttpResponse executeMethodNoRetry(CloseableHttpClient client, HttpRequestBase httpMethod, HttpClientContext context) throws IOException { - if (rateLimitNextRequest) { - rateLimitNextRequest = false; - return createRateLimitResponse(); - } + protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { String path = httpMethod.getURI().toString(); audit.request(httpMethod); @@ -160,7 +149,7 @@ private BitbucketClouldIntegrationClient(String payloadRootPath, String owner, S } @Override - protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws InterruptedException, IOException { + protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { String path = httpMethod.getURI().toString(); audit.request(httpMethod); diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/hooks/WebhookConfigurationTest.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/hooks/WebhookConfigurationTest.java index 5ab3fd3f8..0452dab85 100644 --- a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/hooks/WebhookConfigurationTest.java +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/hooks/WebhookConfigurationTest.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2020, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package com.cloudbees.jenkins.plugins.bitbucket.hooks; import com.cloudbees.jenkins.plugins.bitbucket.BitbucketSCMSource; diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java new file mode 100644 index 000000000..cfae330b8 --- /dev/null +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java @@ -0,0 +1,96 @@ +/* + * The MIT License + * + * Copyright (c) 2024, Nikolas Falco + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.cloudbees.jenkins.plugins.bitbucket.impl.client; + +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketApi; +import com.cloudbees.jenkins.plugins.bitbucket.api.BitbucketAuthenticator; +import com.cloudbees.jenkins.plugins.bitbucket.server.BitbucketServerWebhookImplementation; +import com.cloudbees.jenkins.plugins.bitbucket.server.client.BitbucketServerAPIClient; +import java.io.IOException; +import org.apache.http.HttpException; +import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.HttpContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.junit.jupiter.MockServerExtension; +import org.mockserver.model.HttpRequest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIOException; +import static org.mockito.Mockito.mock; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +@ExtendWith(MockServerExtension.class) +class ExponentialBackOffServiceUnavailableRetryStrategyTest { + + @Test + void test_retry(ClientAndServer mockServer) throws Exception { + HttpRequest request = request() // + .withMethod("GET") // + .withPath("/rest/api/1.0/projects/test/repos/testRepos/tags"); + mockServer.when(request) + .respond( // + response() // + .withStatusCode(429) // + ); + + final RetryInterceptor counterInterceptor = new RetryInterceptor(); + BitbucketApi client = new BitbucketServerAPIClient("http://localhost:" + mockServer.getPort(), + "test", + "testRepos", + (BitbucketAuthenticator) null, + false, + mock(BitbucketServerWebhookImplementation.class)) { + @Override + protected HttpClientBuilder setupClientBuilder(String host) { + return super.setupClientBuilder(host) + .disableAutomaticRetries() + .setServiceUnavailableRetryStrategy(new ExponentialBackOffServiceUnavailableRetryStrategy(2, 5, 100)) + .addInterceptorFirst(counterInterceptor); + } + }; + + assertThatIOException().isThrownBy(() -> client.getTags()); + assertThat(counterInterceptor.getRetry()).isEqualTo(6); // retry after 5ms, 10ms, 20ms, 40ms, 80ms, 100ms + } + + private static class RetryInterceptor implements HttpResponseInterceptor { + private int retry = 0; + + @Override + public void process(HttpResponse response, HttpContext context) throws HttpException, IOException { + if (response.getStatusLine().getStatusCode() == 429) { + retry += 1; + } + } + + public int getRetry() { + return retry; + } + } +} diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java index b73fe3853..a98d02e44 100644 --- a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClientTest.java @@ -7,19 +7,17 @@ import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketIntegrationClientFactory; import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketIntegrationClientFactory.BitbucketServerIntegrationClient; import com.cloudbees.jenkins.plugins.bitbucket.client.BitbucketIntegrationClientFactory.IRequestAudit; -import com.damnhandy.uri.template.UriTemplate; -import com.damnhandy.uri.template.impl.Operator; import io.jenkins.cli.shaded.org.apache.commons.lang.RandomStringUtils; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.logging.Level; import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.junit.Assert; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -29,15 +27,13 @@ import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; -import static com.cloudbees.jenkins.plugins.bitbucket.server.client.BitbucketServerAPIClient.API_BROWSE_PATH; import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.verify; @@ -79,36 +75,31 @@ private HttpRequestBase extractRequest(IRequestAudit clientAudit) { @Test @WithoutJenkins - public void repoBrowsePathFolder() { - String expand = UriTemplate - .fromTemplate(API_BROWSE_PATH) - .set("owner", "test") - .set("repo", "test") - .set("path", "folder/Jenkinsfile".split(Operator.PATH.getSeparator())) - .set("at", "fix/test") - .expand(); - Assert.assertEquals("/rest/api/1.0/projects/test/repos/test/browse/folder/Jenkinsfile?at=fix%2Ftest", expand); + public void verify_checkPathExists_given_a_path() throws Exception { + BitbucketApi client = BitbucketIntegrationClientFactory.getApiMockClient("https://acme.bitbucket.org"); + assertThat(client.checkPathExists("feature/pipeline", "folder/Jenkinsfile")).isTrue(); + + IRequestAudit clientAudit = ((IRequestAudit) client).getAudit(); + HttpRequestBase request = extractRequest(clientAudit); + assertThat(request).isNotNull() + .isInstanceOfSatisfying(HttpHead.class, head -> { + assertThat(head.getURI()) + .hasPath("/rest/api/1.0/projects/amuniz/repos/test-repos/browse/folder/Jenkinsfile") + .hasQuery("at=feature/pipeline"); + }); } @Test @WithoutJenkins - public void repoBrowsePathFile() { - String expand = UriTemplate - .fromTemplate(API_BROWSE_PATH) - .set("owner", "test") - .set("repo", "test") - .set("path", "Jenkinsfile".split(Operator.PATH.getSeparator())) - .expand(); - Assert.assertEquals("/rest/api/1.0/projects/test/repos/test/browse/Jenkinsfile", expand); - } + public void verify_checkPathExists_given_file() throws Exception { + BitbucketApi client = BitbucketIntegrationClientFactory.getApiMockClient("https://acme.bitbucket.org"); + assertThat(client.checkPathExists("feature/pipeline", "Jenkinsfile")).isTrue(); - @Test - public void retryWhenRateLimited() throws Exception { - logger.capture(50); - BitbucketApi client = BitbucketIntegrationClientFactory.getClient("localhost", "amuniz", "test-repos"); - ((BitbucketServerIntegrationClient)client).rateLimitNextRequest(); - assertThat(client.getRepository().getProject().getKey(), equalTo("AMUNIZ")); - assertThat(logger.getMessages(), hasItem(containsString("Bitbucket server API rate limit reached"))); + IRequestAudit clientAudit = ((IRequestAudit) client).getAudit(); + HttpRequestBase request = extractRequest(clientAudit); + assertThat(request).isNotNull() + .isInstanceOfSatisfying(HttpHead.class, head -> + assertThat(head.getURI()).hasPath("/rest/api/1.0/projects/amuniz/repos/test-repos/browse/Jenkinsfile")); } @Test @@ -130,10 +121,10 @@ public void sortRepositoriesByName() throws Exception { @Test public void disableCookieManager() throws Exception { - try(MockedStatic staticHttpClientBuilder = mockStatic(HttpClientBuilder.class)) { - HttpClientBuilder httpClientBuilder = mock(HttpClientBuilder.class); - CloseableHttpClient httpClient = mock(CloseableHttpClient.class); + try (MockedStatic staticHttpClientBuilder = mockStatic(HttpClientBuilder.class)) { + HttpClientBuilder httpClientBuilder = mock(HttpClientBuilder.class, RETURNS_SELF); staticHttpClientBuilder.when(HttpClientBuilder::create).thenReturn(httpClientBuilder); + CloseableHttpClient httpClient = mock(CloseableHttpClient.class); when(httpClientBuilder.build()).thenReturn(httpClient); BitbucketApi client = BitbucketIntegrationClientFactory.getClient("localhost", "amuniz", "test-repos"); client.getRepositories(); diff --git a/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json new file mode 100644 index 000000000..aca597f12 --- /dev/null +++ b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-Jenkinsfile_at_feature_2Fpipeline.json @@ -0,0 +1,3 @@ +node() { + echo "ciao" +} \ No newline at end of file diff --git a/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json new file mode 100644 index 000000000..aca597f12 --- /dev/null +++ b/src/test/resources/com/cloudbees/jenkins/plugins/bitbucket/server/payload/1.0-projects-amuniz-repos-test-repos-browse-folder-Jenkinsfile_at_feature_2Fpipeline.json @@ -0,0 +1,3 @@ +node() { + echo "ciao" +} \ No newline at end of file