Skip to content

Commit

Permalink
Implement an abuse handler
Browse files Browse the repository at this point in the history
If too many requests are made within X amount of time (not the traditional hourly rate limit), github may begin returning 403.  Then we should wait for a bit to attempt to access the API again.  In this case, we parse out the Retry-After field returned and sleep until that (it's usually 60 seconds)
  • Loading branch information
mmitche committed Jul 21, 2016
1 parent a2f0837 commit 8f389d8
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 2 deletions.
4 changes: 3 additions & 1 deletion src/main/java/org/kohsuke/github/GitHub.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public class GitHub {
private final String apiUrl;

/*package*/ final RateLimitHandler rateLimitHandler;
/*package*/ final RateLimitHandler abuseLimitHandler;

private HttpConnector connector = HttpConnector.DEFAULT;

Expand Down Expand Up @@ -122,7 +123,7 @@ public class GitHub {
* @param connector
* HttpConnector to use. Pass null to use default connector.
*/
/* package */ GitHub(String apiUrl, String login, String oauthAccessToken, String password, HttpConnector connector, RateLimitHandler rateLimitHandler) throws IOException {
/* package */ GitHub(String apiUrl, String login, String oauthAccessToken, String password, HttpConnector connector, RateLimitHandler rateLimitHandler, RateLimitHandler abuseLimitHandler) throws IOException {
if (apiUrl.endsWith("/")) apiUrl = apiUrl.substring(0, apiUrl.length()-1); // normalize
this.apiUrl = apiUrl;
if (null != connector) this.connector = connector;
Expand All @@ -140,6 +141,7 @@ public class GitHub {
}

this.rateLimitHandler = rateLimitHandler;
this.abuseLimitHandler = abuseLimitHandler;

if (login==null && encodedAuthorization!=null)
login = getMyself().getLogin();
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/kohsuke/github/GitHubBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class GitHubBuilder {
private HttpConnector connector;

private RateLimitHandler rateLimitHandler = RateLimitHandler.WAIT;
private RateLimitHandler abuseLimitHandler = RateLimitHandler.ABUSE;

public GitHubBuilder() {
}
Expand Down Expand Up @@ -193,6 +194,6 @@ public HttpURLConnection connect(URL url) throws IOException {
}

public GitHub build() throws IOException {
return new GitHub(endpoint, user, oauthToken, password, connector, rateLimitHandler);
return new GitHub(endpoint, user, oauthToken, password, connector, rateLimitHandler, abuseLimitHandler);
}
}
21 changes: 21 additions & 0 deletions src/main/java/org/kohsuke/github/RateLimitHandler.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,27 @@ private long parseWaitTime(HttpURLConnection uc) {
return Math.max(10000, Long.parseLong(v)*1000 - System.currentTimeMillis());
}
};

/**
* Wait until the API abuse "wait time" is passed.
*/
public static final RateLimitHandler ABUSE = new RateLimitHandler() {
@Override
public void onError(IOException e, HttpURLConnection uc) throws IOException {
try {
Thread.sleep(parseWaitTime(uc));
} catch (InterruptedException _) {
throw (InterruptedIOException)new InterruptedIOException().initCause(e);
}
}

private long parseWaitTime(HttpURLConnection uc) {
String v = uc.getHeaderField("Retry-After");
if (v==null) return 60 * 1000; // can't tell, return 1 min

return Math.max(1000, Long.parseLong(v)*1000);
}
};

/**
* Fail immediately.
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/org/kohsuke/github/Requester.java
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,15 @@ private InputStream wrapStream(InputStream in) throws IOException {
InputStream es = wrapStream(uc.getErrorStream());
try {
if (es!=null) {
String errorString = IOUtils.toString(es, "UTF-8");
// Check to see whether we hit a 403, and the Retry-After field is
// available.
if (responseCode == HttpURLConnection.HTTP_FORBIDDEN &&
uc.getHeaderField("Retry-After") != null) {
this.root.abuseLimitHandler.onError(e,uc);
return;
}

if (e instanceof FileNotFoundException) {
// pass through 404 Not Found to allow the caller to handle it intelligently
throw (IOException) new FileNotFoundException(IOUtils.toString(es, "UTF-8")).initCause(e);
Expand Down

0 comments on commit 8f389d8

Please sign in to comment.