@Override public final T onCompleted() throws Exception { return onCompleted(builder.build()); }
@Override public void onThrowable(final Throwable t) { //nginx does not send status code so this http client explodes final boolean maybeUriTooLongErrorFromNgingx = t.getMessage().contains("invalid version format: <HTML>"); final String message = maybeUriTooLongErrorFromNgingx ? "There is a problem, maybe the request URI was too long due to an inefficient query." : "The underlying HTTP client detected a problem."; future.completeExceptionally(new HttpException(message, t)); super.onThrowable(t); } });
@Override public STATE onStatusReceived(HttpResponseStatus responseStatus) throws Exception { this.httpStatus = responseStatus.getStatusCode(); switch (this.httpStatus) { case 404: LOG.warn("Abandoning missing page: " + url); connections.decrementAndGet(); return STATE.ABORT; case 500: retry(); return STATE.ABORT; } return super.onStatusReceived(responseStatus); }
@Override public STATE onHeadersReceived(HttpResponseHeaders headers) throws Exception { switch (this.httpStatus) { case 301: case 302: String redirect = headers.getHeaders().getFirstValue("Location"); if (redirect.contains("protected_redirect=true")) { LOG.warn("Abandoning protected account: " + url); connections.decrementAndGet(); } else if (redirect.contains("account/suspended")) { LOG.warn("Abandoning suspended account: " + url); connections.decrementAndGet(); } else if (redirect.contains("//status") || redirect.contains("login?redirect_after_login")) { LOG.warn("Abandoning deleted account: " + url); connections.decrementAndGet(); } else if (followRedirects) { crawlURL(redirect, new TweetFetcherHandler(id, username, redirect, numRetries, followRedirects, line)); } else { LOG.warn("Abandoning redirect: " + url); connections.decrementAndGet(); } return STATE.ABORT; } return super.onHeadersReceived(headers); }
@Override public STATE onBodyPartReceived(HttpResponseBodyPart content) throws Exception { times[0] = millisTime(); return super.onBodyPartReceived(content); }
@Override public void onThrowable(final Throwable t) { //nginx does not send status code so this http client explodes final boolean maybeUriTooLongErrorFromNgingx = t.getMessage().contains("invalid version format: <HTML>"); final String message = maybeUriTooLongErrorFromNgingx ? "There is a problem, maybe the request URI was too long due to an inefficient query." : "The underlying HTTP client detected a problem."; future.completeExceptionally(new HttpException(message, t)); super.onThrowable(t); } });
@Override public STATE onStatusReceived(HttpResponseStatus responseStatus) throws Exception { this.httpStatus = responseStatus.getStatusCode(); switch (this.httpStatus) { case 404: LOG.warn("Abandoning missing page: " + url); connections.decrementAndGet(); return STATE.ABORT; case 500: retry(); return STATE.ABORT; } return super.onStatusReceived(responseStatus); }
@Override public STATE onHeadersReceived(HttpResponseHeaders headers) throws Exception { switch (this.httpStatus) { case 301: case 302: String redirect = headers.getHeaders().getFirstValue("Location"); if (redirect.contains("protected_redirect=true")) { LOG.warn("Abandoning protected account: " + url); connections.decrementAndGet(); } else if (redirect.contains("account/suspended")) { LOG.warn("Abandoning suspended account: " + url); connections.decrementAndGet(); } else if (redirect.contains("//status") || redirect.contains("login?redirect_after_login")) { LOG.warn("Abandoning deleted account: " + url); connections.decrementAndGet(); } else if (followRedirects) { crawlURL(redirect, new TweetFetcherHandler(id, username, redirect, numRetries, followRedirects, line)); } else { LOG.warn("Abandoning redirect: " + url); connections.decrementAndGet(); } return STATE.ABORT; } return super.onHeadersReceived(headers); }
@Override public void onThrowable(Throwable t) { counts.incrementAndGet(); super.onThrowable(t); } });
@Override public final T onCompleted() throws Exception { return onCompleted(builder.build()); }
@Override public STATE onStatusReceived(HttpResponseStatus status) throws Exception { super.onStatusReceived(status); if (status.getStatusCode() != 200 && status.getStatusCode() != 204) { LOG.warn("Request failed, status code: {}", status.getStatusCode() + status.getStatusText()); LOG.warn("request: {}", request.toString()); } return STATE.CONTINUE; }
@Override public void onThrowable(Throwable t) { times[1] = millisTime(); super.onThrowable(t); } });
@Override public final T onCompleted() throws Exception { return onCompleted(builder.build()); }
public synchronized void executeRequest(final File file, final AsyncCompletionHandler<Response> completionHandler) { if (isClosed.get()) { return; } if (client == null || client.isClosed()) { client = createClient(); } final Request request = createPostRequest(file); try { client.executeRequest(request, completionHandler); } catch (Exception e) { // Recycle the client on IOException and RuntimeExceptions client.close(); client = createClient(); completionHandler.onThrowable(e); } }
@Override public final T onCompleted() throws Exception { return onCompleted(builder.build()); }