public void handleError(HttpCommand command, HttpResponse response) { // it is important to always read fully and close streams byte[] data = closeClientButKeepContentStream(response); String message = data != null ? new String(data) : null; Exception exception = message != null ? new HttpResponseException(command, response, message) : new HttpResponseException(command, response); message = message != null ? message : String.format("%s -> %s", command.getCurrentRequest().getRequestLine(), response.getStatusLine()); switch (response.getStatusCode()) { case 400: break; case 401: case 403: exception = new AuthorizationException(message, exception); break; case 404: if (!command.getCurrentRequest().getMethod().equals("DELETE")) { exception = new ResourceNotFoundException(message, exception); } break; } command.setException(exception); } }
public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { if (command.getFailureCount() > retryCountLimit) return false; if (response.getStatusCode() == 400 && command.getCurrentRequest().getMethod().equals("PUT") && command.getCurrentRequest().getEndpoint().getPath().indexOf("sandboxes") != -1) { if (response.getPayload() != null) { String error = new String(closeClientButKeepContentStream(response)); if (error != null && error.indexOf("was not uploaded") != -1) { return backoffLimitedRetryHandler.shouldRetryRequest(command, response); } } } return false; }
@Override public boolean shouldRetryRequest(final HttpCommand command, final HttpResponse response) { command.incrementFailureCount(); // Do not retry client errors that are not rate limit errors if (response.getStatusCode() != rateLimitErrorStatus()) { return false; } else if (!command.isReplayable()) { logger.error("Cannot retry after rate limit error, command is not replayable: %1$s", command); return false; } else if (command.getFailureCount() > retryCountLimit) { logger.error("Cannot retry after rate limit error, command has exceeded retry limit %1$d: %2$s", retryCountLimit, command); return false; } else { return delayRequestUntilAllowed(command, response); } }
private boolean ifReplayableBackoffAndReturnTrue(HttpCommand command) { command.incrementFailureCount(); if (!command.isReplayable()) { logger.error("Cannot retry after server error, command is not replayable: %1$s", command); return false; } else if (command.getFailureCount() > retryCountLimit) { logger.error("Cannot retry after server error, command has exceeded retry limit %1$d: %2$s", retryCountLimit, command); return false; } else { imposeBackoffExponentialDelay(command.getFailureCount(), "server error: " + command.toString()); return true; } }
@Override public HttpResponse invoke(HttpCommand command) { if (callCounter == 1) assertEquals(command.getCurrentRequest().getRequestLine(), "GET http://localhost:1111/client/1/bar/2 HTTP/1.1"); else assertEquals(command.getCurrentRequest().getRequestLine(), "GET http://localhost:1111/client/1/foo HTTP/1.1"); callCounter++; return HttpResponse.builder().build(); } });
public void testQueueDeletedRecentlyRetriesWhen60DoesntTry() { SQSErrorRetryHandler retry = new SQSErrorRetryHandler(createMock(AWSUtils.class), createMock(BackoffLimitedRetryHandler.class), ImmutableSet.<String> of(), 60, 100); HttpCommand command = createHttpCommandForFailureCount(60); Stopwatch watch = Stopwatch.createStarted(); assertFalse(retry.shouldRetryRequestOnError(command, response, error)); assertEquals(command.getFailureCount(), 61); assertTrue(watch.stop().elapsed(TimeUnit.MILLISECONDS) < 100); }
public void testQueueDeletedRecentlyRetriesWhen59SleepsAndTries() { SQSErrorRetryHandler retry = new SQSErrorRetryHandler(createMock(AWSUtils.class), createMock(BackoffLimitedRetryHandler.class), ImmutableSet.<String> of(), 60, 100); HttpCommand command = createHttpCommandForFailureCount(59); Stopwatch watch = Stopwatch.createStarted(); assertTrue(retry.shouldRetryRequestOnError(command, response, error)); assertEquals(command.getFailureCount(), 60); // allow for slightly inaccurate system timers assertTrue(watch.stop().elapsed(TimeUnit.MILLISECONDS) >= 98); }
public void handleError(HttpCommand command, HttpResponse from) { String content; try { content = from.getPayload() != null ? Strings2.toString(from.getPayload()) : null; command.setException(new HttpResponseException(command, from, content)); } catch (IOException e) { command.setException(new HttpResponseException(command, from)); } finally { releasePayload(from); } } }
@Test void testIncrementsFailureCount() throws InterruptedException, IOException, SecurityException, NoSuchMethodException { HttpCommand command = createCommand(); HttpResponse response = HttpResponse.builder().statusCode(400).build(); handler.shouldRetryRequest(command, response); assertEquals(command.getFailureCount(), 1); handler.shouldRetryRequest(command, response); assertEquals(command.getFailureCount(), 2); handler.shouldRetryRequest(command, response); assertEquals(command.getFailureCount(), 3); }
@Test(timeOut = TEST_SAFE_TIMEOUT) public void testIncrementsFailureCount() { HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost").build()); HttpResponse response = HttpResponse.builder().statusCode(429).build(); rateLimitRetryHandler.shouldRetryRequest(command, response); assertEquals(command.getFailureCount(), 1); rateLimitRetryHandler.shouldRetryRequest(command, response); assertEquals(command.getFailureCount(), 2); rateLimitRetryHandler.shouldRetryRequest(command, response); assertEquals(command.getFailureCount(), 3); }
private void assertCodeMakes(String method, URI uri, int statusCode, String message, String contentType, String content, Class<? extends Exception> expected) { SoftLayerErrorHandler function = Guice.createInjector().getInstance(SoftLayerErrorHandler.class); HttpCommand command = createMock(HttpCommand.class); HttpRequest request = HttpRequest.builder().method(method).endpoint(uri).build(); HttpResponse response = HttpResponse.builder().statusCode(statusCode).message(message).payload(content).build(); response.getPayload().getContentMetadata().setContentType(contentType); expect(command.getCurrentRequest()).andReturn(request).atLeastOnce(); command.setException(classEq(expected)); replay(command); function.handleError(command, response); verify(command); }
public static HttpResponseException returnResponseException(int code) { HttpResponse response = HttpResponse.builder().statusCode(code).build(); return new HttpResponseException(new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://stub") .build()), response); }
@Test public void test504DoesRetry() { AWSUtils utils = createMock(AWSUtils.class); HttpCommand command = createMock(HttpCommand.class); expect(command.getFailureCount()).andReturn(1).anyTimes(); expect(command.incrementFailureCount()).andReturn(1); expect(command.isReplayable()).andReturn(true); replay(utils, command); AWSServerErrorRetryHandler retry = new AWSServerErrorRetryHandler(utils, ImmutableSet.<String> of()); assertTrue(retry.shouldRetryRequest(command, HttpResponse.builder().statusCode(504).build())); verify(utils, command); } }
@Test(timeOut = TEST_SAFE_TIMEOUT) public void testDoNotRetryIfNotReplayable() { // InputStream payloads are not replayable Payload payload = newInputStreamPayload(new ByteArrayInputStream(new byte[0])); HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost") .payload(payload).build()); HttpResponse response = HttpResponse.builder().statusCode(429).build(); try { assertFalse(rateLimitRetryHandler.shouldRetryRequest(command, response)); } finally { releasePayload(command.getCurrentRequest()); } }
@Test public void testRetriesWhenRetryableError() { String retryable = "{\"error\":{\"code\":\"RetryableError\",\"message\":\"Resource busy\"}}"; HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost").build()); HttpResponse response = HttpResponse.builder().statusCode(429).payload(retryable).build(); assertTrue(handler.shouldRetryRequest(command, response)); } }
public HttpResponseException(HttpCommand command, HttpResponse response) { this(String.format("request: %s %sfailed with response: %s", command.getCurrentRequest().getRequestLine(), requestPayloadIfStringOrFormIfNotReturnEmptyString(command.getCurrentRequest()), response.getStatusLine()), command, response); }
@Test public void testDoesNotRetryWhenCannotParseError() { HttpCommand command = new HttpCommand(HttpRequest.builder().method("GET").endpoint("http://localhost").build()); HttpResponse response = HttpResponse.builder().statusCode(429).payload("foo").build(); assertFalse(handler.shouldRetryRequest(command, response)); }
/** * clones or proxies can mess up the error message. */ @Test public void testNoExceptionParsingTextPlain() { HttpResponse response = HttpResponse.builder().statusCode(BAD_REQUEST.getStatusCode()).payload("foo bar").build(); response.getPayload().getContentMetadata().setContentType(TEXT_PLAIN); assertNull(utils.parseAWSErrorFromContent(command.getCurrentRequest(), response)); }
boolean shouldContinue(HttpCommand command, IOException response) { // Even though Java does not want to handle it this way, // treat a Protocol Exception on PUT with 100-Continue as a case of Unauthorized (and attempt to retry) if (command.getCurrentRequest().getMethod().equals("PUT") && command.getCurrentRequest().getHeaders().containsEntry("Expect", "100-continue") && response instanceof ProtocolException && response.getMessage().equals("Server rejected operation") ) { logger.debug("Caught a protocol exception on a 100-continue PUT request. Attempting to retry."); return isIdempotent(command) && retryHandler.shouldRetryRequest(command, HttpResponse.builder().statusCode(401).message("Unauthorized").build()); } return isIdempotent(command) && ioRetryHandler.shouldRetryRequest(command, response); }
public Void apply(Exception from) { HttpResponseException exception = Throwables2.getFirstThrowableOfType(from, HttpResponseException.class); if (exception != null && exception.getCommand().getCurrentRequest().getMethod().equals(HttpMethod.DELETE) && exception.getResponse().getStatusCode() == 302) { return null; } throw Throwables.propagate(from); } }