private static RetryableException createRetryableExceptionWithGenericMessage(Exception cause, Date retryAfter) { return new RetryableException("Timeout", cause, retryAfter); } }
@Test public void shouldCreateNewRetryableExceptionWithMatchingNonnullRetryAfterWhen503AndNotRetryableException() { Response response = makeDefaultDecoderReplyWithHeadersWhenReceivingResponse( STATUS_503, NON_RETRYABLE_EXCEPTION, HEADERS_WITH_RETRY_AFTER); Exception exception = atlasDbDecoder.decode(EMPTY_METHOD_KEY, response); assertThat(((RetryableException) exception).retryAfter(), is(RETRY_AFTER_DATE)); }
@Override public void continueOrPropagate(RetryableException exception) { failedAttemptsForCurrentServer.increment(); if (backoffStrategy.backoff(failedAttemptsForCurrentServer.get())) { // Use same server again. log.info("{}: {}. Attempt #{} failed for server {}. Retrying the same server.", exception.getCause(), exception.getMessage(), failedAttemptsForCurrentServer.get(), servers.get(currentServer.get())); } else { // Use next server or fail if all servers have failed. failedServers.increment(); if (failedServers.get() >= servers.size()) { // Attempted to call all servers - propagate exception. // Note: Not resetting state here since Feign calls clone() before re-using this retryer. throw exception; } else { // Call next server in list. log.info("{}: {}. Server #{} ({}) failed {} times - trying next server", exception.getCause(), exception.getMessage(), failedServers.get(), servers.get(currentServer.get()), failedAttemptsForCurrentServer.get()); currentServer.set((currentServer.get() + 1) % servers.size()); failedAttemptsForCurrentServer.set(0); } } }
@Test @Theory public void testConnectionError_whenOneCallFailsThenSubsequentNewCallsCanStillSucceed( @FromDataPoints("AllStrategies") FailoverTestCase failoverTestCase) throws Exception { TestService proxy = failoverTestCase.getProxy(); // Call fails when servers are down. failoverTestCase.server1.shutdown(); failoverTestCase.server2.shutdown(); try { proxy.string(); fail(); } catch (RetryableException e) { assertThat(e.getMessage(), startsWith("Failed to complete the request due to an IOException")); } // Subsequent call (with the same proxy instance) succeeds. MockWebServer anotherServer1 = new MockWebServer(); // Not a @Rule so we can control start/stop/port explicitly anotherServer1.start(failoverTestCase.server1.getPort()); anotherServer1.enqueue(new MockResponse().setBody("\"foo\"")); assertThat(proxy.string(), is("foo")); anotherServer1.shutdown(); }
private RetryableException retryable(Throwable ex) { return new RetryableException("", ex, Date.from(Instant.EPOCH)); }
@Test public void shouldCreateNewRetryableExceptionWithMatchingNullRetryAfterWhen503AndNotRetryableException() { Response response = makeDefaultDecoderReplyWhenReceivingResponse(STATUS_503, NON_RETRYABLE_EXCEPTION); Exception exception = atlasDbDecoder.decode(EMPTY_METHOD_KEY, response); assertNull(((RetryableException) exception).retryAfter()); }
static FeignException errorExecuting(Request request, IOException cause) { return new RetryableException( format("%s executing %s %s", cause.getMessage(), request.method(), request.url()), cause, null); } }
@Test public void shouldCreateNewRetryableExceptionWithNullRetryAfterWhen429WithNullRetryAfterAndNotRetryableException() { Response response = makeDefaultDecoderReplyWhenReceivingResponse(STATUS_429, NON_RETRYABLE_EXCEPTION); Exception exception = atlasDbDecoder.decode(EMPTY_METHOD_KEY, response); assertNull(((RetryableException) exception).retryAfter()); }
@Override public Exception decode(String methodKey, Response response) { Exception exception = super.decode(methodKey, response); if (response.status() >= 500) { return new RetryableException( exception.getMessage(), exception, null ); } return exception; } }
@Test public void shouldCreateNewRetryableExceptionWithNullRetryAfterWhen429WithRetryAfterAndNotRetryableException() { Response response = makeDefaultDecoderReplyWithHeadersWhenReceivingResponse( STATUS_429, NON_RETRYABLE_EXCEPTION, HEADERS_WITH_RETRY_AFTER); Exception exception = atlasDbDecoder.decode(EMPTY_METHOD_KEY, response); assertNull(((RetryableException) exception).retryAfter()); }
/** * Returns the ID of the object created or null. */ static String awaitComplete(CloudDNS api, Job job) { RetryableException retryableException = new RetryableException( format("Job %s did not complete. Check your logs.", job.id), null); Retryer retryer = new Retryer.Default(500, 1000, 30); while (true) { job = api.getStatus(job.id); if ("COMPLETED".equals(job.status)) { return job.resultId; } else if ("ERROR".equals(job.status)) { throw new IllegalStateException( format("Job %s failed with error: %s", job.id, job.errorDetails)); } retryer.continueOrPropagate(retryableException); } }
public static ReactiveRetryPolicy retryWithBackoff(int maxRetries, long periodInMs) { return (error, attemptNo) -> { if (attemptNo <= maxRetries) { long delay; Date retryAfter; // "Retry-After" header set if (error instanceof RetryableException && (retryAfter = ((RetryableException) error) .retryAfter()) != null) { delay = retryAfter.getTime() - System.currentTimeMillis(); delay = Math.min(delay, periodInMs); delay = Math.max(delay, 0); } else { delay = periodInMs; } return delay; } else { return -1; } }; }
@Override public Exception decode(String methodKey, Response response) { FeignException exception = errorStatus(methodKey, response); Date retryAfter = retryAfterDecoder.apply(firstOrNull(response.headers(), RETRY_AFTER)); if (retryAfter != null) { return new RetryableException(exception.getMessage(), exception, retryAfter); } return exception; }
public static ReactiveRetryPolicy retryWithDelay( int maxRetries, long period) { return (error, attemptNo) -> { if (attemptNo <= maxRetries) { long delay; Date retryAfter; // "Retry-After" header set if (error instanceof RetryableException && (retryAfter = ((RetryableException) error) .retryAfter()) != null) { delay = retryAfter.getTime() - System.currentTimeMillis(); delay = Math.min(delay, period); delay = Math.max(delay, 0); } else { delay = period; } return delay; } else { return -1; } }; }
@Override public Exception decode(String methodKey, Response response) { String details; try { details = IOUtils.toString(response.body().asInputStream(), "UTF-8"); } catch (NullPointerException | IOException e) { details = "Unable to read response body"; } DCOSException e = new DCOSException(response.status(), response.reason(), methodKey, details); // WARNING: THIS IF STATEMENT IS NOT THREAD SAFE // TODO: Fix the thread safety of this if statement or token refresh/clear in general if (authTokenHeaderInterceptor != null && response.status() == 401 && authTokenHeaderInterceptor.hasToken()) { authTokenHeaderInterceptor.clearToken(); return new RetryableException(response.reason(), e, null); } return e; } }
public void continueOrPropagate(RetryableException e) { if (attempt++ >= maxAttempts) { throw e; } long interval; if (e.retryAfter() != null) { interval = e.retryAfter().getTime() - currentTimeMillis(); if (interval > maxPeriod) { interval = maxPeriod; } if (interval < 0) { return; } } else { interval = nextMaxInterval(); } try { Thread.sleep(interval); } catch (InterruptedException ignored) { Thread.currentThread().interrupt(); } sleptForMillis += interval; }
return new RetryableException("getting service unavailable, retrying ...", Request.HttpMethod.GET, null);
return new RetryableException("getting service unavailable, retrying ...", Request.HttpMethod.GET, null);
return new RetryableException("getting service unavailable, retrying ...", Request.HttpMethod.GET, null);
case 500: //INTERNAL_SERVER_ERROR case 503: //SERVICE_UNAVAILABLE return new RetryableException(message, null); default: return new MetacatException(message);