@Override public TimedAttemptSettings createNextAttempt( Throwable prevThrowable, ResponseT prevResponse, TimedAttemptSettings prevSettings) { if (prevThrowable != null && prevThrowable instanceof DeadlineExceededException) { return TimedAttemptSettings.newBuilder() .setGlobalSettings(prevSettings.getGlobalSettings()) .setRetryDelay(prevSettings.getRetryDelay()) .setRpcTimeout(prevSettings.getRpcTimeout()) .setRandomizedRetryDelay(DEADLINE_SLEEP_DURATION) .setAttemptCount(prevSettings.getAttemptCount() + 1) .setFirstAttemptStartTimeNanos(prevSettings.getFirstAttemptStartTimeNanos()) .build(); } return null; }
/** Initializes a new stream to the backend with backoff. */ private void initStream() { firestoreExecutor.schedule( new Runnable() { @Override public void run() { if (!isActive.get()) { return; } synchronized (Watch.this) { if (!isActive.get()) { return; } Preconditions.checkState(stream == null); current = false; nextAttempt = backoff.createNextAttempt(nextAttempt); stream = firestore.streamRequest(Watch.this, firestore.getClient().listenCallable()); ListenRequest.Builder request = ListenRequest.newBuilder(); request.setDatabase(firestore.getDatabaseName()); request.setAddTarget(target); if (resumeToken != null) { request.getAddTargetBuilder().setResumeToken(resumeToken); } stream.onNext(request.build()); } } }, nextAttempt.getRandomizedRetryDelay().toMillis(), TimeUnit.MILLISECONDS); }
@Test public void testNoRpcTimeout() { parentFuture.timedAttemptSettings = parentFuture.timedAttemptSettings.toBuilder().setRpcTimeout(Duration.ZERO).build(); MutateRowsRequest request = MutateRowsRequest.newBuilder().addEntries(Entry.getDefaultInstance()).build(); innerCallable.response.add( MutateRowsResponse.newBuilder() .addEntries( MutateRowsResponse.Entry.newBuilder().setIndex(0).setStatus(OK_STATUS_PROTO)) .build()); MutateRowsAttemptCallable attemptCallable = new MutateRowsAttemptCallable(innerCallable, request, callContext, retryCodes); attemptCallable.setExternalFuture(parentFuture); attemptCallable.call(); assertThat(innerCallable.lastContext.getTimeout()).isNull(); }
@Test public void testCreateFirstAttempt() { TimedAttemptSettings attempt = algorithm.createFirstAttempt(); // Checking only the most core values, to not make this test too implementation specific. assertEquals(0, attempt.getAttemptCount()); assertEquals(0, attempt.getOverallAttemptCount()); assertEquals(Duration.ZERO, attempt.getRetryDelay()); assertEquals(Duration.ZERO, attempt.getRandomizedRetryDelay()); assertEquals(Duration.ofMillis(1L), attempt.getRpcTimeout()); assertEquals(Duration.ZERO, attempt.getRetryDelay()); }
/** * Returns {@code true} if another attempt should be made, or {@code false} otherwise. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if {@code nextAttemptSettings} does not exceed either maxAttempts limit or * totalTimeout limit, or {@code false} otherwise */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) { RetrySettings globalSettings = nextAttemptSettings.getGlobalSettings(); long totalTimeSpentNanos = clock.nanoTime() - nextAttemptSettings.getFirstAttemptStartTimeNanos() + nextAttemptSettings.getRandomizedRetryDelay().toNanos(); return totalTimeSpentNanos <= globalSettings.getTotalTimeout().toNanos() && (globalSettings.getMaxAttempts() <= 0 || nextAttemptSettings.getAttemptCount() < globalSettings.getMaxAttempts()); }
protected Long getNextBackoff() { if (currentBackoff == null) { // Historically, the client waited for "total timeout" after the first failure. For now, // that behavior is preserved, even though that's not the ideal. // // TODO: Think through retries, and create policy that works with the mental model most // users would have of relating to retries. That would likely involve updating some // default settings in addition to changing the algorithm. currentBackoff = exponentialRetryAlgorithm.createFirstAttempt(); } currentBackoff = exponentialRetryAlgorithm.createNextAttempt(currentBackoff); if (!exponentialRetryAlgorithm.shouldRetry(currentBackoff)) { // TODO: consider creating a subclass of exponentialRetryAlgorithm to encapsulate this logic long timeLeftNs = currentBackoff.getGlobalSettings().getTotalTimeout().toNanos() - (clock.nanoTime() - currentBackoff.getFirstAttemptStartTimeNanos()); long timeLeftMs = TimeUnit.NANOSECONDS.toMillis(timeLeftNs); if (timeLeftMs > currentBackoff.getGlobalSettings().getInitialRetryDelay().toMillis()) { // The backoff algorithm doesn't always wait until the timeout is achieved. Wait // one final time so that retries hit return timeLeftMs; } else { // Finish for real. return null; } } else { return currentBackoff.getRetryDelay().toMillis(); } }
if (!outerRetryingFuture.getAttemptSettings().getRpcTimeout().isZero()) { attemptContext = attemptContext.withStreamWaitTimeout( outerRetryingFuture.getAttemptSettings().getRpcTimeout()); .attemptStarted(outerRetryingFuture.getAttemptSettings().getOverallAttemptCount());
/** * {@inheritDoc} * * <p>The attempt settings will be reset if the stream attempt produced any messages. */ @Override public TimedAttemptSettings createNextAttempt( Throwable prevThrowable, ResponseT prevResponse, TimedAttemptSettings prevSettings) { if (prevThrowable instanceof ServerStreamingAttemptException) { ServerStreamingAttemptException attemptException = (ServerStreamingAttemptException) prevThrowable; prevThrowable = prevThrowable.getCause(); // If we have made progress in the last attempt, then reset the delays if (attemptException.hasSeenResponses()) { prevSettings = createFirstAttempt() .toBuilder() .setFirstAttemptStartTimeNanos(prevSettings.getFirstAttemptStartTimeNanos()) .setOverallAttemptCount(prevSettings.getOverallAttemptCount()) .build(); } } return super.createNextAttempt(prevThrowable, prevResponse, prevSettings); }
@Test public void testCancelByRetryingAlgorithm() throws Exception { FailingCallable callable = new FailingCallable(6, "FAILURE", tracer); RetryingExecutorWithContext<String> executor = getExecutor(getAlgorithm(FAST_RETRY_SETTINGS, 5, new CancellationException())); RetryingFuture<String> future = executor.createFuture(callable, retryingContext); future.setAttemptFuture(executor.submit(future)); assertFutureCancel(future); assertEquals(4, future.getAttemptSettings().getAttemptCount()); verify(tracer, times(5)).attemptStarted(anyInt()); // Pre-apocalypse failures verify(tracer, times(4)).attemptFailed(any(Throwable.class), any(Duration.class)); // Apocalypse failure verify(tracer, times(1)).attemptFailedRetriesExhausted(any(CancellationException.class)); verifyNoMoreInteractions(tracer); }
MockRetryingFuture(Duration totalTimeout) { this.timedAttemptSettings = TimedAttemptSettings.newBuilder() .setRpcTimeout(Duration.ofSeconds(1)) .setRetryDelay(Duration.ZERO) .setRandomizedRetryDelay(Duration.ZERO) .setAttemptCount(0) .setFirstAttemptStartTimeNanos(0) .setGlobalSettings(RetrySettings.newBuilder().setTotalTimeout(totalTimeout).build()) .build(); }
/** * Returns {@code true} if another poll operation should be made or throws {@link PollException}, * if either total timeout or total number of attempts is exceeded. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if more attempts should be made, never returns {@code false} (throws * {@code PollException} instead) * @throws PollException if no more attempts should be made */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) throws PollException { if (super.shouldRetry(nextAttemptSettings)) { return true; } throw new PollException( "total timeout or maximum number of attempts exceeded; current settings: " + nextAttemptSettings.getGlobalSettings()); } }
if (!externalFuture.getAttemptSettings().getRpcTimeout().isZero()) { currentCallContext = currentCallContext.withTimeout(externalFuture.getAttemptSettings().getRpcTimeout());
@Test public void testCreateNextAttempt() { TimedAttemptSettings firstAttempt = algorithm.createFirstAttempt(); TimedAttemptSettings secondAttempt = algorithm.createNextAttempt(firstAttempt); // Checking only the most core values, to not make this test too implementation specific. assertEquals(1, secondAttempt.getAttemptCount()); assertEquals(1, secondAttempt.getOverallAttemptCount()); assertEquals(Duration.ofMillis(1L), secondAttempt.getRetryDelay()); assertEquals(Duration.ofMillis(1L), secondAttempt.getRandomizedRetryDelay()); assertEquals(Duration.ofMillis(2L), secondAttempt.getRpcTimeout()); TimedAttemptSettings thirdAttempt = algorithm.createNextAttempt(secondAttempt); assertEquals(2, thirdAttempt.getAttemptCount()); assertEquals(Duration.ofMillis(2L), thirdAttempt.getRetryDelay()); assertEquals(Duration.ofMillis(2L), thirdAttempt.getRandomizedRetryDelay()); assertEquals(Duration.ofMillis(4L), thirdAttempt.getRpcTimeout()); }
/** * Returns {@code true} if another attempt should be made, or {@code false} otherwise. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if {@code nextAttemptSettings} does not exceed either maxAttempts limit or * totalTimeout limit, or {@code false} otherwise */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) { RetrySettings globalSettings = nextAttemptSettings.getGlobalSettings(); long totalTimeSpentNanos = clock.nanoTime() - nextAttemptSettings.getFirstAttemptStartTimeNanos() + nextAttemptSettings.getRandomizedRetryDelay().toNanos(); return totalTimeSpentNanos <= globalSettings.getTotalTimeout().toNanos() && (globalSettings.getMaxAttempts() <= 0 || nextAttemptSettings.getAttemptCount() < globalSettings.getMaxAttempts()); }
if (!outerRetryingFuture.getAttemptSettings().getRpcTimeout().isZero()) { attemptContext = attemptContext.withStreamWaitTimeout( outerRetryingFuture.getAttemptSettings().getRpcTimeout()); .attemptStarted(outerRetryingFuture.getAttemptSettings().getOverallAttemptCount());
/** * {@inheritDoc} * * <p>The attempt settings will be reset if the stream attempt produced any messages. */ @Override public TimedAttemptSettings createNextAttempt( Throwable prevThrowable, ResponseT prevResponse, TimedAttemptSettings prevSettings) { if (prevThrowable instanceof ServerStreamingAttemptException) { ServerStreamingAttemptException attemptException = (ServerStreamingAttemptException) prevThrowable; prevThrowable = prevThrowable.getCause(); // If we have made progress in the last attempt, then reset the delays if (attemptException.hasSeenResponses()) { prevSettings = createFirstAttempt() .toBuilder() .setFirstAttemptStartTimeNanos(prevSettings.getFirstAttemptStartTimeNanos()) .setOverallAttemptCount(prevSettings.getOverallAttemptCount()) .build(); } } return super.createNextAttempt(prevThrowable, prevResponse, prevSettings); }
@Test public void testMaxRetriesExceeded() throws Exception { FailingCallable callable = new FailingCallable(6, "FAILURE", tracer); RetryingExecutorWithContext<String> executor = getExecutor(getAlgorithm(FAST_RETRY_SETTINGS, 0, null)); RetryingFuture<String> future = executor.createFuture(callable, retryingContext); future.setAttemptFuture(executor.submit(future)); assertFutureFail(future, CustomException.class); assertEquals(5, future.getAttemptSettings().getAttemptCount()); verify(tracer, times(6)).attemptStarted(anyInt()); verify(tracer, times(5)).attemptFailed(any(Throwable.class), any(Duration.class)); verify(tracer, times(1)).attemptFailedRetriesExhausted(any(Throwable.class)); verifyNoMoreInteractions(tracer); }
/** * Creates a first attempt {@link TimedAttemptSettings}. The first attempt is configured to be * executed immediately. * * @return first attempt settings */ @Override public TimedAttemptSettings createFirstAttempt() { return TimedAttemptSettings.newBuilder() .setGlobalSettings(globalSettings) .setRetryDelay(Duration.ZERO) .setRpcTimeout(globalSettings.getInitialRpcTimeout()) .setRandomizedRetryDelay(Duration.ZERO) .setAttemptCount(0) .setOverallAttemptCount(0) .setFirstAttemptStartTimeNanos(clock.nanoTime()) .build(); }
/** * Returns {@code true} if another poll operation should be made or throws {@link PollException}, * if either total timeout or total number of attempts is exceeded. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if more attempts should be made, never returns {@code false} (throws * {@code PollException} instead) * @throws PollException if no more attempts should be made */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) throws PollException { if (super.shouldRetry(nextAttemptSettings)) { return true; } throw new PollException( "total timeout or maximum number of attempts exceeded; current settings: " + nextAttemptSettings.getGlobalSettings()); } }
if (!externalFuture.getAttemptSettings().getRpcTimeout().isZero()) { currentCallContext = currentCallContext.withTimeout(externalFuture.getAttemptSettings().getRpcTimeout());