public static <V> V runWithRetries( Callable<V> callable, RetrySettings retrySettings, ResultRetryAlgorithm<?> resultRetryAlgorithm, ApiClock clock) throws RetryHelperException { try { // Suppressing should be ok as a workaraund. Current and only ResultRetryAlgorithm // implementation does not use response at all, so ignoring its type is ok. @SuppressWarnings("unchecked") ResultRetryAlgorithm<V> algorithm = (ResultRetryAlgorithm<V>) resultRetryAlgorithm; return run(callable, new ExponentialRetryAlgorithm(retrySettings, clock), algorithm); } catch (Exception e) { // TODO: remove RetryHelperException, throw InterruptedException or // ExecutionException#getCause() explicitly throw new RetryHelperException(e.getCause()); } }
/** * @param firestore The Firestore Database client. * @param query The query that is used to order the document snapshots returned by this watch. * @param target A Firestore 'Target' proto denoting the target to listen on. */ private Watch(FirestoreImpl firestore, Query query, Target target) { this.firestore = firestore; this.target = target; this.query = query; this.comparator = query.comparator(); this.backoff = new ExponentialRetryAlgorithm(RETRY_SETTINGS, CurrentMillisClock.getDefaultClock()); this.firestoreExecutor = firestore.getClient().getExecutor(); this.isActive = new AtomicBoolean(); this.nextAttempt = backoff.createFirstAttempt(); }
/** * Re-opens the stream unless the specified error is considered permanent. Clears the change map. */ private void maybeReopenStream(Throwable throwable) { if (isActive.get() && !isPermanentError(throwable)) { if (isResourceExhaustedError(throwable)) { nextAttempt = backoff.createNextAttempt(nextAttempt); } changeMap.clear(); resetStream(); } else { closeStream(throwable); } }
@Test public void testShouldRetryTrue() { TimedAttemptSettings attempt = algorithm.createFirstAttempt(); for (int i = 0; i < 2; i++) { attempt = algorithm.createNextAttempt(attempt); } assertTrue(algorithm.shouldRetry(attempt)); }
@Test public void testCreateNextAttempt() { TimedAttemptSettings firstAttempt = algorithm.createFirstAttempt(); TimedAttemptSettings secondAttempt = algorithm.createNextAttempt(firstAttempt); // Checking only the most core values, to not make this test too implementation specific. assertEquals(1, secondAttempt.getAttemptCount()); assertEquals(1, secondAttempt.getOverallAttemptCount()); assertEquals(Duration.ofMillis(1L), secondAttempt.getRetryDelay()); assertEquals(Duration.ofMillis(1L), secondAttempt.getRandomizedRetryDelay()); assertEquals(Duration.ofMillis(2L), secondAttempt.getRpcTimeout()); TimedAttemptSettings thirdAttempt = algorithm.createNextAttempt(secondAttempt); assertEquals(2, thirdAttempt.getAttemptCount()); assertEquals(Duration.ofMillis(2L), thirdAttempt.getRetryDelay()); assertEquals(Duration.ofMillis(2L), thirdAttempt.getRandomizedRetryDelay()); assertEquals(Duration.ofMillis(4L), thirdAttempt.getRpcTimeout()); }
/** * Returns {@code true} if another poll operation should be made or throws {@link * CancellationException} otherwise. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if more attempts should be made, never returns {@code false} (throws * {@code CancellationException} instead) * @throws CancellationException if no more attempts should be made */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) throws CancellationException { if (super.shouldRetry(nextAttemptSettings)) { return true; } throw new CancellationException(); } }
nextAttempt = backoff.createFirstAttempt();
.setRetryDelay(Duration.ofMillis(newRetryDelay)) .setRpcTimeout(Duration.ofMillis(newRpcTimeout)) .setRandomizedRetryDelay(Duration.ofMillis(nextRandomLong(newRetryDelay))) .setAttemptCount(prevSettings.getAttemptCount() + 1) .setOverallAttemptCount(prevSettings.getOverallAttemptCount() + 1)
@Test public void testShouldRetryFalseOnMaxAttempts() { TimedAttemptSettings attempt = algorithm.createFirstAttempt(); for (int i = 0; i < 6; i++) { assertTrue(algorithm.shouldRetry(attempt)); attempt = algorithm.createNextAttempt(attempt); } assertFalse(algorithm.shouldRetry(attempt)); }
/** * Returns {@code true} if another poll operation should be made or throws {@link * CancellationException} otherwise. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if more attempts should be made, never returns {@code false} (throws * {@code CancellationException} instead) * @throws CancellationException if no more attempts should be made */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) throws CancellationException { if (super.shouldRetry(nextAttemptSettings)) { return true; } throw new CancellationException(); } }
nextAttempt = backoff.createFirstAttempt();
.setRetryDelay(Duration.ofMillis(newRetryDelay)) .setRpcTimeout(Duration.ofMillis(newRpcTimeout)) .setRandomizedRetryDelay(Duration.ofMillis(nextRandomLong(newRetryDelay))) .setAttemptCount(prevSettings.getAttemptCount() + 1) .setOverallAttemptCount(prevSettings.getOverallAttemptCount() + 1)
/** * Internal helper to create the base MutateRows callable chain. The chain is responsible for * retrying individual entry in case of error. * * @see MutateRowsRetryingCallable for more details */ private UnaryCallable<MutateRowsRequest, Void> createMutateRowsBaseCallable() { RetryAlgorithm<Void> retryAlgorithm = new RetryAlgorithm<>( new ApiResultRetryAlgorithm<Void>(), new ExponentialRetryAlgorithm( settings.bulkMutateRowsSettings().getRetrySettings(), clientContext.getClock())); RetryingExecutor<Void> retryingExecutor = new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); return new MutateRowsRetryingCallable( clientContext.getDefaultCallContext(), stub.mutateRowsCallable(), retryingExecutor, settings.bulkMutateRowsSettings().getRetryableCodes()); }
@Test public void testShouldRetryFalseOnMaxTimeout() { TimedAttemptSettings attempt = algorithm.createFirstAttempt(); for (int i = 0; i < 4; i++) { assertTrue(algorithm.shouldRetry(attempt)); attempt = algorithm.createNextAttempt(attempt); clock.incrementNanoTime(Duration.ofMillis(50L).toNanos()); } assertFalse(algorithm.shouldRetry(attempt)); } }
/** * @param firestore The Firestore Database client. * @param query The query that is used to order the document snapshots returned by this watch. * @param target A Firestore 'Target' proto denoting the target to listen on. */ private Watch(FirestoreImpl firestore, Query query, Target target) { this.firestore = firestore; this.target = target; this.query = query; this.comparator = query.comparator(); this.backoff = new ExponentialRetryAlgorithm(RETRY_SETTINGS, CurrentMillisClock.getDefaultClock()); this.firestoreExecutor = firestore.getClient().getExecutor(); this.isActive = new AtomicBoolean(); this.nextAttempt = backoff.createFirstAttempt(); }
/** * Returns {@code true} if another poll operation should be made or throws {@link PollException}, * if either total timeout or total number of attempts is exceeded. * * @param nextAttemptSettings attempt settings, which will be used for the next attempt, if * accepted * @return {@code true} if more attempts should be made, never returns {@code false} (throws * {@code PollException} instead) * @throws PollException if no more attempts should be made */ @Override public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings) throws PollException { if (super.shouldRetry(nextAttemptSettings)) { return true; } throw new PollException( "total timeout or maximum number of attempts exceeded; current settings: " + nextAttemptSettings.getGlobalSettings()); } }
@Override public void run() { if (!isActive.get()) { return; } synchronized (Watch.this) { if (!isActive.get()) { return; } Preconditions.checkState(stream == null); current = false; nextAttempt = backoff.createNextAttempt(nextAttempt); stream = firestore.streamRequest(Watch.this, firestore.getClient().listenCallable()); ListenRequest.Builder request = ListenRequest.newBuilder(); request.setDatabase(firestore.getDatabaseName()); request.setAddTarget(target); if (resumeToken != null) { request.getAddTargetBuilder().setResumeToken(resumeToken); } stream.onNext(request.build()); } } },
@Test public void testCreateFirstAttempt() { TimedAttemptSettings attempt = algorithm.createFirstAttempt(); // Checking only the most core values, to not make this test too implementation specific. assertEquals(0, attempt.getAttemptCount()); assertEquals(0, attempt.getOverallAttemptCount()); assertEquals(Duration.ZERO, attempt.getRetryDelay()); assertEquals(Duration.ZERO, attempt.getRandomizedRetryDelay()); assertEquals(Duration.ofMillis(1L), attempt.getRpcTimeout()); assertEquals(Duration.ZERO, attempt.getRetryDelay()); }
public static <V> V runWithRetries( Callable<V> callable, RetrySettings retrySettings, ResultRetryAlgorithm<?> resultRetryAlgorithm, ApiClock clock) throws RetryHelperException { try { // Suppressing should be ok as a workaraund. Current and only ResultRetryAlgorithm // implementation does not use response at all, so ignoring its type is ok. @SuppressWarnings("unchecked") ResultRetryAlgorithm<V> algorithm = (ResultRetryAlgorithm<V>) resultRetryAlgorithm; return run(callable, new ExponentialRetryAlgorithm(retrySettings, clock), algorithm); } catch (Exception e) { // TODO: remove RetryHelperException, throw InterruptedException or // ExecutionException#getCause() explicitly throw new RetryHelperException(e.getCause()); } }
protected Long getNextBackoff() { if (currentBackoff == null) { // Historically, the client waited for "total timeout" after the first failure. For now, // that behavior is preserved, even though that's not the ideal. // // TODO: Think through retries, and create policy that works with the mental model most // users would have of relating to retries. That would likely involve updating some // default settings in addition to changing the algorithm. currentBackoff = exponentialRetryAlgorithm.createFirstAttempt(); } currentBackoff = exponentialRetryAlgorithm.createNextAttempt(currentBackoff); if (!exponentialRetryAlgorithm.shouldRetry(currentBackoff)) { // TODO: consider creating a subclass of exponentialRetryAlgorithm to encapsulate this logic long timeLeftNs = currentBackoff.getGlobalSettings().getTotalTimeout().toNanos() - (clock.nanoTime() - currentBackoff.getFirstAttemptStartTimeNanos()); long timeLeftMs = TimeUnit.NANOSECONDS.toMillis(timeLeftNs); if (timeLeftMs > currentBackoff.getGlobalSettings().getInitialRetryDelay().toMillis()) { // The backoff algorithm doesn't always wait until the timeout is achieved. Wait // one final time so that retries hit return timeLeftMs; } else { // Finish for real. return null; } } else { return currentBackoff.getRetryDelay().toMillis(); } }