@Override public CompletableFuture<SegmentRead> read(long offset, int length) { Exceptions.checkNotClosed(closed.get(), this); WireCommands.ReadSegment request = new WireCommands.ReadSegment(segmentId.getScopedName(), offset, length, this.delegationToken); return backoffSchedule.retryWhen(t -> { Throwable ex = Exceptions.unwrap(t); if (closed.get()) { log.debug("Exception while reading from Segment : {}", segmentId, ex); } else { log.warn("Exception while reading from Segment : {}", segmentId, ex); } return ex instanceof Exception && !(ex instanceof ConnectionClosedException) && !(ex instanceof SegmentTruncatedException); }).runAsync(() -> { return getConnection() .whenComplete((connection, ex) -> { if (ex != null) { log.warn("Exception while establishing connection with Pravega " + "node", ex); closeConnection(new ConnectionFailedException(ex)); } }).thenCompose(c -> sendRequestOverConnection(request, c)); }, connectionFactory.getInternalExecutor()); }
private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) { for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) { String segmentName = e.getKey(); byte[] expectedData = e.getValue().toByteArray(); long segmentLength = store.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength(); Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, segmentLength); AtomicLong expectedCurrentOffset = new AtomicLong(0); // We retry a number of times on StreamSegmentNotExists. It is possible that waitForSegmentsInStorage may have // returned successfully because it detected the Segment was complete there, but the internal callback to the // ReadIndex (completeMerge) may not yet have been executed. The ReadIndex has a mechanism to cope with this, // but it only retries once, after a fixed time interval, which is more than generous on any system. // However, on very slow systems, it is possible that that callback may take a significant amount of time to even // begin executing, hence the trying to read data that was merged from a Transaction may result in a spurious // StreamSegmentNotExistsException. // This is gracefully handled by retries in AppendProcessor and/or Client, but in this case, we simply have to // do the retries ourselves, hoping that the callback eventually executes. Retry.withExpBackoff(100, 2, 10, TIMEOUT.toMillis() / 5) .retryWhen(ex -> Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) .run(() -> { checkSegmentReads(segmentName, expectedCurrentOffset, segmentLength, store, expectedData); return null; }); } }
public RetryAndThrowConditionally retryWhen(Predicate<Throwable> predicate) { Preconditions.checkNotNull(predicate); return new RetryAndThrowConditionally(predicate, this); } }
Retry.withExpBackoff(500, 2, 10) .retryWhen(ex -> true) .run(() -> this.streamManager.get().createScope(SCOPE));
@Override public CompletableFuture<Void> processEvent(TestBase event) { receivedForProcessing.add(event); CompletableFuture<Void> result = new CompletableFuture<>(); Retry.withExpBackoff(100, 1, 5, 100) .retryWhen(RetryableException::isRetryable) .runAsync(() -> event.process(null), executor) .whenCompleteAsync((r, e) -> { if (e != null) { Throwable cause = Exceptions.unwrap(e); if (cause instanceof OperationDisallowedException) { Retry.indefinitelyWithExpBackoff("Error writing event back into requeststream") .runAsync(() -> writer.write(event), executor) .thenAccept(v -> result.completeExceptionally(cause)); } else { result.completeExceptionally(cause); } } else { result.complete(r); } }, executor); return result; } }
/** * Creates a new Segment Store Instance, with retries. * Normally we have the Controller coordinating which instances are the rightful survivors, however in this case * we need to simulate some of this behavior ourselves, by being insistent. It is possible that previous instances * meddle with the BKLog ZK metadata during the new instance's initialization, causing the new instance to wrongfully * assume it's not the rightful survivor. A quick retry solves this problem, as there is no other kind of information * available to disambiguate this. */ void createNewInstance() { this.newInstanceRetry.run(() -> { int instanceId = getIteration() + 1; log.info("Starting Instance {}.", instanceId); ServiceBuilder b = createBuilder(instanceId); this.builders.add(b); this.activeStore.set(b.createStreamSegmentService()); this.iteration.incrementAndGet(); log.info("Instance {} Started.", instanceId); return null; }); } }
@Override protected void doStart() { log.info("{}: Starting.", this.traceObjectId); this.delayedStartRetry .runAsync(() -> tryStartOnce() .whenComplete((v, ex) -> { if (ex == null) { // We are done. log.info("{}: Online.", this.traceObjectId); notifyDelayedStartComplete(null); } else { if (Exceptions.unwrap(ex) instanceof DataLogDisabledException) { // Place the DurableLog in a Started State, but keep trying to restart. notifyStartComplete(null); } throw new CompletionException(ex); } }), this.executor) .exceptionally(this::notifyDelayedStartComplete); }
/** * Executes the given request on the given FencingTextContext.. We retry all expected exceptions, and when we do, we * make sure to execute them on the current (active) Segment Store instance (since the previous one may be unusable). */ private CompletableFuture<Void> executeWithFencing(StoreRequest request, int index, FencingTestContext context) { log.debug("Initiating Operation #{} on iteration {}.", index, context.getIteration()); AtomicReference<StreamSegmentStore> requestStore = new AtomicReference<>(context.getActiveStore()); return Retry.withExpBackoff(50, 2, 10, TIMEOUT.toMillis() / 10) .retryWhen(ex -> { requestStore.getAndSet(context.getActiveStore()); ex = Exceptions.unwrap(ex); log.info("Operation #{} (Iteration = {}) failed due to {}.", index, context.getIteration(), ex.toString()); return isExpectedFencingException(ex); }) .runAsync(() -> request.apply(requestStore.get()), executorService()); }
public static <U> CompletableFuture<U> withRetries(Supplier<CompletableFuture<U>> futureSupplier, ScheduledExecutorService executor) { return RETRY.runAsync(futureSupplier, executor); }
@Test public void retryPredicateTest() { AtomicInteger i = new AtomicInteger(0); try { Retry.withExpBackoff(10, 10, 10) .retryWhen(e -> i.getAndIncrement() != 1) .run(() -> { throw new Exception("test"); }); } catch (Exception e) { assert i.get() == 2; } }
static <U> CompletableFuture<U> withRetries(Supplier<CompletableFuture<U>> futureSupplier, ScheduledExecutorService executor) { return RETRY.runAsync(futureSupplier, executor); } }
public static <U> U withRetries(Supplier<U> supplier, Predicate<Throwable> predicate, int numOfTries) { return Retry.withExpBackoff(100, 2, numOfTries, 1000) .retryWhen(predicate) .run(supplier::get); }
private CompletableFuture<Map<UUID, Long>> getAttributesForSegment(long segmentId, Collection<UUID> attributeIds, boolean cache, TimeoutTimer timer) { SegmentMetadata metadata = this.metadata.getStreamSegmentMetadata(segmentId); if (cache) { return CACHE_ATTRIBUTES_RETRY.runAsync(() -> getAndCacheAttributes(metadata, attributeIds, cache, timer), StreamSegmentContainer.this.executor); } else { return getAndCacheAttributes(metadata, attributeIds, cache, timer); } }
public static <U> CompletableFuture<U> withRetriesAsync(Supplier<CompletableFuture<U>> futureSupplier, Predicate<Throwable> predicate, int numOfTries, ScheduledExecutorService executor) { return Retry .withExpBackoff(100, 2, numOfTries, 10000) .retryWhen(predicate) .runAsync(futureSupplier, executor); }