public CompletableFuture<Void> notifyDeleteSegments(String scope, String stream, Set<Long> segmentsToDelete, String delegationToken, long requestId) { return Futures.allOf(segmentsToDelete .stream() .parallel() .map(segment -> notifyDeleteSegment(scope, stream, segment, delegationToken, requestId)) .collect(Collectors.toList())); }
public CompletableFuture<Void> notifySealedSegments(String scope, String stream, List<Long> sealedSegments, String delegationToken, long requestId) { return Futures.allOf( sealedSegments .stream() .parallel() .map(id -> notifySealedSegment(scope, stream, id, delegationToken, requestId)) .collect(Collectors.toList())); }
public CompletableFuture<Void> notifyTxnCommit(final String scope, final String stream, final List<Long> segments, final UUID txnId) { return Futures.allOf(segments.stream() .parallel() .map(segment -> notifyTxnCommit(scope, stream, segment, txnId)) .collect(Collectors.toList())); }
public CompletableFuture<Void> notifyTxnAbort(final String scope, final String stream, final List<Long> segments, final UUID txnId) { return Futures.allOf(segments.stream() .parallel() .map(segment -> notifyTxnAbort(scope, stream, segment, txnId)) .collect(Collectors.toList())); }
private CompletableFuture<Void> notifyDeleteSegments(String scope, String stream, Set<Long> segmentsToDelete, String delegationToken, long requestId) { log.debug(requestId, "{}/{} deleting segments {}", scope, stream, segmentsToDelete); return Futures.allOf(segmentsToDelete.stream() .parallel() .map(segment -> streamMetadataTasks.notifyDeleteSegment(scope, stream, segment, delegationToken, requestId)) .collect(Collectors.toList())); }
private CompletableFuture<Void> sweepAll(Supplier<Set<String>> processes) { return Futures.allOf(sweepers.stream().map(sweeper -> RetryHelper.withIndefiniteRetriesAsync(() -> { if (!sweeper.isReady()) { log.trace("sweeper not ready, retrying with exponential backoff"); throw new RuntimeException("sweeper not ready"); } return sweeper.sweepFailedProcesses(processes); }, e -> log.warn(e.getMessage()), executor)).collect(Collectors.toList())); }
@Override public CompletableFuture<Void> sweepFailedProcesses(final Supplier<Set<String>> runningProcesses) { return withRetriesAsync(taskMetadataStore::getHosts, RETRYABLE_PREDICATE, Integer.MAX_VALUE, executor) .thenComposeAsync(registeredHosts -> { log.info("Hosts {} have ongoing tasks", registeredHosts); registeredHosts.removeAll(withRetries(runningProcesses, UNCONDITIONAL_PREDICATE, Integer.MAX_VALUE)); log.info("Failed hosts {} have orphaned tasks", registeredHosts); return Futures.allOf(registeredHosts.stream() .map(this::handleFailedProcess).collect(Collectors.toList())); }, executor); }
private CompletableFuture<Void> handleHostRemoved(Host host) { return Futures.allOf(sweepers.stream().map(sweeper -> { if (sweeper.isReady()) { // Note: if we find sweeper to be ready, it is possible that this processes can be swept by both // sweepFailedProcesses and handleFailedProcess. A sweep is safe and idempotent operation. return RetryHelper.withIndefiniteRetriesAsync(() -> sweeper.handleFailedProcess(host.getHostId()), e -> log.warn(e.getMessage()), executor); } else { return CompletableFuture.completedFuture((Void) null); } }).collect(Collectors.toList())); }
@Override protected void doStart() { streamMetadataStore.createBucketsRoot() .thenCompose(v -> Futures.allOf(IntStream.range(0, bucketCount).boxed().map(this::tryTakeOwnership).collect(Collectors.toList()))) .thenAccept(x -> streamMetadataStore.registerBucketOwnershipListener(this)) .whenComplete((r, e) -> { if (e != null) { notifyFailed(e); } else { notifyStarted(); } }); }
private CompletableFuture<Void> deleteSegments(Collection<String> segmentNames) { ArrayList<CompletableFuture<Void>> deletionFutures = new ArrayList<>(); for (String segmentName : segmentNames) { try { deletionFutures.add(deleteStream(segmentName)); } catch (Throwable ex) { if (Exceptions.mustRethrow(ex) || !(Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException)) { throw ex; } } } return Futures.allOf(deletionFutures); }
static CompletableFuture<Void> allOf(Collection<OperationWithCompletion> operations) { List<CompletableFuture<Void>> futures = new ArrayList<>(); operations.forEach(oc -> futures.add(oc.completion)); return Futures.allOf(futures); } }
private void executeRemovals(HashMap<String, ArrayList<TableKey>> removals, TableStore tableStore) throws Exception { val updateResult = removals.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> tableStore.remove(e.getKey(), e.getValue(), TIMEOUT))); Futures.allOf(updateResult.values()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); }
private CompletableFuture<Void> deleteSegments(Collection<String> segmentNames, StreamSegmentStore store) { val result = new ArrayList<CompletableFuture<Void>>(); for (String segmentName : segmentNames) { result.add(store.deleteStreamSegment(segmentName, TIMEOUT)); } return Futures.allOf(result); }
/** * Ensures that all Segments defined in the given collection are loaded up into the Container's metadata. * This is used to simplify a few tests that do not expect interference from MetadataStore's assignment logic * (that is, they execute operations in a certain order and assume that those ops are added to OperationProcessor queue * in that order; if MetadataStore interferes, there is no guarantee as to what this order will be). */ @SneakyThrows private void activateAllSegments(Collection<String> segmentNames, TestContext context) { val futures = segmentNames.stream() .map(s -> activateSegment(s, context.container)) .collect(Collectors.toList()); Futures.allOf(futures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); }
void waitForTxnsToComplete() { log.info("Wait for txns to complete"); if (!Futures.await(Futures.allOf(testState.txnStatusFutureList))) { log.error("Transaction futures did not complete with exceptions"); } // check for exceptions during transaction commits if (testState.getTxnWriteException.get() != null) { log.info("Unable to commit transaction:", testState.getTxnWriteException.get()); fail("Unable to commit transaction. Test failure"); } }
private ArrayList<String> createSegments(SegmentContainer container) { ArrayList<String> segmentNames = new ArrayList<>(); ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(); for (int i = 0; i < SEGMENT_COUNT; i++) { String segmentName = getSegmentName(i); segmentNames.add(segmentName); futures.add(container.createStreamSegment(segmentName, null, TIMEOUT)); } Futures.allOf(futures).join(); return segmentNames; }
private ArrayList<String> createSegments(TableStore store) throws Exception { ArrayList<String> segmentNames = new ArrayList<>(); ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(); for (int i = 0; i < SEGMENT_COUNT; i++) { String segmentName = getSegmentName(i); segmentNames.add(segmentName); futures.add(store.createSegment(segmentName, TIMEOUT)); } Futures.allOf(futures).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); return segmentNames; }
@Override public CompletableFuture<Void> deleteStreamCutBefore(StreamCutReferenceRecord record) { return getRetentionSetData() .thenCompose(data -> { RetentionSet retention = RetentionSet.fromBytes(data.getData()); RetentionSet update = RetentionSet.removeStreamCutBefore(retention, record); List<StreamCutReferenceRecord> toRemove = retention.retentionRecordsBefore(record); return Futures.allOf(toRemove.stream().map(x -> deleteStreamCutRecordData(x.getRecordingTime())).collect(Collectors.toList())) .thenCompose(x -> Futures.toVoid(updateRetentionSetData(new Data(update.toBytes(), data.getVersion())))); }); }
@Override public CompletableFuture<Void> commitTransaction(Stream stream, UUID txId) { List<CompletableFuture<Void>> futures = new ArrayList<>(); for (Segment segment : getSegmentsForStream(stream)) { futures.add(commitTxSegment(txId, segment)); } return Futures.allOf(futures); }
@Override public CompletableFuture<Void> abortTransaction(Stream stream, UUID txId) { List<CompletableFuture<Void>> futures = new ArrayList<>(); for (Segment segment : getSegmentsForStream(stream)) { futures.add(abortTxSegment(txId, segment)); } return Futures.allOf(futures); }