private ListenableFuture<?> fetchSplits() { if (splits.size() >= min) { return immediateFuture(null); } ListenableFuture<SplitBatch> future = splitSource.getNextBatch(partitionHandle, lifespan, max - splits.size()); return Futures.transformAsync(future, splitBatch -> { splits.addAll(splitBatch.getSplits()); if (splitBatch.isLastBatch()) { noMoreSplits = true; return immediateFuture(null); } return fetchSplits(); }, directExecutor()); } }
@Override public ListenableFuture<SplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) { ListenableFuture<ConnectorSplitBatch> nextBatch = toListenableFuture(source.getNextBatch(partitionHandle, maxSize)); return Futures.transform(nextBatch, splitBatch -> { ImmutableList.Builder<Split> result = ImmutableList.builder(); for (ConnectorSplit connectorSplit : splitBatch.getSplits()) { result.add(new Split(connectorId, transactionHandle, connectorSplit, lifespan)); } return new SplitBatch(result.build(), splitBatch.isNoMoreSplits()); }, directExecutor()); }
private static List<Split> getNextBatch(SplitSource splitSource) { return getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)).getSplits(); }
SplitBatch nextSplits = getFutureValue(scheduleGroup.nextSplitBatchFuture); scheduleGroup.nextSplitBatchFuture = null; pendingSplits.addAll(nextSplits.getSplits()); if (nextSplits.isLastBatch()) { if (scheduleGroup.state == ScheduleGroupState.INITIALIZED && pendingSplits.isEmpty()) {
while (!Thread.currentThread().isInterrupted()) { SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)); List<Split> splits = splitBatch.getSplits(); if (splitBatch.isLastBatch()) { break;
@Override public ListenableFuture<SplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) { if (partitionHandle != NOT_PARTITIONED) { throw new UnsupportedOperationException(); } checkArgument(Lifespan.taskWide().equals(lifespan)); checkState(nextBatchFuture.isDone(), "concurrent getNextBatch invocation"); nextBatchFuture = SettableFuture.create(); nextBatchMaxSize = maxSize; nextBatchInvocationCount++; doGetNextBatch(); return Futures.transform(nextBatchFuture, splits -> new SplitBatch(splits, isFinished()), directExecutor()); }
private static List<Split> getNextBatch(SplitSource splitSource) { return getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)).getSplits(); }
private static List<Split> getNextBatch(SplitSource splitSource) { return getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000)).getSplits(); }
public NextBatchResult assertNoMoreSplits(boolean expectedNoMoreSplits) { assertEquals(splitBatch.isLastBatch(), expectedNoMoreSplits); return this; } }
public NextBatchResult assertSize(int expectedSize) { assertEquals(splitBatch.getSplits().size(), expectedSize); return this; }
public static ListenableFuture<SplitBatch> fetchNextBatchAsync( SplitSource splitSource, int min, int max, ConnectorPartitionHandle partitionHandle, Lifespan lifespan) { GetNextBatch getNextBatch = new GetNextBatch(splitSource, min, max, partitionHandle, lifespan); ListenableFuture<?> future = getNextBatch.fetchSplits(); return Futures.transform(future, ignored -> new SplitBatch(getNextBatch.splits, getNextBatch.noMoreSplits), directExecutor()); }
@Override public ListenableFuture<SplitBatch> getNextBatch(ConnectorPartitionHandle partitionHandle, Lifespan lifespan, int maxSize) { ListenableFuture<SplitBatch> batch = splitSource.getNextBatch(partitionHandle, lifespan, maxSize); return Futures.transform(batch, splitBatch -> new SplitBatch( splitBatch.getSplits().stream() .filter(input -> ThreadLocalRandom.current().nextDouble() < sampleRatio) .collect(toImmutableList()), splitBatch.isLastBatch()), directExecutor()); }