@Override public boolean isFinished() { State state = stateReference.get(); switch (state.getKind()) { case INITIAL: return false; case NO_MORE_SPLITS: return bufferedInternalSplitCount.get() == 0; case FAILED: throw propagatePrestoException(state.getThrowable()); case CLOSED: throw new IllegalStateException("HiveSplitSource is already closed"); default: throw new UnsupportedOperationException(); } }
ListenableFuture<?> addToQueue(InternalHiveSplit split) { if (stateReference.get().getKind() != INITIAL) { return immediateFuture(null); } if (estimatedSplitSizeInBytes.addAndGet(split.getEstimatedSizeInBytes()) > maxOutstandingSplitsBytes) { // TODO: investigate alternative split discovery strategies when this error is hit. // This limit should never be hit given there is a limit of maxOutstandingSplits. // If it's hit, it means individual splits are huge. if (loggedHighMemoryWarning.compareAndSet(false, true)) { highMemorySplitSourceCounter.update(1); log.warn("Split buffering for %s.%s in query %s exceeded memory limit (%s). %s splits are buffered.", databaseName, tableName, queryId, succinctBytes(maxOutstandingSplitsBytes), getBufferedInternalSplitCount()); } throw new PrestoException(HIVE_EXCEEDED_SPLIT_BUFFERING_LIMIT, format( "Split buffering for %s.%s exceeded memory limit (%s). %s splits are buffered.", databaseName, tableName, succinctBytes(maxOutstandingSplitsBytes), getBufferedInternalSplitCount())); } bufferedInternalSplitCount.incrementAndGet(); OptionalInt bucketNumber = split.getBucketNumber(); return queues.offer(bucketNumber, split); }
switch (state.getKind()) { case INITIAL: noMoreSplits = false;
void fail(Throwable e) { // The error must be recorded before setting the finish marker to make sure // isFinished will observe failure instead of successful completion. // Only record the first error message. if (setIf(stateReference, State.failed(e), state -> state.getKind() == INITIAL)) { // Stop the split loader before finishing the queue. // Once the queue is finished, it will always return a completed future to avoid blocking any caller. // This could lead to a short period of busy loop in splitLoader (although unlikely in general setup). splitLoader.stop(); queues.finish(); } }
void noMoreSplits() { if (setIf(stateReference, State.noMoreSplits(), state -> state.getKind() == INITIAL)) { // Stop the split loader before finishing the queue. // Once the queue is finished, it will always return a completed future to avoid blocking any caller. // This could lead to a short period of busy loop in splitLoader (although unlikely in general setup). splitLoader.stop(); queues.finish(); } }
@Override public void close() { if (setIf(stateReference, State.closed(), state -> state.getKind() == INITIAL || state.getKind() == NO_MORE_SPLITS)) { // Stop the split loader before finishing the queue. // Once the queue is finished, it will always return a completed future to avoid blocking any caller. // This could lead to a short period of busy loop in splitLoader (although unlikely in general setup). splitLoader.stop(); queues.finish(); } }