@Override protected void enqueueInner(RawFragmentBatch batch) { assert batch.getHeader().getSendingMajorFragmentId() == oppositeId; logger.debug("Enqueue batch. Current buffer size: {}. Sending fragment: {}", bufferQueue.size(), batch.getHeader().getSendingMajorFragmentId()); RawFragmentBatchWrapper wrapper; boolean spoolCurrentBatch = isCurrentlySpooling(); wrapper = new RawFragmentBatchWrapper(batch, !spoolCurrentBatch); currentBatchesInMemory++; if (spoolCurrentBatch) { addBatchForSpooling(wrapper); } bufferQueue.add(wrapper); if (!spoolCurrentBatch && currentBatchesInMemory >= threshold) { logger.debug("Buffer size {} greater than threshold {}. Start spooling to disk", currentBatchesInMemory, threshold); startSpooling(); } }
hash = (53 * hash) + getSendingMajorFragmentId();
public void handle(IncomingDataBatch batch) throws FragmentSetupException, IOException { Preconditions.checkState(fragmentStarted, formatError("data batch", batch.getHeader().getSendingMajorFragmentId(), batch.getHeader().getSendingMinorFragmentId())); if (executor != null) { executor.getListener().handle(batch); } // A missing executor means it already terminated. We can simply drop this message. }
result = result && (hasSendingMajorFragmentId() == other.hasSendingMajorFragmentId()); if (hasSendingMajorFragmentId()) { result = result && (getSendingMajorFragmentId() == other.getSendingMajorFragmentId());
public Builder mergeFrom(com.dremio.exec.proto.ExecRPC.FragmentRecordBatch other) { if (other == com.dremio.exec.proto.ExecRPC.FragmentRecordBatch.getDefaultInstance()) return this; if (other.hasQueryId()) { mergeQueryId(other.getQueryId()); } if (other.hasReceivingMajorFragmentId()) { setReceivingMajorFragmentId(other.getReceivingMajorFragmentId()); } if (!other.receivingMinorFragmentId_.isEmpty()) { if (receivingMinorFragmentId_.isEmpty()) { receivingMinorFragmentId_ = other.receivingMinorFragmentId_; bitField0_ = (bitField0_ & ~0x00000004); } else { ensureReceivingMinorFragmentIdIsMutable(); receivingMinorFragmentId_.addAll(other.receivingMinorFragmentId_); } onChanged(); } if (other.hasSendingMajorFragmentId()) { setSendingMajorFragmentId(other.getSendingMajorFragmentId()); } if (other.hasSendingMinorFragmentId()) { setSendingMinorFragmentId(other.getSendingMinorFragmentId()); } if (other.hasArrowRecordBatch()) { setArrowRecordBatch(other.getArrowRecordBatch()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
public void batchArrived(final IncomingDataBatch incomingBatch) throws FragmentSetupException, IOException { if(!incomingBatch.checkAcceptance(allocator.getHeadroom())){ deferredException.addException(UserException.memoryError() .message("Out of memory while receiving incoming message. Message size: %d, Current thread allocation: %d, thread limit: %d.", incomingBatch.size(), allocator.getAllocatedMemory(), allocator.getLimit()) .build(logger)); return; } // we want to make sure that we only generate local record batch reference in the case that we're not closed. // Otherwise we would leak memory. try (AutoCloseableLock lock = sharedIncomingBatchLock.open()) { if (closed) { return; } final DataCollector collector = collector(incomingBatch.getHeader().getSendingMajorFragmentId()); synchronized (collector) { try(final RawFragmentBatch newRawFragmentBatch = incomingBatch.newRawFragmentBatch(allocator)){ collector.batchArrived(incomingBatch.getHeader().getSendingMinorFragmentId(), newRawFragmentBatch); } } } }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.ExecRPC.FragmentRecordBatch message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), com.dremio.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasReceivingMajorFragmentId()) output.writeInt32(2, message.getReceivingMajorFragmentId(), false); for(int receivingMinorFragmentId : message.getReceivingMinorFragmentIdList()) output.writeInt32(3, receivingMinorFragmentId, true); if(message.hasSendingMajorFragmentId()) output.writeInt32(4, message.getSendingMajorFragmentId(), false); if(message.hasSendingMinorFragmentId()) output.writeInt32(5, message.getSendingMinorFragmentId(), false); if(message.hasArrowRecordBatch()) output.writeByteArray(6, message.getArrowRecordBatch().toByteArray(), false); } public boolean isInitialized(com.dremio.exec.proto.ExecRPC.FragmentRecordBatch message)