break; if (subSerializationResult.isFullBuffer()) { break;
@Override public SerializationResult setNextBuffer(Buffer buffer) throws IOException { this.targetBuffer = buffer; this.position = 0; this.limit = buffer.getSize(); if (this.lengthBuffer.hasRemaining()) { copyToTargetBufferFrom(this.lengthBuffer); } if (this.dataBuffer.hasRemaining()) { copyToTargetBufferFrom(this.dataBuffer); } SerializationResult result = getSerializationResult(); // make sure we don't hold onto the large buffers for too long if (result.isFullRecord()) { this.serializationBuffer.clear(); this.serializationBuffer.pruneBuffer(); this.dataBuffer = this.serializationBuffer.wrapAsByteBuffer(); } return result; }
private boolean copyFromSerializerToTargetChannel(int targetChannel, boolean isBroadcast, boolean flushAlways) throws IOException, InterruptedException { // We should reset the initial position of the intermediate serialization data buffer before // copying, so the serialization results can be copied to many different target buffers. serializer.reset(); boolean pruneTriggered = false; BufferBuilder bufferBuilder = getBufferBuilder(targetChannel, isBroadcast); RecordSerializer.SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { updateMetrics(bufferBuilder, isBroadcast); // If this was a full record, we are done. Not breaking // out of the loop at this point will lead to another // buffer request before breaking out (that would not be // a problem per se, but it can lead to stalls in the pipeline). if (result.isFullRecord()) { pruneTriggered = true; break; } bufferBuilder = requestNewBufferBuilder(targetChannel, isBroadcast); result = serializer.copyToBufferBuilder(bufferBuilder); } checkState(!serializer.hasSerializedData(), "All data should be written at once"); if (flushAlways) { if (isBroadcast) { flushAll(); } else { flush(targetChannel); } } return pruneTriggered; }
BufferBuilder bufferBuilder = getBufferBuilder(targetChannel); SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { numBytesOut.inc(bufferBuilder.finish()); numBuffersOut.inc(); if (result.isFullRecord()) { pruneTriggered = true; bufferBuilders[targetChannel] = Optional.empty();
BufferBuilder bufferBuilder = getBufferBuilder(targetChannel); SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { numBytesOut.inc(bufferBuilder.finish()); numBuffersOut.inc(); if (result.isFullRecord()) { pruneTriggered = true; bufferBuilders[targetChannel] = Optional.empty();
private void sendToTarget(T record, int targetChannel) throws IOException, InterruptedException { RecordSerializer<T> serializer = serializers[targetChannel]; synchronized (serializer) { SerializationResult result = serializer.addRecord(record); while (result.isFullBuffer()) { Buffer buffer = serializer.getCurrentBuffer(); if (buffer != null) { numBytesOut.inc(buffer.getSize()); writeAndClearBuffer(buffer, targetChannel, serializer); // If this was a full record, we are done. Not breaking // out of the loop at this point will lead to another // buffer request before breaking out (that would not be // a problem per se, but it can lead to stalls in the // pipeline). if (result.isFullRecord()) { break; } } else { buffer = targetPartition.getBufferProvider().requestBufferBlocking(); result = serializer.setNextBuffer(buffer); } } } }
private void copyToTargetFile(int partition) throws IOException, InterruptedException { recordSerializer.reset(); BufferBuilder bufferBuilder = getCurrentBufferBuilder(partition); RecordSerializer.SerializationResult result = recordSerializer.copyToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { tryFinishCurrentBufferBuilder(partition); if (result.isFullRecord()) { break; } bufferBuilder = getCurrentBufferBuilder(partition); result = recordSerializer.copyToBufferBuilder(bufferBuilder); } checkState(!recordSerializer.hasSerializedData(), "All data should be written at once"); }
private void copyToFile() throws IOException { recordSerializer.reset(); BufferBuilder bufferBuilder = getCurrentBufferBuilder(); RecordSerializer.SerializationResult result = recordSerializer.copyToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { tryFinishCurrentBufferBuilder(); if (result.isFullRecord()) { break; } bufferBuilder = getCurrentBufferBuilder(); result = recordSerializer.copyToBufferBuilder(bufferBuilder); } }
private void flushInternalSerializer() throws IOException { BufferBuilder bufferBuilder = getCurrentBufferBuilder(); RecordSerializer.SerializationResult result = recordSerializer.flushToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { tryFinishCurrentBufferBuilder(); if (result.isFullRecord()) { break; } bufferBuilder = getCurrentBufferBuilder(); result = recordSerializer.flushToBufferBuilder(bufferBuilder); } }