final RecordSerializer<SerializationDelegate<Object>> recordSerializer = new SpanningRecordSerializer<SerializationDelegate<Object>>(); final SerializationDelegate<Object> delegate = (SerializationDelegate<Object>) (SerializationDelegate<?>) new SerializationDelegate<StreamElement>(new StreamElementSerializer<T>(serializer));
@Override public SerializationResult flushToBufferBuilder(BufferBuilder targetBuffer) { return copyToBufferBuilder(targetBuffer); } }
@Override public SerializationResult setNextBuffer(Buffer buffer) throws IOException { this.targetBuffer = buffer; this.position = 0; this.limit = buffer.getSize(); if (this.lengthBuffer.hasRemaining()) { copyToTargetBufferFrom(this.lengthBuffer); } if (this.dataBuffer.hasRemaining()) { copyToTargetBufferFrom(this.dataBuffer); } SerializationResult result = getSerializationResult(); // make sure we don't hold onto the large buffers for too long if (result.isFullRecord()) { this.serializationBuffer.clear(); this.serializationBuffer.pruneBuffer(); this.dataBuffer = this.serializationBuffer.wrapAsByteBuffer(); } return result; }
/** * Copies the intermediate data serialization buffer to target BufferBuilder. * * @param targetBuffer the target BufferBuilder to copy to * @return how much information was written to the target buffer and * whether this buffer is full */ @Override public SerializationResult copyToBufferBuilder(BufferBuilder targetBuffer) { targetBuffer.append(dataBuffer); targetBuffer.commit(); return getSerializationResult(targetBuffer); }
/** * Serializes the complete record to an intermediate data serialization * buffer and starts copying it to the target buffer (if available). * * @param record the record to serialize * @return how much information was written to the target buffer and * whether this buffer is full * @throws IOException */ @Override public SerializationResult addRecord(T record) throws IOException { if (CHECKED) { if (this.dataBuffer.hasRemaining()) { throw new IllegalStateException("Pending serialization of previous record."); } } this.serializationBuffer.clear(); this.lengthBuffer.clear(); // write data and length record.write(this.serializationBuffer); int len = this.serializationBuffer.length(); this.lengthBuffer.putInt(0, len); this.dataBuffer = this.serializationBuffer.wrapAsByteBuffer(); // Copy from intermediate buffers to current target memory segment copyToTargetBufferFrom(this.lengthBuffer); copyToTargetBufferFrom(this.dataBuffer); return getSerializationResult(); }
/** * Copies an intermediate data serialization buffer into the target BufferBuilder. * * @param targetBuffer the target BufferBuilder to copy to * @return how much information was written to the target buffer and * whether this buffer is full */ @Override public SerializationResult copyToBufferBuilder(BufferBuilder targetBuffer) { targetBuffer.append(lengthBuffer); targetBuffer.append(dataBuffer); targetBuffer.commit(); return getSerializationResult(targetBuffer); }
public RecordSerializer<IOReadableWritable> getRecordSerializer() { if (!useCompression) { return new SpanningRecordSerializer<IOReadableWritable>(); } else { SpanningRecordSerializer<BufferSerializationDelegate> internalSer = new SpanningRecordSerializer<BufferSerializationDelegate>(); BufferSerializationDelegate internalSerDelegate = new CompressionBufferTransformer(compressionFactory.get()); return new CompositeSpanningRecordSerializer<IOReadableWritable>(internalSer, internalSerDelegate, compressionBufferSize); } }
/** * Copies an intermediate data serialization buffer into the target BufferBuilder. * * @param targetBuffer the target BufferBuilder to copy to * @return how much information was written to the target buffer and * whether this buffer is full */ @Override public SerializationResult copyToBufferBuilder(BufferBuilder targetBuffer) { targetBuffer.append(lengthBuffer); targetBuffer.append(dataBuffer); targetBuffer.commit(); return getSerializationResult(targetBuffer); }
public RecordWriter(ResultPartitionWriter writer, ChannelSelector<T> channelSelector, boolean flushAlways) { this.flushAlways = flushAlways; this.targetPartition = writer; this.channelSelector = channelSelector; this.numChannels = writer.getNumberOfSubpartitions(); this.serializer = new SpanningRecordSerializer<T>(); this.bufferBuilders = new Optional[numChannels]; this.broadcastChannels = new int[numChannels]; for (int i = 0; i < numChannels; i++) { broadcastChannels[i] = i; bufferBuilders[i] = Optional.empty(); } }
public RecordWriter(ResultPartitionWriter writer, ChannelSelector<T> channelSelector, boolean flushAlways) { this.flushAlways = flushAlways; this.targetPartition = writer; this.channelSelector = channelSelector; this.numChannels = writer.getNumberOfSubpartitions(); this.serializer = new SpanningRecordSerializer<T>(); this.bufferBuilders = new Optional[numChannels]; this.broadcastChannels = new int[numChannels]; for (int i = 0; i < numChannels; i++) { broadcastChannels[i] = i; bufferBuilders[i] = Optional.empty(); } }
@SuppressWarnings("unchecked") public RecordWriter(ResultPartitionWriter writer, ChannelSelector<T> channelSelector) { this.targetPartition = writer; this.channelSelector = channelSelector; this.numChannels = writer.getNumberOfOutputChannels(); /** * The runtime exposes a channel abstraction for the produced results * (see {@link ChannelSelector}). Every channel has an independent * serializer. */ this.serializers = new SpanningRecordSerializer[numChannels]; for (int i = 0; i < numChannels; i++) { serializers[i] = new SpanningRecordSerializer<T>(); } }
this.partitionConsumableNotifier = checkNotNull(partitionConsumableNotifier); this.sendScheduleOrUpdateConsumersMessage = sendScheduleOrUpdateConsumersMessage; this.serializer = new SpanningRecordSerializer<>();
this.memory = memory; this.recordSerializer = new SpanningRecordSerializer<>(); this.serializationDelegate = new SerializationDelegate<>(serializer);