recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
innerDeserializationResult = innerDeserializer.getNextRecord(target); } else if (innerDeserializationResult == DeserializationResult.LAST_RECORD_FROM_BUFFER) { subDeserializationResult = subDeserializer.getNextRecord(subDeserializationDelegate); if (subDeserializationResult.isFullRecord()) { innerDeserializer.setNextBuffer(subDeserializationDelegate.getBuffer()); innerDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; subDeserializationResult = subDeserializer.getNextRecord(subDeserializationDelegate); if (subDeserializationResult.isFullRecord()) { innerDeserializer.setNextBuffer(subDeserializationDelegate.getBuffer()); if (!innerDeserializationResult.isFullRecord()) { innerDeserializationResult = innerDeserializer.getNextRecord(target); } else { innerDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER;
@Override public boolean hasUnfinishedData() { return innerDeserializer.hasUnfinishedData() || subDeserializer.hasUnfinishedData(); }
@Override public void clear() { this.innerDeserializer.clear(); this.innerDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; this.subDeserializer.clear(); this.subDeserializationDelegate.clear(); this.subDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; }
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
public CompositeSpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories, RecordDeserializer<BufferDeserializationDelegate> subDeserializer, BufferDeserializationDelegate subDeserializationDelegate) { assert subDeserializer != null; assert subDeserializationDelegate != null; this.innerDeserializer = new SpillingAdaptiveSpanningRecordDeserializer<>(tmpDirectories); this.innerDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; this.subDeserializer = subDeserializer; this.subDeserializationDelegate = subDeserializationDelegate; this.subDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; }
public RecordDeserializer<T> getRecordDeserializer(String[] tmpDirectories) { if (!useCompression) { return new SpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories); } else { RecordDeserializer<BufferDeserializationDelegate> internalDeser = new SpillingAdaptiveSpanningRecordDeserializer<>(tmpDirectories); BufferDeserializationDelegate internalDeserDelegate = new DecompressionBufferTransformer(compressionFactory.get()); return new CompositeSpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories, internalDeser, internalDeserDelegate); } }
/** * Creates a new AbstractRecordReader that de-serializes records from the given input gate and * can spill partial records to disk, if they grow large. * * @param inputGate The input gate to read from. * @param tmpDirectories The temp directories. USed for spilling if the reader concurrently * reconstructs multiple large records. */ @SuppressWarnings("unchecked") protected AbstractRecordReader(InputGate inputGate, String[] tmpDirectories) { super(inputGate); // Initialize one deserializer per input channel this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()]; for (int i = 0; i < recordDeserializers.length; i++) { recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories); } }
/** * Creates a new AbstractRecordReader that de-serializes records from the given input gate and * can spill partial records to disk, if they grow large. * * @param inputGate The input gate to read from. * @param tmpDirectories The temp directories. USed for spilling if the reader concurrently * reconstructs multiple large records. */ @SuppressWarnings("unchecked") protected AbstractRecordReader(InputGate inputGate, String[] tmpDirectories) { super(inputGate); // Initialize one deserializer per input channel this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()]; for (int i = 0; i < recordDeserializers.length; i++) { recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories); } }
/** * Creates a new AbstractRecordReader that de-serializes records from the given input gate and * can spill partial records to disk, if they grow large. * * @param inputGate The input gate to read from. * @param tmpDirectories The temp directories. USed for spilling if the reader concurrently * reconstructs multiple large records. */ @SuppressWarnings("unchecked") protected AbstractRecordReader(InputGate inputGate, String[] tmpDirectories) { super(inputGate); // Initialize one deserializer per input channel this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()]; for (int i = 0; i < recordDeserializers.length; i++) { recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories); } }
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());