@Override public void clear() { vector.clear(); }
@Override public int next() { final Stopwatch watch = Stopwatch.createStarted(); if (keyVector != null) { keyVector.clear(); keyVector.allocateNew(); } if (valueVector != null) { valueVector.clear(); valueVector.allocateNew(); } int recordCount = 0; int batchSize = 0; try { while (recordCount < numRowsPerBatch && batchSize < numBytesPerBatch && reader.next(key, value)) { keyVector.setSafe(recordCount, key.getBytes(), 0, key.getLength()); valueVector.setSafe(recordCount, value.getBytes(), 0, value.getLength()); batchSize += (key.getLength() + value.getLength()); ++recordCount; } keyVector.setValueCount(recordCount); valueVector.setValueCount(recordCount); logger.debug("Read {} records in {} ms", recordCount, watch.elapsed(TimeUnit.MILLISECONDS)); return recordCount; } catch (IOException ioe) { close(); throw UserException.dataReadError(ioe).addContext("File Path", split.getPath().toString()).build(logger); } }
public void load(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int bitsLength = bitsField.getBufferLength(); final int capacity = buffer.capacity(); final int valuesLength = capacity - bitsLength; /* load inner validity buffer */ loadValidityBuffer(bitsField, buffer); /* load inner offset and value buffers */ loadOffsetAndDataBuffer(valuesField, buffer.slice(bitsLength, valuesLength)); vector.setLastSet(metadata.getValueCount() - 1); vector.valueCount = metadata.getValueCount(); }
Stopwatch watch = Stopwatch.createStarted(); if (rowKeyVector != null) { rowKeyVector.clear(); rowKeyVector.allocateNew();
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(valuesLength, metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner offset and value buffers */ vector.offsetBuffer.close(); vector.valueBuffer.close(); loadOffsetAndDataBuffer(valuesField, buffer.slice(0, valuesLength)); vector.setLastSet(metadata.getValueCount() - 1); }