@Override public Iterable<ArrowBuf> apply(VectorWrapper<?> vectorWrapper) { return Arrays.asList(vectorWrapper.getValueVector().getBuffers(true)); } }).toList().toArray(new ArrowBuf[0]);
public static boolean checkBufRefs(final ValueVector vv) { for (final ArrowBuf buffer : vv.getBuffers(false)) { if (buffer.refCnt() <= 0) { throw new IllegalStateException("zero refcount"); } } return true; }
@Override public ArrowBuf[] getBuffers(boolean clear) { final List<ArrowBuf> buffers = new ArrayList<>(); for (final ValueVector vector : vectors.values()) { for (final ArrowBuf buf : vector.getBuffers(false)) { buffers.add(buf); if (clear) { buf.retain(1); } } if (clear) { vector.clear(); } } return buffers.toArray(new ArrowBuf[buffers.size()]); }
@Override public int getBufferSize() { int actualBufSize = 0; for (final ValueVector v : vectors.values()) { for (final ArrowBuf buf : v.getBuffers(false)) { actualBufSize += buf.writerIndex(); } } return actualBufSize; } }
public long getSize(ValueVector v, SizeType type){ long size = 0; ArrowBuf[] buffers = v.getBuffers(false); for(ArrowBuf b : buffers){ switch(type){ case ACCOUNTED: size += b.getActualMemoryConsumed(); break; case WORSE_CASE: UnsafeDirectLittleEndian udle = (UnsafeDirectLittleEndian) b.unwrap(); // avoid counting the same underlying buffer multiple times if (accountedBuffers.containsKey(udle)) { continue; } size += b.getPossibleMemoryConsumed(); accountedBuffers.put(udle, null); break; default: throw new UnsupportedOperationException("Invalid case: " + type.name()); } } return size; } }
public static WritableBatch getBatchNoHV(int recordCount, Iterable<ValueVector> vectors, boolean isSV2) { List<ArrowBuf> buffers = Lists.newArrayList(); List<SerializedField> metadata = Lists.newArrayList(); for (ValueVector vv : vectors) { metadata.add(TypeHelper.getMetadata(vv)); // don't try to get the buffers if we don't have any records. It is possible the buffers are dead buffers. if (recordCount == 0) { vv.clear(); continue; } for (ArrowBuf b : vv.getBuffers(true)) { buffers.add(b); } // remove vv access to buffers. vv.clear(); } RecordBatchDef batchDef = RecordBatchDef.newBuilder().addAllField(metadata).setRecordCount(recordCount) .setCarriesTwoByteSelectionVector(isSV2).build(); WritableBatch b = new WritableBatch(batchDef, buffers); return b; }