@Override protected OffHeapColumnVector reserveNewColumn(int capacity, DataType type) { return new OffHeapColumnVector(capacity, type); } }
public OffHeapColumnVector(int capacity, DataType type) { super(capacity, type); nulls = 0; data = 0; lengthData = 0; offsetData = 0; reserveInternal(capacity); reset(); }
/** * Allocates columns to store elements of each field of the schema off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
@Override public int putByteArray(int rowId, byte[] value, int offset, int length) { int result = arrayData().appendBytes(length, value, offset); Platform.putInt(null, lengthData + 4 * rowId, length); Platform.putInt(null, offsetData + 4 * rowId, result); return result; }
@Override public void putNotNulls(int rowId, int count) { if (!hasNull()) return; long offset = nulls + rowId; for (int i = 0; i < count; ++i, ++offset) { Platform.putByte(null, offset, (byte) 0); } }
@Override protected void reserveInternal(int newCapacity) { int oldCapacity = (nulls == 0L) ? 0 : capacity; if (isArray() || type instanceof MapType) { this.lengthData = Platform.reallocateMemory(lengthData, oldCapacity * 4L, newCapacity * 4L); this.offsetData = Platform.reallocateMemory(offsetData, oldCapacity * 4L, newCapacity * 4L); } else if (type instanceof ByteType || type instanceof BooleanType) { this.data = Platform.reallocateMemory(data, oldCapacity, newCapacity); } else if (type instanceof ShortType) { this.data = Platform.reallocateMemory(data, oldCapacity * 2L, newCapacity * 2L); } else if (type instanceof IntegerType || type instanceof FloatType || type instanceof DateType || DecimalType.is32BitDecimalType(type)) { this.data = Platform.reallocateMemory(data, oldCapacity * 4L, newCapacity * 4L); } else if (type instanceof LongType || type instanceof DoubleType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { this.data = Platform.reallocateMemory(data, oldCapacity * 8L, newCapacity * 8L); } else if (childColumns != null) { // Nothing to store. } else { throw new RuntimeException("Unhandled " + type); } this.nulls = Platform.reallocateMemory(nulls, oldCapacity, newCapacity); Platform.setMemory(nulls + oldCapacity, (byte)0, newCapacity - oldCapacity); capacity = newCapacity; }
@Override public int putByteArray(int rowId, byte[] value, int offset, int length) { int result = arrayData().appendBytes(length, value, offset); Platform.putInt(null, lengthData + 4L * rowId, length); Platform.putInt(null, offsetData + 4L * rowId, result); return result; }
@Override public void putNotNulls(int rowId, int count) { if (!hasNull()) return; long offset = nulls + rowId; for (int i = 0; i < count; ++i, ++offset) { Platform.putByte(null, offset, (byte) 0); } }
@Override protected void reserveInternal(int newCapacity) { int oldCapacity = (nulls == 0L) ? 0 : capacity; if (isArray() || type instanceof MapType) { this.lengthData = Platform.reallocateMemory(lengthData, oldCapacity * 4L, newCapacity * 4L); this.offsetData = Platform.reallocateMemory(offsetData, oldCapacity * 4L, newCapacity * 4L); } else if (type instanceof ByteType || type instanceof BooleanType) { this.data = Platform.reallocateMemory(data, oldCapacity, newCapacity); } else if (type instanceof ShortType) { this.data = Platform.reallocateMemory(data, oldCapacity * 2L, newCapacity * 2L); } else if (type instanceof IntegerType || type instanceof FloatType || type instanceof DateType || DecimalType.is32BitDecimalType(type)) { this.data = Platform.reallocateMemory(data, oldCapacity * 4L, newCapacity * 4L); } else if (type instanceof LongType || type instanceof DoubleType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { this.data = Platform.reallocateMemory(data, oldCapacity * 8L, newCapacity * 8L); } else if (childColumns != null) { // Nothing to store. } else { throw new RuntimeException("Unhandled " + type); } this.nulls = Platform.reallocateMemory(nulls, oldCapacity, newCapacity); Platform.setMemory(nulls + oldCapacity, (byte)0, newCapacity - oldCapacity); capacity = newCapacity; }
public OffHeapColumnVector(int capacity, DataType type) { super(capacity, type); nulls = 0; data = 0; lengthData = 0; offsetData = 0; reserveInternal(capacity); reset(); }
@Override protected OffHeapColumnVector reserveNewColumn(int capacity, DataType type) { return new OffHeapColumnVector(capacity, type); } }
@Override public int putByteArray(int rowId, byte[] value, int offset, int length) { int result = arrayData().appendBytes(length, value, offset); Platform.putInt(null, lengthData + 4L * rowId, length); Platform.putInt(null, offsetData + 4L * rowId, result); return result; }
/** * Allocates columns to store elements of each field of the schema off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
protected OffHeapColumnVector(int capacity, DataType type) { super(capacity, type, MemoryMode.OFF_HEAP); nulls = 0; data = 0; lengthData = 0; offsetData = 0; reserveInternal(capacity); reset(); }
/** * Allocates a column to store elements of `type` on or off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static ColumnVector allocate(int capacity, DataType type, MemoryMode mode) { if (mode == MemoryMode.OFF_HEAP) { return new OffHeapColumnVector(capacity, type); } else { return new OnHeapColumnVector(capacity, type); } }
@Override public int putByteArray(int rowId, byte[] value, int offset, int length) { int result = arrayData().appendBytes(length, value, offset); Platform.putInt(null, lengthData + 4 * rowId, length); Platform.putInt(null, offsetData + 4 * rowId, result); return result; }
/** * Converts an iterator of rows into a single ColumnBatch. */ public static ColumnarBatch toBatch( StructType schema, MemoryMode memMode, Iterator<Row> row) { int capacity = 4 * 1024; WritableColumnVector[] columnVectors; if (memMode == MemoryMode.OFF_HEAP) { columnVectors = OffHeapColumnVector.allocateColumns(capacity, schema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, schema); } int n = 0; while (row.hasNext()) { Row r = row.next(); for (int i = 0; i < schema.fields().length; i++) { appendValue(columnVectors[i], schema.fields()[i].dataType(), r, i); } n++; } ColumnarBatch batch = new ColumnarBatch(columnVectors); batch.setNumRows(n); return batch; } }
protected OffHeapColumnVector(int capacity, DataType type) { super(capacity, type, MemoryMode.OFF_HEAP); nulls = 0; data = 0; lengthData = 0; offsetData = 0; reserveInternal(capacity); reset(); }
/** * Allocates columns to store elements of each field off heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OffHeapColumnVector[] allocateColumns(int capacity, StructField[] fields) { OffHeapColumnVector[] vectors = new OffHeapColumnVector[fields.length]; for (int i = 0; i < fields.length; i++) { vectors[i] = new OffHeapColumnVector(capacity, fields[i].dataType()); } return vectors; }
/** * Converts an iterator of rows into a single ColumnBatch. */ public static ColumnarBatch toBatch( StructType schema, MemoryMode memMode, Iterator<Row> row) { int capacity = 4 * 1024; WritableColumnVector[] columnVectors; if (memMode == MemoryMode.OFF_HEAP) { columnVectors = OffHeapColumnVector.allocateColumns(capacity, schema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, schema); } int n = 0; while (row.hasNext()) { Row r = row.next(); for (int i = 0; i < schema.fields().length; i++) { appendValue(columnVectors[i], schema.fields()[i].dataType(), r, i); } n++; } ColumnarBatch batch = new ColumnarBatch(columnVectors); batch.setNumRows(n); return batch; } }