@Override public int getMemBytesEstimate() { int bytes = 0; if (list != null) { for (ByteArray array : list) { bytes += array.length() + 1; } } return bytes; }
@Override public void encode(ByteArray bodyBytes, ByteArray outputBuf) { Preconditions.checkState(bodyBytes.length() == bodyLength); Preconditions.checkState(bodyBytes.length() + getHeaderLength() == outputBuf.length(), // "bodybytes length: " + bodyBytes.length() + " outputBuf length: " + outputBuf.length() + " header length: " + getHeaderLength()); System.arraycopy(bodyBytes.array(), bodyBytes.offset(), outputBuf.array(), getHeaderLength(), bodyLength); //fill shard and cuboid fillHeader(outputBuf.array()); }
public void exportColumns(int[] fieldIndex, ByteBuffer buf) { for (int i : fieldIndex) { buf.put(cols[i].array(), cols[i].offset(), cols[i].length()); } }
public int sizeOf(ImmutableBitSet selectedCols) { int size = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); size += cols[c].length(); } return size; }
@Override public void serialize(List<ByteArray> values, ByteBuffer out) { if (values == null) { BytesUtil.writeVInt(0, out); } else { BytesUtil.writeVInt(values.size(), out); for (ByteArray array : values) { if (!out.hasRemaining() || out.remaining() < array.length()) { throw new RuntimeException("BufferOverflow! Please use one higher cardinality column for dimension column when build RAW cube!"); } BytesUtil.writeByteArray(BytesUtil.subarray(array.array(), array.offset(), array.offset() + array.length()), out); } } }
/** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); Preconditions.checkNotNull(cols[c].array()); System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length()); pos += cols[c].length(); } buf.setLength(pos); }
public void shallowCopyFrom(GTRecord source) { assert info == source.info; for (int i = 0; i < cols.length; i++) { cols[i].reset(source.cols[i].array(), source.cols[i].offset(), source.cols[i].length()); } }
/** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) { for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); buf.put(cols[c].array(), cols[c].offset(), cols[c].length()); } }
@Override public boolean isNull(ByteArray code) { return DimensionEncoding.isNull(code.array(), code.offset(), code.length()); }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); ByteArray raw = rawIterator.next(); int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length()); String colValue = rawColDict.getValueFromId(key); tuple.setDimensionValue(literalTupleIdx, colValue); } };
private byte[] createKey(GTRecord record) { byte[] result = new byte[keyLength]; int offset = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); final ByteArray byteArray = record.cols[c]; final int columnLength = info.codeSystem.maxCodeLength(c); System.arraycopy(byteArray.array(), byteArray.offset(), result, offset, byteArray.length()); offset += columnLength; } assert offset == result.length; return result; }
@Override public void write(GTRecord r) throws IOException { ByteArray byteArray = r.exportColumns(info.getAllColumns()); assert byteArray.offset() == 0; assert byteArray.array().length == byteArray.length(); rowList.add(byteArray.array()); }
@Override public void serialize(ByteArray code, ByteBuffer buffer) { if (code == null) BytesUtil.writeByteArray(null, 0, 0, buffer); else BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer); }
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes); initCuboid(cuboidId); ByteArray[] splits = rowKeySplitter.getSplitBuffers(); int offset = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].array(), splits[offset].offset(), splits[offset].length()); offset++; } return cuboidId; }
private List<ByteArray> getValueList(int size) { if (size == -1) { return null; } List<ByteArray> valueList = new ArrayList<ByteArray>(size); for (Integer i = 0; i < size; i++) { ByteArray key = new ByteArray(1); BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); valueList.add(key); } return valueList; }
@Override public void serialize(ByteArray value, ByteBuffer out) { if (value != null && value.array() != null) { BytesUtil.writeByteArray(value.array(), value.offset(), value.length(), out); } else { BytesUtil.writeByteArray(null, out); } }
@Test public void testNormal() { int size = 100; List<ByteArray> valueList = new ArrayList<ByteArray>(size); for (Integer i = 0; i < size; i++) { ByteArray key = new ByteArray(1); BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); valueList.add(key); } agg.aggregate(valueList); agg.aggregate(valueList); assertEquals(valueList.size() * 2, agg.getState().size()); }