public static int readUnsigned(ByteArray in, int offset, int size) { int integer = 0; offset += in.offset(); byte[] bytes = in.array(); for (int i = offset, n = offset + size; i < n; i++) { integer <<= 8; integer |= (int) bytes[i] & 0xFF; } return integer; }
public void exportColumns(int[] fieldIndex, ByteBuffer buf) { for (int i : fieldIndex) { buf.put(cols[i].array(), cols[i].offset(), cols[i].length()); } }
private void export(ByteArray array, ByteBuffer buf) { buf.putInt(array.length()); buf.put(array.array(), array.offset(), array.length()); }
private void export(DataOutputStream out, ByteArray array) throws IOException { out.writeInt(array.length()); out.write(array.array(), array.offset(), array.length()); }
@Override public void serialize(List<ByteArray> values, ByteBuffer out) { if (values == null) { BytesUtil.writeVInt(0, out); } else { BytesUtil.writeVInt(values.size(), out); for (ByteArray array : values) { if (!out.hasRemaining() || out.remaining() < array.length()) { throw new RuntimeException("BufferOverflow! Please use one higher cardinality column for dimension column when build RAW cube!"); } BytesUtil.writeByteArray(BytesUtil.subarray(array.array(), array.offset(), array.offset() + array.length()), out); } } }
/** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); Preconditions.checkNotNull(cols[c].array()); System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length()); pos += cols[c].length(); } buf.setLength(pos); }
public void shallowCopyFrom(GTRecord source) { assert info == source.info; for (int i = 0; i < cols.length; i++) { cols[i].reset(source.cols[i].array(), source.cols[i].offset(), source.cols[i].length()); } }
/** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteBuffer buf) { for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); buf.put(cols[c].array(), cols[c].offset(), cols[c].length()); } }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); ByteArray raw = rawIterator.next(); int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length()); String colValue = rawColDict.getValueFromId(key); tuple.setDimensionValue(literalTupleIdx, colValue); } };
@Override public boolean isNull(ByteArray code) { return DimensionEncoding.isNull(code.array(), code.offset(), code.length()); }
public byte[] toBytes() { return Bytes.copy(this.array(), this.offset(), this.length()); }
@Override public void write(GTRecord r) throws IOException { ByteArray byteArray = r.exportColumns(info.getAllColumns()); assert byteArray.offset() == 0; assert byteArray.array().length == byteArray.length(); rowList.add(byteArray.array()); }
private byte[] createKey(GTRecord record) { byte[] result = new byte[keyLength]; int offset = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); final ByteArray byteArray = record.cols[c]; final int columnLength = info.codeSystem.maxCodeLength(c); System.arraycopy(byteArray.array(), byteArray.offset(), result, offset, byteArray.length()); offset += columnLength; } assert offset == result.length; return result; }
@Override public void serialize(ByteArray code, ByteBuffer buffer) { if (code == null) BytesUtil.writeByteArray(null, 0, 0, buffer); else BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer); }
public long decode(byte[] bytes) throws IOException { this.values.clear(); long cuboidId = rowKeySplitter.split(bytes); initCuboid(cuboidId); ByteArray[] splits = rowKeySplitter.getSplitBuffers(); int offset = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboid id part for (int i = 0; i < this.cuboid.getColumns().size(); i++) { TblColRef col = this.cuboid.getColumns().get(i); collectValue(col, splits[offset].array(), splits[offset].offset(), splits[offset].length()); offset++; } return cuboidId; }
@Override public void serialize(ByteArray value, ByteBuffer out) { if (value != null && value.array() != null) { BytesUtil.writeByteArray(value.array(), value.offset(), value.length(), out); } else { BytesUtil.writeByteArray(null, out); } }
@Override public void encode(ByteArray bodyBytes, ByteArray outputBuf) { Preconditions.checkState(bodyBytes.length() == bodyLength); Preconditions.checkState(bodyBytes.length() + getHeaderLength() == outputBuf.length(), // "bodybytes length: " + bodyBytes.length() + " outputBuf length: " + outputBuf.length() + " header length: " + getHeaderLength()); System.arraycopy(bodyBytes.array(), bodyBytes.offset(), outputBuf.array(), getHeaderLength(), bodyLength); //fill shard and cuboid fillHeader(outputBuf.array()); }
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN); r.addChild(col); List<ByteArray> list = Lists.newArrayList(); for (int v : values) { int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); list.add(bytes); } r.addChild(new ConstantTupleFilter(list)); return r; }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }