public static ByteArray copyOf(byte[] array, int offset, int length) { byte[] space = new byte[length]; System.arraycopy(array, offset, space, 0, length); return new ByteArray(space, 0, length); }
@Override protected ByteArray getRecordFromKeyValue(Text key, Text value) { keyValueBuffer.clear(); keyValueBuffer.put(key.getBytes(), keyOffset, key.getBytes().length - keyOffset); keyValueBuffer.put(value.getBytes()); byte[] keyValue = new byte[keyValueBuffer.position()]; System.arraycopy(keyValueBuffer.array(), 0, keyValue, 0, keyValueBuffer.position()); return new ByteArray(keyValue); } }
@Override public ByteArray deserialize(ByteBuffer in) { //the array in ByteArray is garanteed to be completed owned by the ByteArray return new ByteArray(BytesUtil.readByteArray(in)); } }
private ByteArray[] getNewColHashValues(int rowLength) { ByteArray[] colHashValues = new ByteArray[rowLength]; for (int i = 0; i < rowLength; i++) { colHashValues[i] = new ByteArray(); } return colHashValues; }
public GTRecord(GTInfo info) { this.cols = new ByteArray[info.getColumnCount()]; for (int i = 0; i < this.cols.length; i++) { // consider column projection by pass in another bit set this.cols[i] = new ByteArray(); } this.info = info; }
@Override public ByteArray deserialize(ByteBuffer buffer) { return new ByteArray(BytesUtil.readByteArray(buffer)); } };
@Override public int compare(T o1, T o2) { //return BytesUtil.safeCompareBytes(converter.convertToBytes(o1),converter.convertToBytes(o2)); byte[] b1 = converter.convertToBytes(o1); byte[] b2 = converter.convertToBytes(o2); ByteArray ba1 = new ByteArray(b1, 0, b1.length); ByteArray ba2 = new ByteArray(b2, 0, b2.length); return ba1.compareTo(ba2); } }
public Generator(long nRows) { this.nRows = nRows; this.rand = new Random(); rec = new GTRecord(info); dist = new Distribution[colSpecs.size()]; for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); rec.set(i, new ByteArray(spec.length)); dist[i] = new Distribution(spec, 0); } }
/** set record to the codes of specified values, new space allocated to hold the codes */ public GTRecord setValues(Object... values) { setValues(info.colAll, new ByteArray(info.getMaxRecordLength()), values); return this; }
public void setValue(int i, Object value) { ByteArray space = new ByteArray(info.codeSystem.maxCodeLength(i)); ByteBuffer buf = space.asBuffer(); info.codeSystem.encodeColumnValue(i, value, buf); set(i, space); cols[i].reset(buf.array(), buf.arrayOffset(), buf.position()); }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); encodeDims(record, keyColumns, byteArray, defaultValue()); //fill shard and cuboid fillHeader(buf); }
private List<ByteArray> getValueList(int size) { if (size == -1) { return null; } List<ByteArray> valueList = new ArrayList<ByteArray>(size); for (Integer i = 0; i < size; i++) { ByteArray key = new ByteArray(1); BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); valueList.add(key); } return valueList; }
public void testWriteReadUnsignedInt(int testInt, int length) { ByteArray ba = new ByteArray(new byte[length]); BytesUtil.writeUnsigned(testInt, length, ba.asBuffer()); byte[] newBytes = new byte[length]; System.arraycopy(ba.array(), 0, newBytes, 0, length); int value = BytesUtil.readUnsigned(new ByteArray(newBytes).asBuffer(), length); assertEquals(value, testInt); byte[] anOtherNewBytes = new byte[length]; BytesUtil.writeUnsigned(testInt, anOtherNewBytes, 0, length); assertTrue(Arrays.equals(anOtherNewBytes, ba.array())); }
@Test public void testOverflow() { String text = StringUtils.repeat("h", 21); ExtendedColumnSerializer serializer = new ExtendedColumnSerializer(DataType.getType("extendedcolumn(20)")); MeasureIngester<ByteArray> ingester = measureType.newIngester(); ByteArray array = ingester.valueOf(new String[] { null, text }, null, null); ByteBuffer buffer = ByteBuffer.allocate(serializer.maxLength()); serializer.serialize(array, buffer); buffer.flip(); ByteArray des = serializer.deserialize(buffer); Assert.assertTrue(new ByteArray(StringUtils.repeat("h", 20).getBytes(StandardCharsets.UTF_8)).equals(des)); } }
@Test public void testNormal() { String text = StringUtils.repeat("h", 20); ExtendedColumnSerializer serializer = new ExtendedColumnSerializer(DataType.getType("extendedcolumn(20)")); MeasureIngester<ByteArray> ingester = measureType.newIngester(); ByteArray array = ingester.valueOf(new String[] { null, text }, null, null); ByteBuffer buffer = ByteBuffer.allocate(serializer.maxLength()); serializer.serialize(array, buffer); buffer.flip(); ByteArray des = serializer.deserialize(buffer); Assert.assertTrue(new ByteArray(text.getBytes(StandardCharsets.UTF_8)).equals(des)); }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }
public static CompareTupleFilter getCompareTupleFilter(int col, Object value) { TblColRef colRef = gtInfo.colRef(col); ColumnTupleFilter colFilter = new ColumnTupleFilter(colRef); ByteArray space = new ByteArray(gtInfo.getCodeSystem().maxCodeLength(col)); gtInfo.getCodeSystem().encodeColumnValue(col, value, space.asBuffer()); ConstantTupleFilter constFilter = new ConstantTupleFilter(space); CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ); compareFilter.addChild(colFilter); compareFilter.addChild(constFilter); return compareFilter; }