@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<TblColRef> columns = cuboid.getColumns(); String[] values = new String[columns.size()]; for (int i = 0; i < columns.size(); i++) { values[i] = valueMap.get(columns.get(i)); } return encode(values); }
public static CoprocessorProjector makeForObserver(final CubeSegment cubeSegment, final Cuboid cuboid, final Collection<TblColRef> dimensionColumns) { RowKeyEncoder rowKeyMaskEncoder = new RowKeyEncoder(cubeSegment, cuboid) { @Override public void fillHeader(byte[] bytes) { Arrays.fill(bytes, 0, this.getHeaderLength(), (byte) 0xff); } @Override protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) { byte bits = dimensionColumns.contains(column) ? (byte) 0xff : 0x00; Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, bits); } }; byte[] mask = rowKeyMaskEncoder.encode(new String[cuboid.getColumns().size()]); return new CoprocessorProjector(mask, dimensionColumns.size() != 0); }
/** * translate GTRecord format fuzzy keys to hbase expected format * @return */ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf(); fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey); fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask); ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask)); } return ret; }
private int buildKey(Cuboid cuboid, ByteArray[] splitBuffers) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int endIdx = startIdx + Long.bitCount(cuboid.getId()); int offset = 0; for (int i = startIdx; i < endIdx; i++) { System.arraycopy(splitBuffers[i].array(), splitBuffers[i].offset(), newKeyBodyBuf, offset, splitBuffers[i].length()); offset += splitBuffers[i].length(); } int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); } newKeyBuf.setLength(fullKeySize); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf); return fullKeySize; }
rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, bufOffset), newKeyBuf);
@Test public void testEncodeWithoutSlr() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); // CubeSegment seg = cube.getTheOnlySegment(); CubeDesc cubeDesc = cube.getDescriptor(); // String data = // "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular"; String[] data = new String[8]; data[0] = "2012-12-15"; data[1] = "11848"; data[2] = "Health & Beauty"; data[3] = "Fragrances"; data[4] = "Women"; data[5] = "FP-GTC"; data[6] = "0"; data[7] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength()); byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length); assertEquals(255, Bytes.toLong(cuboidId)); assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest); }
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length); byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<byte[]> valueList = new ArrayList<byte[]>(); for (TblColRef bdCol : cuboid.getColumns()) { String value = valueMap.get(bdCol); valueList.add(valueStringToBytes(value)); } byte[][] values = valueList.toArray(RowConstants.BYTE_ARR_MARKER); return encode(values); }
@Test public void testEncodeAndDecodeWithUtf8() throws IOException { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY"); CubeDesc cubeDesc = cube.getDescriptor(); String[] data = new String[8]; data[0] = "2012-12-15"; data[1] = "11848"; data[2] = "Health & Beauty"; data[3] = "Fragrances"; data[4] = "Women"; data[5] = "刊登格式测试";// UTF-8 data[6] = "0"; data[7] = "15"; long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId); RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid); byte[] encodedKey = rowKeyEncoder.encode(data); assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length); RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment()); rowKeyDecoder.decode(encodedKey); List<String> values = rowKeyDecoder.getValues(); assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString()); }
public static CoprocessorProjector makeForObserver(final CubeSegment cubeSegment, final Cuboid cuboid, final Collection<TblColRef> dimensionColumns) { RowKeyEncoder rowKeyMaskEncoder = new RowKeyEncoder(cubeSegment, cuboid) { @Override protected int fillHeader(byte[] bytes, byte[][] values) { Arrays.fill(bytes, 0, this.headerLength, (byte) 0xff); return this.headerLength; } @Override protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) { byte bits = dimensionColumns.contains(column) ? (byte) 0xff : 0x00; Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, bits); } }; byte[] mask = rowKeyMaskEncoder.encode(new byte[cuboid.getColumns().size()][]); return new CoprocessorProjector(mask); }
@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<TblColRef> columns = cuboid.getColumns(); String[] values = new String[columns.size()]; for (int i = 0; i < columns.size(); i++) { values[i] = valueMap.get(columns.get(i)); } return encode(values); }
@Override public byte[] encode(Map<TblColRef, String> valueMap) { List<byte[]> valueList = new ArrayList<byte[]>(); for (TblColRef bdCol : cuboid.getColumns()) { String value = valueMap.get(bdCol); valueList.add(valueStringToBytes(value)); } byte[][] values = valueList.toArray(RowConstants.BYTE_ARR_MARKER); return encode(values); }
public static CoprocessorProjector makeForObserver(final CubeSegment cubeSegment, final Cuboid cuboid, final Collection<TblColRef> dimensionColumns) { RowKeyEncoder rowKeyMaskEncoder = new RowKeyEncoder(cubeSegment, cuboid) { @Override public void fillHeader(byte[] bytes) { Arrays.fill(bytes, 0, this.getHeaderLength(), (byte) 0xff); } @Override protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) { byte bits = dimensionColumns.contains(column) ? (byte) 0xff : 0x00; Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, bits); } }; byte[] mask = rowKeyMaskEncoder.encode(new String[cuboid.getColumns().size()]); return new CoprocessorProjector(mask, dimensionColumns.size() != 0); }
/** * translate GTRecord format fuzzy keys to hbase expected format * @return */ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf(); fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey); fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask); ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask)); } return ret; }
private int buildKey(Cuboid cuboid, ByteArray[] splitBuffers) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int endIdx = startIdx + Long.bitCount(cuboid.getId()); int offset = 0; for (int i = startIdx; i < endIdx; i++) { System.arraycopy(splitBuffers[i].array(), splitBuffers[i].offset(), newKeyBodyBuf, offset, splitBuffers[i].length()); offset += splitBuffers[i].length(); } int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); } newKeyBuf.setLength(fullKeySize); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf); return fullKeySize; }
rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, bufOffset), newKeyBuf);