private byte[] getByteArrayForShort(short v) { byte[] split = new byte[Bytes.SIZEOF_SHORT]; BytesUtil.writeUnsigned(v, split, 0, Bytes.SIZEOF_SHORT); return split; }
private static byte[][] getSplitsByRegionCount(int regionCount) { byte[][] result = new byte[regionCount - 1][]; for (int i = 1; i < regionCount; ++i) { byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN]; BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN); result[i - 1] = split; } return result; }
private byte[][] getSplitsByShardNum(int shardNum) { byte[][] result = new byte[shardNum - 1][]; for (int i = 1; i < shardNum; ++i) { byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN]; BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN); result[i - 1] = split; } return result; }
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) { int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset); trieBytes[parentOffset] |= flags; }
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) { int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset); trieBytes[parentOffset] |= flags; }
@Override public void serialize(Integer value, ByteBuffer out) { BytesUtil.writeUnsigned(value, 4, out); }
private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) { int o = offset; if (o > _2GB) throw new IllegalStateException(); // childOffset if (isLastChild) trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD; if (n.isEndOfValue) trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE; o += sizeChildOffset; // nValuesBeneath BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath); o += sizeNoValuesBeneath; // nValueBytes if (n.part.length > 255) throw new RuntimeException(); BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1); o++; // valueBytes System.arraycopy(n.part, 0, trieBytes, o, n.part.length); o += n.part.length; return o; }
@Override public void encode(String valueStr, byte[] output, int outputOffset) { try { int id = dict.getIdFromValue(valueStr, roundingFlag); BytesUtil.writeUnsigned(id, output, outputOffset, fixedLen); } catch (IllegalArgumentException ex) { for (int i = outputOffset; i < outputOffset + fixedLen; i++) { output[i] = defaultByte; } logger.error("Can't translate value " + valueStr + " to dictionary ID, roundingFlag " + roundingFlag + "." + " Using default value " + String.format(Locale.ROOT, "\\x%02X", defaultByte)); } }
@Override public void serialize(Object value, ByteBuffer buf) { int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag); BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf); }
private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId, byte[] trieBytes) { int o = offset; // childOffset if (isLastChild) trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD; if (n.isEndOfValue) trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE; o += sizeChildOffset; // nValueBytes if (n.part.length > 255) throw new RuntimeException( "Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part)); BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1); o++; // valueBytes System.arraycopy(n.part, 0, trieBytes, o, n.part.length); o += n.part.length; if (n.isEndOfValue) { checkValidId(n.id); BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId); o += sizeId; } return o; }
headOut.close(); head = byteBuf.toByteArray(); BytesUtil.writeUnsigned(head.length, head, AppendTrieDictionary.HEAD_SIZE_I, 2); } catch (IOException e) { throw new RuntimeException(e); // shall not happen, as we are
headOut.close(); head = byteBuf.toByteArray(); BytesUtil.writeUnsigned(head.length, head, TrieDictionary.MAGIC_SIZE_I, 2); } catch (IOException e) { throw new RuntimeException(e); // shall not happen, as we are writing in memory
@Override public List<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length != 1) throw new IllegalArgumentException(); //source input column value String literal = values[0]; // encode literal using dictionary TblColRef literalCol = getRawColumn(measureDesc.getFunction()); Dictionary<String> dictionary = dictionaryMap.get(literalCol); int keyEncodedValue = dictionary.getIdFromValue(literal); ByteArray key = new ByteArray(dictionary.getSizeOfId()); BytesUtil.writeUnsigned(keyEncodedValue, key.array(), key.offset(), dictionary.getSizeOfId()); List<ByteArray> valueList = new ArrayList<ByteArray>(1); valueList.add(key); return valueList; }
BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN); splits.add(split); logger.info("Region {0}'s hfile {1} size is {2} mb", String.valueOf(i), String.valueOf(j), String.valueOf(accumulatedSize)); byte[] split = new byte[RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN]; BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(cuboid), 0, split, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN);
private List<ByteArray> getValueList(int size) { if (size == -1) { return null; } List<ByteArray> valueList = new ArrayList<ByteArray>(size); for (Integer i = 0; i < size; i++) { ByteArray key = new ByteArray(1); BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); valueList.add(key); } return valueList; }
public void testWriteReadUnsignedInt(int testInt, int length) { ByteArray ba = new ByteArray(new byte[length]); BytesUtil.writeUnsigned(testInt, length, ba.asBuffer()); byte[] newBytes = new byte[length]; System.arraycopy(ba.array(), 0, newBytes, 0, length); int value = BytesUtil.readUnsigned(new ByteArray(newBytes).asBuffer(), length); assertEquals(value, testInt); byte[] anOtherNewBytes = new byte[length]; BytesUtil.writeUnsigned(testInt, anOtherNewBytes, 0, length); assertTrue(Arrays.equals(anOtherNewBytes, ba.array())); }
@Override public List<ByteArray> reEncodeDictionary(List<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { TblColRef colRef = getRawColumn(measureDesc.getFunction()); Dictionary<String> sourceDict = oldDicts.get(colRef); Dictionary<String> mergedDict = newDicts.get(colRef); int valueSize = value.size(); byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()]; int bufOffset = 0; for (ByteArray c : value) { int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length()); int newId; String v = sourceDict.getValueFromId(oldId); if (v == null) { newId = mergedDict.nullId(); } else { newId = mergedDict.getIdFromValue(v); } BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId()); c.reset(newIdBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); } return value; } };
@Test public void testNormal() { int size = 100; List<ByteArray> valueList = new ArrayList<ByteArray>(size); for (Integer i = 0; i < size; i++) { ByteArray key = new ByteArray(1); BytesUtil.writeUnsigned(i, key.array(), 0, key.length()); valueList.add(key); } agg.aggregate(valueList); agg.aggregate(valueList); assertEquals(valueList.size() * 2, agg.getState().size()); }
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) { int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE); BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset); trieBytes[parentOffset] |= flags; }
BytesUtil.writeUnsigned(mergedDict.nullId(), newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); continue; BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); } else {