private static void writeSimpleGeometry(DynamicSliceOutput output, GeometrySerializationType type, OGCGeometry geometry) { output.appendByte(type.code()); Geometry esriGeometry = requireNonNull(geometry.getEsriGeometry(), "esriGeometry is null"); byte[] shape = geometryToEsriShape(esriGeometry); output.appendBytes(shape); }
@Override public byte[] getSerializedData() { byte[] classifierBytes = serialize(classifier).getBytes(); DynamicSliceOutput output = new DynamicSliceOutput(classifierBytes.length + 64 * labelEnumeration.size()); output.appendInt(classifierBytes.length); output.appendBytes(classifierBytes); output.appendInt(labelEnumeration.size()); // Write the enumeration keys for (Map.Entry<Integer, String> entry : labelEnumeration.entrySet()) { output.appendInt(entry.getKey()); byte[] bytes = entry.getValue().getBytes(UTF_8); output.appendInt(bytes.length); output.appendBytes(bytes); } return output.slice().getBytes(); }
@Setup public void setup() { int[] codePointSet = ascii ? ASCII_CODE_POINTS : ALL_CODE_POINTS; ThreadLocalRandom random = ThreadLocalRandom.current(); codePoints = new int[length]; DynamicSliceOutput sliceOutput = new DynamicSliceOutput(length * 4); for (int i = 0; i < codePoints.length; i++) { int codePoint = codePointSet[random.nextInt(codePointSet.length)]; codePoints[i] = codePoint; sliceOutput.appendBytes(new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)); } slice = sliceOutput.slice(); }
private static void writeSimpleGeometry(DynamicSliceOutput output, GeometrySerializationType type, OGCGeometry geometry) { output.appendByte(type.code()); Geometry esriGeometry = requireNonNull(geometry.getEsriGeometry(), "esriGeometry is null"); byte[] shape = geometryToEsriShape(esriGeometry); output.appendBytes(shape); }
@Override public byte[] getSerializedData() { byte[] classifierBytes = serialize(classifier).getBytes(); DynamicSliceOutput output = new DynamicSliceOutput(classifierBytes.length + 64 * labelEnumeration.size()); output.appendInt(classifierBytes.length); output.appendBytes(classifierBytes); output.appendInt(labelEnumeration.size()); // Write the enumeration keys for (Map.Entry<Integer, String> entry : labelEnumeration.entrySet()) { output.appendInt(entry.getKey()); byte[] bytes = entry.getValue().getBytes(UTF_8); output.appendInt(bytes.length); output.appendBytes(bytes); } return output.slice().getBytes(); }
private static void writeSimpleGeometry(DynamicSliceOutput output, GeometrySerializationType type, OGCGeometry geometry) { output.appendByte(type.code()); Geometry esriGeometry = requireNonNull(geometry.getEsriGeometry(), "esriGeometry is null"); byte[] shape = geometryToEsriShape(esriGeometry); output.appendBytes(shape); }
private static void writeSimpleGeometry(DynamicSliceOutput output, GeometrySerializationType type, OGCGeometry geometry) { output.appendByte(type.code()); Geometry esriGeometry = requireNonNull(geometry.getEsriGeometry(), "esriGeometry is null"); byte[] shape = geometryToEsriShape(esriGeometry); output.appendBytes(shape); }
@Setup public void setup() { int[] codePointSet = ascii ? ASCII_CODE_POINTS : ALL_CODE_POINTS; ThreadLocalRandom random = ThreadLocalRandom.current(); codePoints = new int[length]; DynamicSliceOutput sliceOutput = new DynamicSliceOutput(length * 4); for (int i = 0; i < codePoints.length; i++) { int codePoint = codePointSet[random.nextInt(codePointSet.length)]; codePoints[i] = codePoint; sliceOutput.appendBytes(new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)); } slice = sliceOutput.slice(); }
@Setup public void setup() { int[] codePointSet = ascii ? ASCII_CODE_POINTS : ALL_CODE_POINTS; ThreadLocalRandom random = ThreadLocalRandom.current(); codePoints = new int[length]; DynamicSliceOutput sliceOutput = new DynamicSliceOutput(length * 4); for (int i = 0; i < codePoints.length; i++) { int codePoint = codePointSet[random.nextInt(codePointSet.length)]; codePoints[i] = codePoint; sliceOutput.appendBytes(new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)); } slice = sliceOutput.slice(); }
@Setup public void setup() { int[] codePointSet = ascii ? ASCII_CODE_POINTS : ALL_CODE_POINTS; ThreadLocalRandom random = ThreadLocalRandom.current(); codePoints = new int[length]; DynamicSliceOutput sliceOutput = new DynamicSliceOutput(length * 4); for (int i = 0; i < codePoints.length; i++) { int codePoint = codePointSet[random.nextInt(codePointSet.length)]; codePoints[i] = codePoint; sliceOutput.appendBytes(new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)); } slice = sliceOutput.slice(); }
@Setup public void setup() { int[] codePointSet = ascii ? ASCII_CODE_POINTS : ALL_CODE_POINTS; ThreadLocalRandom random = ThreadLocalRandom.current(); codePoints = new int[length]; DynamicSliceOutput sliceOutput = new DynamicSliceOutput(length * 4); for (int i = 0; i < codePoints.length; i++) { int codePoint = codePointSet[random.nextInt(codePointSet.length)]; codePoints[i] = codePoint; sliceOutput.appendBytes(new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)); } slice = sliceOutput.slice(); }
@Setup public void setup() { int[] codePointSet = ascii ? ASCII_CODE_POINTS : ALL_CODE_POINTS; ThreadLocalRandom random = ThreadLocalRandom.current(); codePoints = new int[length]; DynamicSliceOutput sliceOutput = new DynamicSliceOutput(length * 4); for (int i = 0; i < codePoints.length; i++) { int codePoint = codePointSet[random.nextInt(codePointSet.length)]; codePoints[i] = codePoint; sliceOutput.appendBytes(new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8)); } slice = sliceOutput.slice(); }
public Slice serialize() { int size = estimatedSerializedSize(); DynamicSliceOutput output = new DynamicSliceOutput(size) .appendByte(Format.DENSE_V2.getTag()) .appendByte(indexBitLength) .appendByte(baseline) .appendBytes(deltas) .appendShort(overflows); // sort overflow arrays to get consistent serialization for equivalent HLLs sortOverflows(); for (int i = 0; i < overflows; i++) { output.appendShort(overflowBuckets[i]); } for (int i = 0; i < overflows; i++) { output.appendByte(overflowValues[i]); } return output.slice(); }
public Slice serialize() { int size = estimatedSerializedSize(); DynamicSliceOutput output = new DynamicSliceOutput(size) .appendByte(Format.DENSE_V2.getTag()) .appendByte(indexBitLength) .appendByte(baseline) .appendBytes(deltas) .appendShort(overflows); // sort overflow arrays to get consistent serialization for equivalent HLLs sortOverflows(); for (int i = 0; i < overflows; i++) { output.appendShort(overflowBuckets[i]); } for (int i = 0; i < overflows; i++) { output.appendByte(overflowValues[i]); } return output.slice(); }
@Test public void testDeserializeDenseV1Overflow() throws Exception { // bucket 1 has a value of 20 (i.e., baseline = 2, delta == 15, overflow == 3) int indexBitLength = 4; int numberOfBuckets = numberOfBuckets(indexBitLength); Slice serialized = new DynamicSliceOutput(1) .appendByte(Format.DENSE_V1.getTag()) // format tag .appendByte(indexBitLength) // p .appendByte(2) // baseline .appendBytes(new byte[] {0x0F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}) // buckets // overflow bucket .appendByte(0x01) .appendByte(0x00) // overflow value .appendByte(3) .slice(); DenseHll deserialized = new DenseHll(serialized); for (int i = 0; i < numberOfBuckets; i++) { if (i == 1) { assertEquals(deserialized.getValue(i), 20); } else { assertEquals(deserialized.getValue(i), 2); } } deserialized.verify(); }
@Test public void testDeserializeDenseV1NoOverflows() throws Exception { int indexBitLength = 4; int numberOfBuckets = numberOfBuckets(indexBitLength); Slice serialized = new DynamicSliceOutput(1) .appendByte(Format.DENSE_V1.getTag()) // format tag .appendByte(indexBitLength) // p .appendByte(10) // baseline .appendBytes(new byte[numberOfBuckets / 2]) // buckets // overflow bucket .appendByte(0xFF) .appendByte(0xFF) // overflow value .appendByte(0) .slice(); DenseHll deserialized = new DenseHll(serialized); for (int i = 0; i < numberOfBuckets; i++) { assertEquals(deserialized.getValue(i), 10); } deserialized.verify(); }
@Test public void testDeserializeDenseV1NoOverflows() throws Exception { int indexBitLength = 4; int numberOfBuckets = numberOfBuckets(indexBitLength); Slice serialized = new DynamicSliceOutput(1) .appendByte(Format.DENSE_V1.getTag()) // format tag .appendByte(indexBitLength) // p .appendByte(10) // baseline .appendBytes(new byte[numberOfBuckets / 2]) // buckets // overflow bucket .appendByte(0xFF) .appendByte(0xFF) // overflow value .appendByte(0) .slice(); DenseHll deserialized = new DenseHll(serialized); for (int i = 0; i < numberOfBuckets; i++) { assertEquals(deserialized.getValue(i), 10); } deserialized.verify(); }
@Test public void testBaselineAdjusment() throws Exception { byte[] buckets = new byte[] {0x45, 0x23, 0x01, 0x31, 0x22, 0x05, 0x04, 0x01}; Slice expected = new DynamicSliceOutput(1) .appendByte(3) // format tag .appendByte(4) // p .appendByte(2) // baseline .appendBytes(buckets) // buckets // overflows .appendByte(0) .appendByte(0) .slice(); DenseHll hll = new DenseHll(4); for (int i = 0; i < 100; i++) { hll.insertHash(Murmur3Hash128.hash64(i)); } assertSlicesEqual(hll.serialize(), expected); }
@Test public void testBaselineAdjusment() throws Exception { byte[] buckets = new byte[] {0x45, 0x23, 0x01, 0x31, 0x22, 0x05, 0x04, 0x01}; Slice expected = new DynamicSliceOutput(1) .appendByte(3) // format tag .appendByte(4) // p .appendByte(2) // baseline .appendBytes(buckets) // buckets // overflows .appendByte(0) .appendByte(0) .slice(); DenseHll hll = new DenseHll(4); for (int i = 0; i < 100; i++) { hll.insertHash(Murmur3Hash128.hash64(i)); } assertSlicesEqual(hll.serialize(), expected); }
@Test public void testOverflowAfterBaselineIncrement() throws Exception { byte[] buckets = new byte[] {0x45, 0x23, 0x01, 0x31, 0x22, 0x05, 0x04, (byte) 0xF1}; Slice expected = new DynamicSliceOutput(1) .appendByte(3) // format tag .appendByte(4) // p .appendByte(2) // baseline .appendBytes(buckets) // buckets // overflows .appendByte(1) .appendByte(0) // overflow bucket .appendByte(14) .appendByte(0) // overflow value .appendByte(5) .slice(); DenseHll hll = new DenseHll(4); for (int i = 0; i < 100; i++) { hll.insertHash(Murmur3Hash128.hash64(i)); } hll.insertHash(Murmur3Hash128.hash64(37227)); assertSlicesEqual(hll.serialize(), expected); }