@Override public void reset() { current = factory.newBitmap(); } };
BitmapCounter current = factory.newBitmap();
@Override public BitmapCounter deserialize(ByteBuffer in) { try { //The length of RoaringBitmap is larger than 12 if (peekLength(in) == RESULT_SIZE) { int flag = in.getInt(); return factory.newBitmap(in.getLong()); } else { return factory.newBitmap(in); } } catch (IOException e) { throw new RuntimeException(e); } }
public void add(Object key, List keyList, Object value) { if (this.keyList == null) { this.keyList = keyList; } if (this.keyList != null && this.keyList.contains(key)) { BitmapCounter counter = map.computeIfAbsent(key, o -> factory.newBitmap()); counter.orWith((BitmapCounter) value); } }
@Override public ByteBuffer getFinalResult(ByteBuffer in) { ByteBuffer out = ByteBuffer.allocate(RESULT_SIZE); try { BitmapCounter counter = factory.newBitmap(in); out.putInt(IS_RESULT_FLAG); out.putLong(counter.getCount()); } catch (IOException e) { throw new RuntimeException(e); } out.flip(); return out; } }
@Override public BitmapCounter aggregate(BitmapCounter value1, BitmapCounter value2) { BitmapCounter merged = bitmapFactory.newBitmap(); if (value1 != null) { merged.orWith(value1); } if (value2 != null) { merged.orWith(value2); } return merged; }
public long result() { if (keyList == null || keyList.isEmpty()) { return 0; } // if any specified key not in map, the intersection must be 0 for (Object key : keyList) { if (!map.containsKey(key)) { return 0; } } BitmapCounter counter = null; for (Object key : keyList) { BitmapCounter c = map.get(key); if (counter == null) { counter = factory.newBitmap(); counter.orWith(c); } else { counter.andWith(c); } } return counter != null ? counter.getCount() : 0; } }
private BitmapAggregator createBitmapAggr(boolean lowCardinality) { BitmapCounter counter = bitmapFactory.newBitmap(); counter.orWith(lowCardinality ? bitmaps[0] : bitmaps[3]); BitmapAggregator result = new BitmapAggregator(); result.aggregate(counter); return result; }
@Test public void testAggregatorDeserializedCounter() throws IOException { BitmapCounter counter1 = factory.newBitmap(1, 3, 5); BitmapCounter counter2 = factory.newBitmap(1, 2, 4, 6); BitmapCounter counter3 = factory.newBitmap(1, 5, 7); BitmapCounter next = factory.newBitmap(buffer); assertEquals(counter1, next); next = factory.newBitmap(buffer); assertEquals(counter2, next); next = factory.newBitmap(buffer); assertEquals(counter3, next); assertEquals(7, aggregator.getState().getCount()); BitmapCounter result = factory.newBitmap(); result.orWith(counter1); result.orWith(counter2);
BitmapSerializer serializer = new BitmapSerializer(DataType.ANY); BitmapCounter counter = RoaringBitmapCounterFactory.INSTANCE.newBitmap(1, 1234, 5678, 100000);
@Test public void testAggregatorEstimate() { HLLCAggregator hllcAggregator = new HLLCAggregator(14); hllcAggregator.aggregate(new HLLCounter(14)); BitmapAggregator bitmapAggregator = new BitmapAggregator(); BitmapCounter bitmapCounter = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); for (int i = 4000; i <= 100000; i += 2) { bitmapCounter.add(i); } bitmapAggregator.aggregate(bitmapCounter); ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType( DataType.getType("extendedcolumn(100)")); MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator(); extendedColumnAggregator.aggregate(new ByteArray(100)); List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators()); aggregators.add(hllcAggregator); aggregators.add(bitmapAggregator); aggregators.add(extendedColumnAggregator); System.out.printf(Locale.ROOT, "%40s %10s %10s\n", "Class", "Estimate", "Actual"); for (MeasureAggregator aggregator : aggregators) { String clzName = getAggregatorName(aggregator.getClass()); System.out.printf(Locale.ROOT, "%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator)); } }
@Test public void basicTest() { MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") }; BufferedMeasureCodec codec = new BufferedMeasureCodec(descs); Double d = new Double(1.0); Long l = new Long(2); BigDecimal b = new BigDecimal("333.1234"); HLLCounter hllc = new HLLCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); BitmapCounter bitmap = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); bitmap.add(123); bitmap.add(45678); bitmap.add(Integer.MAX_VALUE - 10); Object[] values = new Object[] { d, l, b, hllc, bitmap }; ByteBuffer buf = codec.encode(values); buf.flip(); System.out.println("size: " + buf.limit()); Object[] copy = new Object[values.length]; codec.decode(buf, copy); for (int i = 0; i < values.length; i++) { Object x = values[i]; Object y = copy[i]; assertEquals(x, y); } }
@Test public void testAggregator() { BitmapAggregator aggregator = new BitmapAggregator(); assertNull(null, aggregator.getState()); aggregator.aggregate(factory.newBitmap(10, 20, 30, 40)); assertEquals(4, aggregator.getState().getCount()); aggregator.aggregate(factory.newBitmap(25, 30, 35, 40, 45)); assertEquals(7, aggregator.getState().getCount()); aggregator.reset(); assertNull(aggregator.getState()); }
@Test public void testBitmapCounter() { BitmapCounter counter = factory.newBitmap(10, 20, 30, 1000); assertEquals(4, counter.getCount()); assertTrue(counter.getMemBytes() > 0); BitmapCounter counter2 = factory.newBitmap(); assertEquals(0, counter2.getCount()); counter2.add(10); counter2.add(30); counter2.add(40); counter2.add(2000); assertEquals(4, counter2.getCount()); counter2.orWith(counter); assertEquals(4, counter.getCount()); assertEquals(6, counter2.getCount()); // in-place change int i = 0; int[] values = new int[(int) counter2.getCount()]; for (int value : counter2) { values[i++] = value; } assertArrayEquals(new int[]{10, 20, 30, 40, 1000, 2000}, values); counter2.clear(); assertEquals(0, counter2.getCount()); }
@Override public void reset() { current = factory.newBitmap(); } };
@Override public BitmapCounter deserialize(ByteBuffer in) { try { //The length of RoaringBitmap is larger than 12 if (peekLength(in) == RESULT_SIZE) { int flag = in.getInt(); return factory.newBitmap(in.getLong()); } else { return factory.newBitmap(in); } } catch (IOException e) { throw new RuntimeException(e); } }
public void add(Object key, List keyList, Object value) { if (this.keyList == null) { this.keyList = keyList; } if (this.keyList != null && this.keyList.contains(key)) { BitmapCounter counter = map.computeIfAbsent(key, o -> factory.newBitmap()); counter.orWith((BitmapCounter) value); } }
@Override public ByteBuffer getFinalResult(ByteBuffer in) { ByteBuffer out = ByteBuffer.allocate(RESULT_SIZE); try { BitmapCounter counter = factory.newBitmap(in); out.putInt(IS_RESULT_FLAG); out.putLong(counter.getCount()); } catch (IOException e) { throw new RuntimeException(e); } out.flip(); return out; } }
@Override public BitmapCounter aggregate(BitmapCounter value1, BitmapCounter value2) { BitmapCounter merged = bitmapFactory.newBitmap(); if (value1 != null) { merged.orWith(value1); } if (value2 != null) { merged.orWith(value2); } return merged; }
public long result() { if (keyList == null || keyList.isEmpty()) { return 0; } // if any specified key not in map, the intersection must be 0 for (Object key : keyList) { if (!map.containsKey(key)) { return 0; } } BitmapCounter counter = null; for (Object key : keyList) { BitmapCounter c = map.get(key); if (counter == null) { counter = factory.newBitmap(); counter.orWith(c); } else { counter.andWith(c); } } return counter != null ? counter.getCount() : 0; } }