@Test public void testBitmapCounter() { BitmapCounter counter = factory.newBitmap(10, 20, 30, 1000); assertEquals(4, counter.getCount()); assertTrue(counter.getMemBytes() > 0); BitmapCounter counter2 = factory.newBitmap(); assertEquals(0, counter2.getCount()); counter2.add(10); counter2.add(30); counter2.add(40); counter2.add(2000); assertEquals(4, counter2.getCount()); counter2.orWith(counter); assertEquals(4, counter.getCount()); assertEquals(6, counter2.getCount()); // in-place change int i = 0; int[] values = new int[(int) counter2.getCount()]; for (int value : counter2) { values[i++] = value; } assertArrayEquals(new int[]{10, 20, 30, 40, 1000, 2000}, values); counter2.clear(); assertEquals(0, counter2.getCount()); }
@Override public int peekLength(ByteBuffer in) { ByteBuffer buffer = in.slice(); if (buffer.getInt(0) == IS_RESULT_FLAG) { return RESULT_SIZE; } else { return DELEGATE.peekLength(in); } }
@Override public void serialize(BitmapCounter value, ByteBuffer out) { try { value.write(out); } catch (IOException e) { throw new RuntimeException(e); } }
public long result() { if (keyList == null || keyList.isEmpty()) { return 0; } // if any specified key not in map, the intersection must be 0 for (Object key : keyList) { if (!map.containsKey(key)) { return 0; } } BitmapCounter counter = null; for (Object key : keyList) { BitmapCounter c = map.get(key); if (counter == null) { counter = factory.newBitmap(); counter.orWith(c); } else { counter.andWith(c); } } return counter != null ? counter.getCount() : 0; } }
@Override public BitmapCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { checkArgument(values.length == 1, "expect 1 value, got %s", Arrays.toString(values)); current.clear(); if (values[0] == null) { return current; } int id; if (needDictionaryColumn(measureDesc.getFunction())) { TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(0); Dictionary<String> dictionary = dictionaryMap.get(literalCol); id = dictionary.getIdFromValue(values[0]); } else { id = Integer.parseInt(values[0]); } current.add(id); return current; }
public static long result(BitmapAggregator agg) { BitmapCounter finalState = agg.getState(); return finalState == null ? 0 : finalState.getCount(); } }
counter1.write(buffer); counter2.write(buffer); counter3.write(buffer); buffer.flip(); assertEquals(6, aggregator.getState().getCount()); assertEquals(7, aggregator.getState().getCount()); result.orWith(counter1); result.orWith(counter2); result.orWith(counter3); assertEquals(result, aggregator.getState());
@Test public void basicTest() { MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") }; BufferedMeasureCodec codec = new BufferedMeasureCodec(descs); Double d = new Double(1.0); Long l = new Long(2); BigDecimal b = new BigDecimal("333.1234"); HLLCounter hllc = new HLLCounter(16); hllc.add("1234567"); hllc.add("abcdefg"); BitmapCounter bitmap = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); bitmap.add(123); bitmap.add(45678); bitmap.add(Integer.MAX_VALUE - 10); Object[] values = new Object[] { d, l, b, hllc, bitmap }; ByteBuffer buf = codec.encode(values); buf.flip(); System.out.println("size: " + buf.limit()); Object[] copy = new Object[values.length]; codec.decode(buf, copy); for (int i = 0; i < values.length; i++) { Object x = values[i]; Object y = copy[i]; assertEquals(x, y); } }
@Override public void aggregate(BitmapCounter value) { // Here we optimize for case when group only has 1 value. In such situation, no // aggregation is needed, so we just keep a reference to the first value, saving // the cost of deserialization and merging. if (sum == null) { sum = value; return; } sum.orWith(value); }
@Override public int getMemBytesEstimate() { return sum == null ? 0 : sum.getMemBytes(); } }
@Override public ByteBuffer getFinalResult(ByteBuffer in) { ByteBuffer out = ByteBuffer.allocate(RESULT_SIZE); try { BitmapCounter counter = factory.newBitmap(in); out.putInt(IS_RESULT_FLAG); out.putLong(counter.getCount()); } catch (IOException e) { throw new RuntimeException(e); } out.flip(); return out; } }
public long result() { if (keyList == null || keyList.isEmpty()) { return 0; } // if any specified key not in map, the intersection must be 0 for (Object key : keyList) { if (!map.containsKey(key)) { return 0; } } BitmapCounter counter = null; for (Object key : keyList) { BitmapCounter c = map.get(key); if (counter == null) { counter = factory.newBitmap(); counter.orWith(c); } else { counter.andWith(c); } } return counter != null ? counter.getCount() : 0; } }
@Test public void testAggregatorEstimate() { HLLCAggregator hllcAggregator = new HLLCAggregator(14); hllcAggregator.aggregate(new HLLCounter(14)); BitmapAggregator bitmapAggregator = new BitmapAggregator(); BitmapCounter bitmapCounter = RoaringBitmapCounterFactory.INSTANCE.newBitmap(); for (int i = 4000; i <= 100000; i += 2) { bitmapCounter.add(i); } bitmapAggregator.aggregate(bitmapCounter); ExtendedColumnMeasureType extendedColumnType = new ExtendedColumnMeasureType( DataType.getType("extendedcolumn(100)")); MeasureAggregator<ByteArray> extendedColumnAggregator = extendedColumnType.newAggregator(); extendedColumnAggregator.aggregate(new ByteArray(100)); List<MeasureAggregator> aggregators = Lists.newArrayList(basicAggregators()); aggregators.add(hllcAggregator); aggregators.add(bitmapAggregator); aggregators.add(extendedColumnAggregator); System.out.printf(Locale.ROOT, "%40s %10s %10s\n", "Class", "Estimate", "Actual"); for (MeasureAggregator aggregator : aggregators) { String clzName = getAggregatorName(aggregator.getClass()); System.out.printf(Locale.ROOT, "%40s %10d %10d\n", clzName, aggregator.getMemBytesEstimate(), meter.measureDeep(aggregator)); } }
@Override public int getMemBytesEstimate() { return sum == null ? 0 : sum.getMemBytes(); } }
@Override public int compare(Object o1, Object o2) { if (o1 == null && o2 == null) return 0; if (o1 == null) // null is bigger to align with CubeCodeSystem return 1; if (o2 == null) // null is bigger to align with CubeCodeSystem return -1; // for the 'having clause', we only concern numbers and BigDecimal // we try to cache the o2, which should be a constant according to CompareTupleFilter.evaluate() double n1; if (o1 instanceof Number) { n1 = ((Number) o1).doubleValue(); } else if (o1 instanceof HLLCounter) { n1 = ((HLLCounter) o1).getCountEstimate(); } else if (o1 instanceof BitmapCounter) { n1 = ((BitmapCounter) o1).getCount(); } else if (o1 instanceof PercentileCounter) { n1 = ((PercentileCounter) o1).getResultEstimate(); } else { throw new RuntimeException("Unknown datatype: value=" + o1 + ", class=" + o1.getClass()); } double n2 = (o2Cache == o2) ? n2Cache : Double.parseDouble((String) o2); if (o2Cache == null) { o2Cache = o2; n2Cache = n2; } return Double.compare(n1, n2); }
@Override public BitmapCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { checkArgument(values.length == 1, "expect 1 value, got %s", Arrays.toString(values)); current.clear(); if (values[0] == null) { return current; } int id; if (needDictionaryColumn(measureDesc.getFunction())) { TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(0); Dictionary<String> dictionary = dictionaryMap.get(literalCol); id = dictionary.getIdFromValue(values[0]); } else { id = Integer.parseInt(values[0]); } current.add(id); return current; }
public void add(long value) { // TODO we need support long later add((int) value); }
public void add(Object key, List keyList, Object value) { if (this.keyList == null) { this.keyList = keyList; } if (this.keyList != null && this.keyList.contains(key)) { BitmapCounter counter = map.computeIfAbsent(key, o -> factory.newBitmap()); counter.orWith((BitmapCounter) value); } }