logger.info("pre aggregating results before returning"); this.doingStorageAggregation = true; result = new GTAggregateScanner(result, this, spillEnabled); } else { logger.info("has no aggregation, skip it");
private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
.setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()) .setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); try (GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req)) { aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);
GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest);
@Test public void testAggregationCacheSpill() throws IOException { IGTScanner inputScanner = new IGTScanner() { @Override public GTInfo getInfo() { return INFO; } @Override public void close() throws IOException { } @Override public Iterator<GTRecord> iterator() { return TEST_DATA.iterator(); } }; GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest(); GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest); int count = 0; for (GTRecord record : scanner) { assertNotNull(record); Object[] returnRecord = record.getValues(); assertEquals(20, ((Long) returnRecord[3]).longValue()); assertEquals(21, ((BigDecimal) returnRecord[4]).longValue()); count++; //System.out.println(record); } assertEquals(DATA_CARDINALITY, count); scanner.close(); }
logger.info("pre aggregating results before returning"); this.doingStorageAggregation = true; result = new GTAggregateScanner(result, this, spillEnabled); } else { logger.info("has no aggregation, skip it");
private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
.setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()) .setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); try (GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req)) { aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);