@SuppressWarnings("unused") private void testAggregate(ImmutableBitSet groupBy) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions).setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records aggregated to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
public GTScanRequest planScanRequest() { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && !scanRanges.isEmpty()) { scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions) .setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs) .setFilterPushDown(gtFilter)// .setRtAggrMetrics(gtRtAggrMetrics).setDynamicColumns(gtDynColumns) .setExprsPushDown(tupleExpressionMap)// .setAllowStorageAggregation(context.isNeedStorageAggregation()) .setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB())// .setStoragePushDownLimit(context.getFinalPushDownLimit()) .setStorageLimitLevel(context.getStorageLimitLevel()).setHavingFilterPushDown(havingFilter) .createGTScanRequest(); } else { scanRequest = null; } return scanRequest; }
@SuppressWarnings("unused") private void testScanRecords(String msg) throws IOException { long t = System.currentTimeMillis(); IGTScanner scan = simpleStore.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int count = 0; for (GTRecord rec : scan) { count++; } scan.close(); t = System.currentTimeMillis() - t; logger.info(msg + ", " + count + " records, " + speed(t) + "K rec/sec"); }
public static CubeVisitProtos.CubeVisitRequest mockScanRequestWithRuntimeDimensions(GTInfo gtInfo, List<RawScan> rawScans) throws IOException { ImmutableBitSet dimensions = setOf(); ImmutableBitSet aggrGroupBy = setOf(3); ImmutableBitSet aggrMetrics = setOf(2); String[] aggrMetricsFuncs = { "SUM" }; ImmutableBitSet dynColumns = setOf(3); TupleFilter whenFilter = getCompareTupleFilter(1, "Ken"); TupleExpression thenExpr = new NumberTupleExpression(1); List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayList(); whenList.add(new Pair<>(whenFilter, thenExpr)); TupleExpression elseExpr = new NumberTupleExpression(2); /** * case * when user = 'Ken' then 1 * else 2 * end */ TupleExpression caseExpression = new CaseTupleExpression(whenList, elseExpr); Map<Integer, TupleExpression> tupleExpressionMap = Maps.newHashMap(); tupleExpressionMap.put(3, caseExpression); GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(null)// .setDimensions(dimensions).setAggrGroupBy(aggrGroupBy)// .setAggrMetrics(aggrMetrics).setAggrMetricsFuncs(aggrMetricsFuncs)// .setDynamicColumns(dynColumns).setExprsPushDown(tupleExpressionMap)// .setStartTime(System.currentTimeMillis()).createGTScanRequest(); final List<CubeVisitProtos.CubeVisitRequest.IntList> intListList = mockIntList(setOf(2)); return mockScanRequest(rawScans, scanRequest, intListList); }
public static CubeVisitProtos.CubeVisitRequest mockFullScanRequest(GTInfo gtInfo, List<RawScan> rawScans) throws IOException { GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(null).setDimensions(null) .setStartTime(System.currentTimeMillis()).createGTScanRequest(); final List<CubeVisitProtos.CubeVisitRequest.IntList> intListList = mockIntList(setOf(2, 3)); return mockScanRequest(rawScans, scanRequest, intListList); }
protected void outputCuboid(long cuboidId, GridTable gridTable, ICuboidWriter output) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { output.write(cuboidId, record); } scanner.close(); logger.debug("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); }
tupleExpressionMap.put(3, caseExpression); GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(null)// .setDimensions(dimensions).setAggrGroupBy(aggrGroupBy)// .setAggrMetrics(aggrMetrics).setAggrMetricsFuncs(aggrMetricsFuncs)// .setRtAggrMetrics(rtAggrMetrics)// .setDynamicColumns(dynColumns).setExprsPushDown(tupleExpressionMap)// .setStartTime(System.currentTimeMillis()).createGTScanRequest();
@SuppressWarnings("unused") private void testAggregate(ImmutableBitSet groupBy) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions).setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records aggregated to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
@SuppressWarnings("unused") private void testFilter(TupleFilter filter) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()).setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records filtered to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
return new GTScanRequestBuilder().setInfo(sInfo).setRanges(sRanges).setDimensions(sColumns) .setAggrGroupBy(sAggGroupBy).setAggrMetrics(sAggrMetrics).setAggrMetricsFuncs(sAggrMetricFuncs) .setRtAggrMetrics(aRuntimeAggrMetrics).setDynamicColumns(aDynCols) .setExprsPushDown(sTupleExpressionMap) .setFilterPushDown(sGTFilter).setHavingFilterPushDown(sGTHavingFilter) .setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB) .setStorageScanRowNumThreshold(storageScanRowNumThreshold) .setStoragePushDownLimit(storagePushDownLimit).setStorageLimitLevel(storageLimitLevel) .setStartTime(startTime).setTimeout(timeout).setStorageBehavior(storageBehavior) .createGTScanRequest();
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
@Override public void write(long cuboidId, GridTable gridTable) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { write(cuboidId, record); } scanner.close(); logger.info("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); } }
public GTScanRequest planScanRequest() { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && !scanRanges.isEmpty()) { scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions) .setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs) .setFilterPushDown(gtFilter)// .setRtAggrMetrics(gtRtAggrMetrics).setDynamicColumns(gtDynColumns) .setExprsPushDown(tupleExpressionMap)// .setAllowStorageAggregation(context.isNeedStorageAggregation()) .setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB())// .setStoragePushDownLimit(context.getFinalPushDownLimit()) .setStorageLimitLevel(context.getStorageLimitLevel()).setHavingFilterPushDown(havingFilter) .createGTScanRequest(); } else { scanRequest = null; } return scanRequest; }
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs) .setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
@SuppressWarnings("unused") private void testFilter(TupleFilter filter) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()).setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records filtered to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
return new GTScanRequestBuilder().setInfo(sInfo).setRanges(sRanges).setDimensions(sColumns) .setAggrGroupBy(sAggGroupBy).setAggrMetrics(sAggrMetrics).setAggrMetricsFuncs(sAggrMetricFuncs) .setRtAggrMetrics(aRuntimeAggrMetrics).setDynamicColumns(aDynCols) .setExprsPushDown(sTupleExpressionMap) .setFilterPushDown(sGTFilter).setHavingFilterPushDown(sGTHavingFilter) .setAllowStorageAggregation(sAllowPreAggr).setAggCacheMemThreshold(sAggrCacheGB) .setStorageScanRowNumThreshold(storageScanRowNumThreshold) .setStoragePushDownLimit(storagePushDownLimit).setStorageLimitLevel(storageLimitLevel) .setStartTime(startTime).setTimeout(timeout).setStorageBehavior(storageBehavior) .createGTScanRequest();
private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
@SuppressWarnings("unused") private void testScanRaw(String msg) throws IOException { long t = System.currentTimeMillis(); IGTScanner scan = simpleStore.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); ResultScanner innerScanner = ((SimpleHBaseStore.Reader) scan).getHBaseScanner(); int count = 0; for (Result r : innerScanner) { count++; } scan.close(); t = System.currentTimeMillis() - t; logger.info(msg + ", " + count + " rows, " + speed(t) + "K row/sec"); }
@Test public void verifyAggregateAndHavingFilter() throws IOException { GTInfo info = table.getInfo(); TblColRef havingCol = TblColRef.newInnerColumn("SUM_OF_BIGDECIMAL", InnerDataTypeEnum.LITERAL); havingCol.getColumnDesc().setId("1"); // point to the first aggregated measure CompareTupleFilter havingFilter = compare(havingCol, FilterOperatorEnum.GT, "20"); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(1)).setAggrMetrics(setOf(4)).setAggrMetricsFuncs(new String[] { "sum" }) .setHavingFilterPushDown(havingFilter).createGTScanRequest(); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[null, 20, null, null, 42.0]", "[null, 30, null, null, 52.5]"); }
public boolean fetchNext() throws IOException { if (recordIterator == null) { currentCuboidId = splitResult.cuboidId; scanner = splitResult.table.scan(new GTScanRequestBuilder().setInfo(splitResult.table.getInfo()) .setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); return true; } else { scanner.close(); recordIterator = null; return false; } }