@SuppressWarnings("unused") private void testScanRecords(String msg) throws IOException { long t = System.currentTimeMillis(); IGTScanner scan = simpleStore.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int count = 0; for (GTRecord rec : scan) { count++; } scan.close(); t = System.currentTimeMillis() - t; logger.info(msg + ", " + count + " records, " + speed(t) + "K rec/sec"); }
protected void outputCuboid(long cuboidId, GridTable gridTable, ICuboidWriter output) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { output.write(cuboidId, record); } scanner.close(); logger.debug("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); }
@SuppressWarnings("unused") private void testFilter(TupleFilter filter) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()).setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records filtered to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
@Override public void write(long cuboidId, GridTable gridTable) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { write(cuboidId, record); } scanner.close(); logger.info("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); } }
@SuppressWarnings("unused") private void testFilter(TupleFilter filter) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(info.getAllColumns()).setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records filtered to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
@SuppressWarnings("unused") private void testScanRaw(String msg) throws IOException { long t = System.currentTimeMillis(); IGTScanner scan = simpleStore.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); ResultScanner innerScanner = ((SimpleHBaseStore.Reader) scan).getHBaseScanner(); int count = 0; for (Result r : innerScanner) { count++; } scan.close(); t = System.currentTimeMillis() - t; logger.info(msg + ", " + count + " rows, " + speed(t) + "K row/sec"); }
@SuppressWarnings("unused") private void testAggregate(ImmutableBitSet groupBy) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions).setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records aggregated to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
@SuppressWarnings("unused") private void testAggregate(ImmutableBitSet groupBy) throws IOException { long t = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(dimensions).setAggrGroupBy(groupBy).setAggrMetrics(metrics).setAggrMetricsFuncs(aggrFuncs).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = req.decorateScanner(gen.generate(N)); long count = 0; for (GTRecord rec : scanner) { count++; } t = System.currentTimeMillis() - t; System.out.println(N + " records aggregated to " + count + ", " + calcSpeed(t) + "K rec/sec"); }
public boolean fetchNext() throws IOException { if (recordIterator == null) { currentCuboidId = splitResult.cuboidId; scanner = splitResult.table.scan(new GTScanRequestBuilder().setInfo(splitResult.table.getInfo()) .setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); return true; } else { scanner.close(); recordIterator = null; return false; } }
@SuppressWarnings("unused") private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException { long start = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { i++; } scanner.close(); long end = System.currentTimeMillis(); System.out.println( (end - start) + "ms with filter cache enabled=" + FilterResultCache.DEFAULT_OPTION + ", " + i + " rows"); }
private IGTScanner scan(GridTable table) throws IOException { GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); for (GTRecord r : scanner) { Object[] v = r.getValues(); assertTrue(((String) v[0]).startsWith("2015-")); assertTrue(((String) v[2]).equals("Food")); assertTrue(((Long) v[3]).longValue() == 10); assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); System.out.println(r); } scanner.close(); return scanner; }
public void run() { try { IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); } scanner.close(); } catch (Exception ex) { ex.printStackTrace(); } } };
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
public static CubeVisitProtos.CubeVisitRequest mockFullScanRequest(GTInfo gtInfo, List<RawScan> rawScans) throws IOException { GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(null).setDimensions(null) .setStartTime(System.currentTimeMillis()).createGTScanRequest(); final List<CubeVisitProtos.CubeVisitRequest.IntList> intListList = mockIntList(setOf(2, 3)); return mockScanRequest(rawScans, scanRequest, intListList); }
public GTScanRequest planScanRequest() { GTScanRequest scanRequest; List<GTScanRange> scanRanges = this.planScanRanges(); if (scanRanges != null && !scanRanges.isEmpty()) { scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions) .setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs) .setFilterPushDown(gtFilter)// .setRtAggrMetrics(gtRtAggrMetrics).setDynamicColumns(gtDynColumns) .setExprsPushDown(tupleExpressionMap)// .setAllowStorageAggregation(context.isNeedStorageAggregation()) .setAggCacheMemThreshold(cubeSegment.getConfig().getQueryCoprocessorMemGB())// .setStoragePushDownLimit(context.getFinalPushDownLimit()) .setStorageLimitLevel(context.getStorageLimitLevel()).setHavingFilterPushDown(havingFilter) .createGTScanRequest(); } else { scanRequest = null; } return scanRequest; }
@Test public void verifyFirstRow() throws IOException { doScanAndVerify(table, new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null) .setFilterPushDown(null).createGTScanRequest(), "[1421193600000, 30, Yang, 10, 10.5]", // "[1421193600000, 30, Luke, 10, 10.5]", // "[1421280000000, 20, Dong, 10, 10.5]", // "[1421280000000, 20, Jason, 10, 10.5]", // "[1421280000000, 30, Xu, 10, 10.5]", // "[1421366400000, 20, Mahone, 10, 10.5]", // "[1421366400000, 20, Qianhao, 10, 10.5]", // "[1421366400000, 30, George, 10, 10.5]", // "[1421366400000, 30, Shaofeng, 10, 10.5]", // "[1421452800000, 10, Kejia, 10, 10.5]"); }
private void verifyWriteAndRead(GridTable table) throws IOException { GTInfo info = table.getInfo(); GTBuilder builder = table.rebuild(); for (GTRecord r : data) { builder.write(r); } builder.close(); IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); } scanner.close(); } }
@Test public void verifyAggregateAndHavingFilter() throws IOException { GTInfo info = table.getInfo(); TblColRef havingCol = TblColRef.newInnerColumn("SUM_OF_BIGDECIMAL", InnerDataTypeEnum.LITERAL); havingCol.getColumnDesc().setId("1"); // point to the first aggregated measure CompareTupleFilter havingFilter = compare(havingCol, FilterOperatorEnum.GT, "20"); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(1)).setAggrMetrics(setOf(4)).setAggrMetricsFuncs(new String[] { "sum" }) .setHavingFilterPushDown(havingFilter).createGTScanRequest(); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[null, 20, null, null, 42.0]", "[null, 30, null, null, 52.5]"); }
@Test public void verifyScanWithEvaluatableFilter() throws IOException { GTInfo info = table.getInfo(); CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10")); LogicalTupleFilter filter = and(fComp1, fComp2); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }) .setFilterPushDown(filter).createGTScanRequest(); // note the evaluatable column 1 in filter is added to returned columns but not in group by assertEquals( "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString()); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]"); }
@Test public void verifyScanWithUnevaluatableFilter() throws IOException { GTInfo info = table.getInfo(); CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1)); LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1))); LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }) .setFilterPushDown(filter).createGTScanRequest(); // note the unEvaluatable column 1 in filter is added to group by assertEquals( "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString()); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]"); }