@Test public void testEmptySegment() { CubeSegment seg = cube.getFirstSegment(); TblColRef col = cube.getModel().findColumn("CUSTOMER.C_NATION"); // a normal hit TupleFilter f = compare(col, FilterOperatorEnum.EQ, "CHINA"); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(seg)); // make the segment empty, it should be pruned seg.setInputRecords(0); Assert.assertFalse(segmentPruner.check(seg)); }
@Override public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo); List<CubeSegmentScanner> scanners = Lists.newArrayList(); SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter); for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) { CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), // request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), // request.getMetrics(), request.getDynFuncs(), // request.getFilter(), request.getHavingFilter(), request.getContext()); if (!scanner.isSegmentSkipped()) scanners.add(scanner); } if (scanners.isEmpty()) return ITupleIterator.EMPTY_TUPLE_ITERATOR; return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, request.getContext(), sqlDigest); }
dimRangeInfo = tryDeduceRangeFromPartitionCol(seg, col); if (dimRangeInfo == null) continue; String maxVal = dimRangeInfo.getMax(); if (!satisfy(comp, minVal, maxVal)) { logger.debug("Prune segment {} due to given filter", seg); return false;
public List<CubeSegment> listSegmentsForQuery(CubeInstance cube) { List<CubeSegment> r = new ArrayList<>(); for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { if (check(seg)) r.add(seg); } return r; }
private DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) { DataModelDesc model = seg.getModel(); PartitionDesc part = model.getPartitionDesc(); if (!part.isPartitioned()) return null; if (!col.equals(part.getPartitionDateColumnRef())) return null; // deduce the dim range from TSRange TSRange tsRange = seg.getTSRange(); if (tsRange.start.isMin || tsRange.end.isMax) return null; // DimensionRangeInfo cannot express infinite String min = tsRangeToStr(tsRange.start.v, part); String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive return new DimensionRangeInfo(min, max); }
String filterVal = toString(comp.getFirstValue());
public List<CubeSegment> listSegmentsForQuery(CubeInstance cube) { List<CubeSegment> r = new ArrayList<>(); for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { if (check(seg)) r.add(seg); } return r; }
private DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) { DataModelDesc model = seg.getModel(); PartitionDesc part = model.getPartitionDesc(); if (!part.isPartitioned()) return null; if (!col.equals(part.getPartitionDateColumnRef())) return null; // deduce the dim range from TSRange TSRange tsRange = seg.getTSRange(); if (tsRange.start.isMin || tsRange.end.isMax) return null; // DimensionRangeInfo cannot express infinite String min = tsRangeToStr(tsRange.start.v, part); String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive return new DimensionRangeInfo(min, max); }
String filterVal = toString(comp.getFirstValue());
SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertFalse(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertFalse(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertFalse(segmentPruner.check(cubeSegment)); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(cubeSegment));
@Override public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) { GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo); List<CubeSegmentScanner> scanners = Lists.newArrayList(); SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter); for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) { CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), // request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), // request.getMetrics(), request.getDynFuncs(), // request.getFilter(), request.getHavingFilter(), request.getContext()); if (!scanner.isSegmentSkipped()) scanners.add(scanner); } if (scanners.isEmpty()) return ITupleIterator.EMPTY_TUPLE_ITERATOR; return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, request.getContext(), sqlDigest); }
dimRangeInfo = tryDeduceRangeFromPartitionCol(seg, col); if (dimRangeInfo == null) continue; String maxVal = dimRangeInfo.getMax(); if (!satisfy(comp, minVal, maxVal)) { logger.debug("Prune segment {} due to given filter", seg); return false;
@Test public void testLegacyCubeSeg() { // legacy cube segments does not have DimensionRangeInfo, but with TSRange can do some pruning CubeInstance cube = CubeManager.getInstance(getTestConfig()) .getCube("test_kylin_cube_without_slr_left_join_ready_2_segments"); TblColRef col = cube.getModel().findColumn("TEST_KYLIN_FACT.CAL_DT"); CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0); TSRange tsRange = seg.getTSRange(); long start = tsRange.start.v; try (SetAndUnsetSystemProp sns = new SetAndUnsetSystemProp("kylin.query.skip-empty-segments", "false")) { { TupleFilter f = compare(col, FilterOperatorEnum.LTE, start); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertTrue(segmentPruner.check(seg)); } { TupleFilter f = compare(col, FilterOperatorEnum.LT, start); SegmentPruner segmentPruner = new SegmentPruner(f); Assert.assertFalse(segmentPruner.check(seg)); } } } }