if (!(filter instanceof LogicalTupleFilter)) { flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND); flatFilter.addChild(filter); return flatFilter; FilterOperatorEnum op = filter.getOperator(); List<TupleFilter> andChildren = new LinkedList<TupleFilter>(); List<TupleFilter> orChildren = new LinkedList<TupleFilter>(); for (TupleFilter child : filter.getChildren()) { TupleFilter flatChild = flattenInternal(child, maxFlatChildrenSize); FilterOperatorEnum childOp = flatChild.getOperator(); if (childOp == FilterOperatorEnum.AND) { andChildren.add(flatChild); flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND); for (TupleFilter andChild : andChildren) { flatFilter.addChildren(andChild.getChildren()); List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter, maxFlatChildrenSize); flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR); flatFilter.addChildren(fullAndFilters); flatFilter.addChildren(orChild.getChildren()); flatFilter.addChildren(andChildren); } else if (op == FilterOperatorEnum.NOT) { assert (filter.children.size() == 1); TupleFilter reverse = filter.children.get(0).reverse(); flatFilter = flattenInternal(reverse, maxFlatChildrenSize);
public boolean evaluate(IEvaluatableTuple tuple) { if (filter == null) return true; else return filter.evaluate(tuple, DictCodeSystem.INSTANCE); }
public static void collectColumns(TupleFilter filter, Set<TblColRef> collector) { if (filter == null || collector == null) return; if (filter instanceof ColumnTupleFilter) { ColumnTupleFilter columnTupleFilter = (ColumnTupleFilter) filter; collector.add(columnTupleFilter.getColumn()); } for (TupleFilter child : filter.getChildren()) { collectColumns(child, collector); } }
public static boolean isEvaluableRecursively(TupleFilter filter) { if (filter == null) return true; if (!filter.isEvaluable()) return false; for (TupleFilter child : filter.getChildren()) { if (!isEvaluableRecursively(child)) return false; } return true; }
private void classifyChildrenByMarking(TupleFilter filter) { if (filter instanceof LogicalTupleFilter) { if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) { for (TupleFilter child : filter.getChildren()) { isInTopLevelANDs.put(child, true); } } } } }
@Override public TupleFilter onSerialize(TupleFilter filter) { if (filter == null) return null; // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE, // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) // will always return FALSE. if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) { TupleFilter.collectColumns(filter, unevaluatableColumnCollector); return ConstantTupleFilter.TRUE; } // shortcut for unEvaluatable filter if (!filter.isEvaluable()) { TupleFilter.collectColumns(filter, unevaluatableColumnCollector); return ConstantTupleFilter.TRUE; } // map to column onto grid table if (colMapping != null && filter instanceof ColumnTupleFilter) { ColumnTupleFilter colFilter = (ColumnTupleFilter) filter; int gtColIdx = mapCol(colFilter.getColumn()); return new ColumnTupleFilter(info.colRef(gtColIdx)); } // encode constants if (useEncodeConstants && filter instanceof CompareTupleFilter) { return encodeConstants((CompareTupleFilter) filter); } return filter; }
private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter, int maxFlatChildrenSize) { List<TupleFilter> oldProductFilters = new LinkedList<TupleFilter>(); oldProductFilters.add(partialAndFilter); for (TupleFilter orFilter : leftOrFilters) { List<TupleFilter> newProductFilters = new LinkedList<TupleFilter>(); for (TupleFilter orChildFilter : orFilter.getChildren()) { for (TupleFilter productFilter : oldProductFilters) { TupleFilter fullAndFilter = productFilter.copy(); fullAndFilter.addChildren(orChildFilter.getChildren()); newProductFilters.add(fullAndFilter); if (newProductFilters.size() > maxFlatChildrenSize) { throw new IllegalStateException("the filter is too large after do the flat, size=" + newProductFilters.size()); } } } oldProductFilters = newProductFilters; } return oldProductFilters; }
public static boolean containsMassInTupleFilter(TupleFilter filter) { if (filter == null) return false; if (filter instanceof MassInTupleFilter) { return true; } for (TupleFilter child : filter.getChildren()) { if (containsMassInTupleFilter(child)) return true; } return false; }
@VisibleForTesting static TupleFilter mergeToInClause(TupleFilter filter) { List<? extends TupleFilter> children = filter.getChildren(); if (children.isEmpty()) { return filter; List<TupleFilter> extraFilters = Lists.newLinkedList(); for (TupleFilter child : children) { if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) { CompareTupleFilter compFilter = (CompareTupleFilter) child; TblColRef column = compFilter.getColumn(); ret.addChildren(extraFilters); for (Map.Entry<TblColRef, Pair<Set<Object>, Map<String, Object>>> entry : inColumnMap.entrySet()) { CompareTupleFilter inFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.IN); inFilter.addChild(new ConstantTupleFilter(entry.getValue().getFirst())); inFilter.getVariables().putAll(entry.getValue().getSecond()); ret.addChild(inFilter); return ret.getChildren().size() == 1 ? ret.getChildren().get(0) : ret;
private void findMustTrueCompareFilters(TupleFilter filter, Set<CompareTupleFilter> result) { if (filter instanceof CompareTupleFilter) { if (((CompareTupleFilter) filter).getColumn() != null) { result.add((CompareTupleFilter) filter); } return; } if (filter instanceof LogicalTupleFilter) { if (filter.getOperator() == FilterOperatorEnum.AND) { for (TupleFilter child : filter.getChildren()) { findMustTrueCompareFilters(child, result); } } return; } }
if (!filter.isEvaluable()) { TupleFilter.collectColumns(filter, inevaluableColumns); return ConstantTupleFilter.TRUE; if (!TupleFilter.isEvaluableRecursively(filter)) { TupleFilter.collectColumns(filter, inevaluableColumns); return ConstantTupleFilter.TRUE;
private ImmutableBitSet collectColumnsInFilter(TupleFilter filter) { Set<TblColRef> columnsInFilter = new HashSet<>(); TupleFilter.collectColumns(filter, columnsInFilter); BitSet result = new BitSet(); for (TblColRef col : columnsInFilter) result.set(col.getColumnDesc().getZeroBasedIndex()); return new ImmutableBitSet(result); }
if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) { TupleFilter.collectColumns(filter, unstrictlyFilteredColumns); return ConstantTupleFilter.TRUE; return filter; if (!TupleFilter.isEvaluableRecursively(filter)) { TupleFilter.collectColumns(filter, unstrictlyFilteredColumns); return ConstantTupleFilter.TRUE;
private void validateFilterPushDown(GTInfo info) { if (!hasFilterPushDown()) return; Set<TblColRef> filterColumns = Sets.newHashSet(); TupleFilter.collectColumns(filterPushDown, filterColumns); for (TblColRef col : filterColumns) { // filter columns must belong to the table info.validateColRef(col); // filter columns must be returned to satisfy upper layer evaluation (calcite) columns = columns.set(col.getColumnDesc().getZeroBasedIndex()); } // un-evaluatable filter must be removed if (!TupleFilter.isEvaluableRecursively(filterPushDown)) { Set<TblColRef> unevaluableColumns = Sets.newHashSet(); filterPushDown = GTUtil.convertFilterUnevaluatable(filterPushDown, info, unevaluableColumns); // columns in un-evaluatable filter must be returned without loss so upper layer can do final evaluation if (hasAggregation()) { for (TblColRef col : unevaluableColumns) { aggrGroupBy = aggrGroupBy.set(col.getColumnDesc().getZeroBasedIndex()); } } } }
protected TupleFilter flattenToOrAndFilter(TupleFilter filter) { if (filter == null) return null; TupleFilter flatFilter = filter.flatFilter(); // normalize to OR-AND filter if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) { LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR); f.addChild(flatFilter); flatFilter = f; } if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR) throw new IllegalStateException(); return flatFilter; }
@Override public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) { boolean matched = false; for (int i = 0; i < whenFilters.size(); i++) { TupleFilter whenFilter = whenFilters.get(i); if (whenFilter.evaluate(tuple, cs)) { TupleFilter thenFilter = thenFilters.get(i); thenFilter.evaluate(tuple, cs); values = thenFilter.getValues(); matched = true; break; } } if (!matched) { if (elseFilter != null) { elseFilter.evaluate(tuple, cs); values = elseFilter.getValues(); } else { values = Collections.emptyList(); } } return true; }
@Override public boolean isEvaluable() { switch (operator) { case NOT: // Un-evaluatable branch will be pruned and be replaced with TRUE. // And this must happen at the top NOT, otherwise NOT (TRUE) becomes false. for (TupleFilter child : children) { if (TupleFilter.isEvaluableRecursively(child) == false) return false; } return true; case OR: // (anything OR un-evaluable) will become (anything or TRUE) which is effectively TRUE. // The "anything" is not evaluated, kinda disabled, by the un-evaluable part. // If it's partially un-evaluable, then "anything" is partially disabled, and the OR is still not fully evaluatable. for (TupleFilter child : children) { if (TupleFilter.isEvaluableRecursively(child) == false) return false; } return true; default: return true; } }
private void collectNonEvaluable(TupleFilter filter, Set<TblColRef> collector) { if (filter == null) return; if (filter.isEvaluable()) { for (TupleFilter child : filter.getChildren()) collectNonEvaluable(child, collector); } else { collectColumnsRecursively(filter, collector); } }
private static void internalSerialize(TupleFilter filter, Decorator decorator, ByteBuffer buffer, IFilterCodeSystem<?> cs) { if (decorator != null) { // give decorator a chance to manipulate the output filter filter = decorator.onSerialize(filter); } if (filter == null) { return; } if (filter.hasChildren()) { // serialize filter+true serializeFilter(1, filter, buffer, cs); // serialize children for (TupleFilter child : filter.getChildren()) { internalSerialize(child, decorator, buffer, cs); } // serialize none serializeFilter(-1, filter, buffer, cs); } else { // serialize filter+false serializeFilter(0, filter, buffer, cs); } }
final public void addChildren(List<? extends TupleFilter> children) { for (TupleFilter c : children) addChild(c); // subclass overrides addChild() }