private static void traverseResolverTreeAndPopulateStartAndEndKeys( FilterResolverIntf filterResolverTree, SegmentProperties segmentProperties, long[] startKeys, SortedMap<Integer, byte[]> setOfStartKeyByteArray, long[] endKeys, SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> startKeyList, List<long[]> endKeyList) { if (null == filterResolverTree) { return; } traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(), segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray, startKeyList, endKeyList); filterResolverTree .getStartKey(segmentProperties, startKeys, setOfStartKeyByteArray, startKeyList); filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray, endKeyList); traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getRight(), segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray, startKeyList, endKeyList); }
/** * constructing the filter resolver tree based on filter expression. * this method will visit each node of the filter resolver and prepares * the surrogates of the filter members which are involved filter * expression. * * @param filterResolverTree * @param tableIdentifier */ private void traverseAndResolveTree(FilterResolverIntf filterResolverTree, AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException { if (null == filterResolverTree) { return; } traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier); filterResolverTree.resolve(tableIdentifier); traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier); }
return new RowLevelRangeLessThanFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getDimColEvaluatorInfoList(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getMsrColEvalutorInfoList(), filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getFilterRangeValues(segmentProperties), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_LESSTHAN_EQUALTO: return new RowLevelRangeLessThanEqualFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getDimColEvaluatorInfoList(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getMsrColEvalutorInfoList(), filterExpressionResolverTree.getFilterExpression(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getFilterRangeValues(segmentProperties), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getMeasureFilterRangeValues(), segmentProperties); case ROWLEVEL_GREATERTHAN_EQUALTO: return new RowLevelRangeGrtrThanEquaToFilterExecuterImpl( ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getDimColEvaluatorInfoList(), ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree) .getMsrColEvalutorInfoList(),
Map<Integer, GenericQueryType> complexDimensionInfoMap, List<CarbonColumn> minMaxCacheColumns) { FilterExecuterType filterExecuterType = filterExpressionResolverTree.getFilterExecuterType(); if (null != filterExecuterType) { switch (filterExecuterType) { case INCLUDE: if (null != filterExpressionResolverTree.getDimColResolvedFilterInfo() && null != filterExpressionResolverTree.getDimColResolvedFilterInfo() .getFilterValues() && filterExpressionResolverTree.getDimColResolvedFilterInfo() .getFilterValues().isOptimized()) { return getExcludeFilterExecuter( filterExpressionResolverTree.getDimColResolvedFilterInfo(), filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties); filterExpressionResolverTree.getDimColResolvedFilterInfo(), filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties); case EXCLUDE: return getExcludeFilterExecuter( filterExpressionResolverTree.getDimColResolvedFilterInfo(), filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties); case OR: return new OrFilterExecuterImpl( createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties, complexDimensionInfoMap, minMaxCacheColumns), createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties, complexDimensionInfoMap, minMaxCacheColumns)); case AND: return new AndFilterExecuterImpl( createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
for (ColumnExpression columnExpression : columnList) { if (columnExpression.isDimension()) { dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo(); ColumnFilterInfo filterInfo = new ColumnFilterInfo(); dimColumnEvaluatorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal()); dimColumnEvaluatorInfo.setRowIndex(index++); dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension()); dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false); if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { if (!isIncludeFilter) { filterInfo.setExcludeFilterList(getDirectSurrogateValues(columnExpression)); } else { filterInfo.setFilterList(getDirectSurrogateValues(columnExpression)); filterInfo.setFilterListForNoDictionaryCols(getNoDictionaryRangeValues()); dimColumnEvaluatorInfo.setFilterValues(filterInfo); dimColumnEvaluatorInfo .addDimensionResolvedFilterInstance(columnExpression.getDimension(), filterInfo); dimColEvaluatorInfoList.add(dimColumnEvaluatorInfo); } else { msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo(); ColumnFilterInfo filterInfo = new ColumnFilterInfo(); msrColumnEvalutorInfo.setMeasure(columnExpression.getMeasure()); msrColumnEvalutorInfo.setRowIndex(index++); msrColumnEvalutorInfo.setCarbonColumn(columnExpression.getCarbonColumn()); msrColumnEvalutorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal()); msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType()); msrColumnEvalutorInfo.setMeasureExistsInCurrentSilce(false);
private DataMapExprWrapper chooseDataMap(DataMapLevel level, FilterResolverIntf resolverIntf) { if (resolverIntf != null) { Expression expression = resolverIntf.getFilterExpression(); List<TableDataMap> datamaps = level == DataMapLevel.CG ? cgDataMaps : fgDataMaps; if (datamaps.size() > 0) { ExpressionTuple tuple = selectDataMap(expression, datamaps, resolverIntf); if (tuple.dataMapExprWrapper != null) { return tuple.dataMapExprWrapper; } } } return null; }
AndExpression andExpression = (AndExpression) expression; ExpressionTuple left = selectDataMap(andExpression.getLeft(), allDataMap, filterResolverIntf.getLeft()); ExpressionTuple right = selectDataMap(andExpression.getRight(), allDataMap, filterResolverIntf.getRight()); Set<ExpressionType> filterExpressionTypes = new HashSet<>(); TrueConditionalResolverImpl resolver = new TrueConditionalResolverImpl( new AndExpression(left.expression, right.expression), false, true); tuple.columnExpressions = columnExpressions; tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, resolver); tuple.expression = resolver.getFilterExpression(); return tuple; } else { tuple.dataMapExprWrapper = new AndDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper, resolver); tuple.expression = resolver.getFilterExpression(); return tuple; OrExpression orExpression = (OrExpression) expression; ExpressionTuple left = selectDataMap(orExpression.getLeft(), allDataMap, filterResolverIntf.getLeft()); ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap, filterResolverIntf.getRight()); TrueConditionalResolverImpl resolver = new TrueConditionalResolverImpl(
switch (filterExpressionType) { case FALSE: return new FalseConditionalResolverImpl(expression, false, false); case TRUE: return new TrueConditionalResolverImpl(expression, false, false); case EQUALS: currentCondExpression = (BinaryConditionalExpression) expression; currentCondExpression.getColumnList().get(0).getCarbonColumn(); if (carbonColumn.hasEncoding(Encoding.IMPLICIT)) { return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()); || FilterUtil .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) { return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true, tableIdentifier); || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) { return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true, tableIdentifier); return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()); || FilterUtil .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) { return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
/** * check if current need to be replaced with TrueFilter expression. This will happen in case * filter column min/max is not cached in the driver * * @param filterExpressionResolverTree * @param segmentProperties * @param minMaxCacheColumns * @return */ private static boolean checkIfCurrentNodeToBeReplacedWithTrueFilterExpression( FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns) { boolean replaceCurrentNodeWithTrueFilter = false; ColumnResolvedFilterInfo columnResolvedFilterInfo = null; if (null != filterExpressionResolverTree.getMsrColResolvedFilterInfo()) { columnResolvedFilterInfo = filterExpressionResolverTree.getMsrColResolvedFilterInfo(); replaceCurrentNodeWithTrueFilter = checkIfFilterColumnIsCachedInDriver(columnResolvedFilterInfo, segmentProperties, minMaxCacheColumns, true); } else { columnResolvedFilterInfo = filterExpressionResolverTree.getDimColResolvedFilterInfo(); if (!columnResolvedFilterInfo.getDimension().hasEncoding(Encoding.IMPLICIT)) { replaceCurrentNodeWithTrueFilter = checkIfFilterColumnIsCachedInDriver(columnResolvedFilterInfo, segmentProperties, minMaxCacheColumns, false); } } return replaceCurrentNodeWithTrueFilter; }
private List<Integer> getDirectSurrogateValues(ColumnExpression columnExpression) throws FilterUnsupportedException { List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType()); if (this.getFilterExpression() instanceof BinaryConditionalExpression) { listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals(); } List<Integer> filterValuesList = new ArrayList<Integer>(20); try { // if any filter member provided by user is invalid throw error else // system can display inconsistent result. for (ExpressionResult result : listOfExpressionResults) { filterValuesList.add(directDictionaryGenerator .generateDirectSurrogateKey(result.getString(), CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)); } } catch (FilterIllegalMemberException e) { throw new FilterUnsupportedException(e); } return filterValuesList; }
/** * Method will return the executer type for particular conditional resolver * basically two types of executers will be formed for the conditional query. * * @return the filter executer type */ @Override public FilterExecuterType getFilterExecuterType() { switch (exp.getFilterExpressionType()) { case NOT_EQUALS: case NOT_IN: return FilterExecuterType.EXCLUDE; case RANGE: if (isColDictionary()) { return FilterExecuterType.INCLUDE; } else { return FilterExecuterType.RANGE; } default: return FilterExecuterType.INCLUDE; } }
case AND: currentExpression = (BinaryExpression) expressionTree; return new LogicalFilterResolverImpl( createFilterResolverTree(currentExpression.getLeft(), tableIdentifier), createFilterResolverTree(currentExpression.getRight(), tableIdentifier), new LiteralExpression(maxValueLimit, literal.getLiteralExpDataType())); currentExpression = new AndExpression(left, right); return new LogicalFilterResolverImpl( createFilterResolverTree(currentExpression.getLeft(), tableIdentifier), createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
@Override public String toString() { return String.format("scan on table %s.%s, %d projection columns with filter (%s)", table.getDatabaseName(), table.getTableName(), projection.getDimensions().size() + projection.getMeasures().size(), filterExpressionResolverTree.getFilterExpression().toString()); }
private List<byte[]> getNoDictionaryRangeValues() { List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); if (this.getFilterExpression() instanceof BinaryConditionalExpression) { listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
public static void getAllFilterDimensionsAndMeasures(FilterResolverIntf filterResolverTree, Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) { if (null == filterResolverTree) { return; } List<ColumnExpression> dimensionResolvedInfos = new ArrayList<ColumnExpression>(); Expression filterExpression = filterResolverTree.getFilterExpression(); addColumnDimensions(filterExpression, filterDimensions, filterMeasure); for (ColumnExpression info : dimensionResolvedInfos) { if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) { filterDimensions.add(info.getDimension()); } } }
private List<Object> getMeasureRangeValues(CarbonMeasure carbonMeasure) { List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20); if (this.getFilterExpression() instanceof BinaryConditionalExpression) { listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals(); } List<Object> filterValuesList = new ArrayList<>(20); boolean invalidRowsPresent = false; for (ExpressionResult result : listOfExpressionResults) { try { if (result.getString() == null) { filterValuesList.add(null); continue; } filterValuesList.add(DataTypeUtil.getMeasureValueBasedOnDataType(result.getString(), result.getDataType(), carbonMeasure.getScale(), carbonMeasure.getPrecision())); } catch (FilterIllegalMemberException e) { // Any invalid member while evaluation shall be ignored, system will log the // error only once since all rows the evaluation happens so inorder to avoid // too much log inforation only once the log will be printed. FilterUtil.logError(e, invalidRowsPresent); } } Collections.sort(filterValuesList, org.apache.carbondata.core.util.comparator.Comparator .getComparatorByDataTypeForMeasure(carbonMeasure.getDataType())); return filterValuesList; }
/** * Return a chosen datamap based on input filter. See {@link DataMapChooser} */ public DataMapExprWrapper choose(FilterResolverIntf filter) { if (filter != null) { Expression expression = filter.getFilterExpression(); // First check for FG datamaps if any exist ExpressionTuple tuple = selectDataMap(expression, fgDataMaps, filter); if (tuple.dataMapExprWrapper == null) { // Check for CG datamap tuple = selectDataMap(expression, cgDataMaps, filter); } if (tuple.dataMapExprWrapper != null) { return tuple.dataMapExprWrapper; } } // Return the default datamap if no other datamap exists. return new DataMapExprWrapperImpl( DataMapStoreManager.getInstance().getDefaultDataMap(carbonTable), filter); }
/** * Whether to use lazy load in vector or not. * @return */ private boolean isUseLazyLoad() { boolean useLazyLoad = false; if (queryModel.getFilterExpressionResolverTree() != null) { Expression expression = queryModel.getFilterExpressionResolverTree().getFilterExpression(); useLazyLoad = true; // In case of join queries only not null filter would e pushed down so check and disable the // lazy load in that case. if (expression instanceof NotEqualsExpression) { try { if (((NotEqualsExpression) expression).getRight().evaluate(null).isNull()) { useLazyLoad = false; } } catch (Exception e) { throw new RuntimeException(e); } } } return useLazyLoad; }
bloomQueryModels = createQueryModel(filterExp.getFilterExpression()); } catch (DictionaryGenerationException | UnsupportedEncodingException e) { LOGGER.error("Exception occurs while creating query model", e);
private void initializeFilter() { List<ColumnSchema> wrapperColumnSchemaList = CarbonUtil .getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getTableName()), carbonTable.getMeasureByTableName(carbonTable.getTableName())); int[] dimLensWithComplex = new int[wrapperColumnSchemaList.size()]; for (int i = 0; i < dimLensWithComplex.length; i++) { dimLensWithComplex[i] = Integer.MAX_VALUE; } int[] dictionaryColumnCardinality = CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchemaList); SegmentProperties segmentProperties = new SegmentProperties(wrapperColumnSchemaList, dictionaryColumnCardinality); Map<Integer, GenericQueryType> complexDimensionInfoMap = new HashMap<>(); FilterResolverIntf resolverIntf = model.getFilterExpressionResolverTree(); filter = FilterUtil.getFilterExecuterTree(resolverIntf, segmentProperties, complexDimensionInfoMap); // for row filter, we need update column index FilterUtil.updateIndexOfColumnExpression(resolverIntf.getFilterExpression(), carbonTable.getDimensionOrdinalMax()); }