/** * method will read the actual data from the direct dictionary generator * by passing direct dictionary value. * * @param dimColumnEvaluatorInfo * @param dictionaryValue * @return */ private Object getFilterActualValueFromDirectDictionaryValue( DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) { if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.DATE) { return dateDictionaryGenerator.getValueFromSurrogate(dictionaryValue); } else if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.TIMESTAMP) { return timestampDictionaryGenerator.getValueFromSurrogate(dictionaryValue); } else { throw new RuntimeException("Invalid data type for dierct dictionary"); } }
/** * This method will initialize the dimension info for the current block to be * used for filtering the data */ private void initDimensionChunkIndexes() { for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) { // find the dimension in the current block dimensions list CarbonDimension dimensionFromCurrentBlock = segmentProperties .getDimensionFromCurrentBlock(dimColEvaluatorInfoList.get(i).getDimension()); if (null != dimensionFromCurrentBlock) { dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); this.dimensionChunkIndex[i] = dimColEvaluatorInfoList.get(i).getColumnIndexInMinMaxByteArray(); isDimensionPresentInCurrentBlock[i] = true; } } }
public RowLevelRangeLessThanEqualFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } ifDefaultValueMatchesFilter(); if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } }
public RowLevelRangeLessThanFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } ifDefaultValueMatchesFilter(); if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } }
RowLevelRangeGrtrThanEquaToFilterExecuterImpl( List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } if (isDimensionPresentInCurrentBlock[0] == true) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } ifDefaultValueMatchesFilter(); }
RowLevelRangeGrtThanFiterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList, List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues, Object[] msrFilterRangeValues, SegmentProperties segmentProperties) { super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; this.msrFilterRangeValues = msrFilterRangeValues; if (!this.msrColEvalutorInfoList.isEmpty()) { CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure(); comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()); } if (isDimensionPresentInCurrentBlock[0]) { isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex() && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn(); } ifDefaultValueMatchesFilter(); }
/** * This method will return the filter values which is present in the range level * conditional expressions. * * @return */ public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) { if (null != dimColResolvedFilterInfo.getFilterValues() && !dimColResolvedFilterInfo .getDimension().hasEncoding(Encoding.DICTIONARY)) { List<byte[]> noDictFilterValuesList = dimColResolvedFilterInfo.getFilterValues().getNoDictionaryFilterValuesList(); return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][])); } else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(), this.dimColResolvedFilterInfo.getDimension(), segmentProperties, false); } return null; }
/** * This method will initialize the dimension info for the current block to be * used for filtering the data */ private void initDimensionChunkIndexes() { // find the dimension in the current block dimensions list CarbonDimension dimensionFromCurrentBlock = segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension()); if (null != dimensionFromCurrentBlock) { dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); isDimensionPresentInCurrentBlock = true; } }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
/** * This method will compute the default value for a dimension * * @param dimColumnEvaluatorInfo * @return */ private Object getDimensionDefaultValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) { Object dimensionDefaultValue = null; CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension(); if (dimension.hasEncoding(Encoding.DICTIONARY) && !dimension .hasEncoding(Encoding.DIRECT_DICTIONARY)) { byte[] defaultValue = dimension.getDefaultValue(); if (null != defaultValue) { dimensionDefaultValue = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); } } else { dimensionDefaultValue = RestructureUtil.validateAndGetDefaultValue(dimension); } return dimensionDefaultValue; }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0) .getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]); } return scanRequired; }
private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int columnIndex) { boolean scanRequired; DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType(); // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0).getDimension() .hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[columnIndex], this.filterRangeValues, dataType); } else { scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[columnIndex], this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[columnIndex]); } return scanRequired; }
/** * This method will check whether default value is present in the given filter values */ private void ifDefaultValueMatchesFilter() { isDefaultValuePresentInFilter = false; if (!this.isDimensionPresentInCurrentBlock && null != filterRangesValues) { CarbonDimension dimension = this.dimColEvaluatorInfo.getDimension(); byte[] defaultValue = dimension.getDefaultValue(); if (null != defaultValue) { int minCompare = FilterUtil.compareValues(filterRangesValues[0], defaultValue, dimension, true); int maxCompare = FilterUtil.compareValues(filterRangesValues[1], defaultValue, dimension, false); if (((greaterThanExp && maxCompare > 0) || (greaterThanEqualExp && maxCompare >= 0)) && ( (lessThanExp && minCompare > 0) || (lessThanEqualExp && minCompare >= 0))) { isDefaultValuePresentInFilter = true; } } } }
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { if (isDimensionPresentInCurrentBlock[0]) { byte[] col = (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal()); return ByteUtil.compare(filterRangeValues[0], col) <= 0; } if (isMeasurePresentInCurrentBlock[0]) { Object col = value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax); return comparator.compare(msrFilterRangeValues[0], col) <= 0; } return false; }
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { if (isDimensionPresentInCurrentBlock[0]) { byte[] col = (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal()); return ByteUtil.compare(filterRangeValues[0], col) < 0; } if (isMeasurePresentInCurrentBlock[0]) { Object col = value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax); return comparator.compare(msrFilterRangeValues[0], col) < 0; } return false; }
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { if (isDimensionPresentInCurrentBlock[0]) { byte[] col = (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal()); return ByteUtil.compare(filterRangeValues[0], col) > 0; } if (isMeasurePresentInCurrentBlock[0]) { Object col = value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax); return comparator.compare(msrFilterRangeValues[0], col) > 0; } return false; }
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) throws FilterUnsupportedException, IOException { if (isDimensionPresentInCurrentBlock[0]) { byte[] col = (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal()); return ByteUtil.compare(filterRangeValues[0], col) >= 0; } if (isMeasurePresentInCurrentBlock[0]) { Object col = value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax); return comparator.compare(msrFilterRangeValues[0], col) >= 0; } return false; }
private boolean isScanRequired(DimensionRawColumnChunk dimensionRawColumnChunk, int columnIndex) { boolean scanRequired; // for no dictionary measure column comparison can be done // on the original data as like measure column if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType()) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) { scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex], dimensionRawColumnChunk.getMinValues()[columnIndex], dimColumnExecuterInfo.getFilterKeys(), dimColumnEvaluatorInfo.getDimension().getDataType()); } else { scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex], dimensionRawColumnChunk.getMinValues()[columnIndex], dimColumnExecuterInfo.getFilterKeys(), dimensionRawColumnChunk.getMinMaxFlagArray()[columnIndex]); } return scanRequired; }
@Override public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException { if (isDimensionPresentInCurrentBlock[0]) { if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) { super.readColumnChunks(rawBlockletColumnChunks); } int chunkIndex = dimensionChunkIndex[0]; if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex); } } else if (isMeasurePresentInCurrentBlock[0]) { int chunkIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) { rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk( rawBlockletColumnChunks.getFileReader(), chunkIndex); } } } }