if (statistics.genericGetMin() == null || statistics.genericGetMax() == null) { return Domain.create(ValueSet.all(type), hasNullValue);
@Override public <T extends Comparable<T>> Boolean visit(GtEq<T> gtEq) { Column<T> filterColumn = gtEq.getColumn(); T value = gtEq.getValue(); ColumnChunkMetaData columnChunk = getColumnChunk(filterColumn.getColumnPath()); Statistics<T> stats = columnChunk.getStatistics(); if (stats.isEmpty()) { // we have no statistics available, we cannot drop any chunks return false; } if (isAllNulls(columnChunk)) { // we are looking for records where v >= someValue // this chunk is all nulls, so we can drop it return true; } // drop if value >= max return value.compareTo(stats.genericGetMax()) > 0; }
@Override public <T extends Comparable<T>> Boolean visit(Gt<T> gt) { Column<T> filterColumn = gt.getColumn(); T value = gt.getValue(); ColumnChunkMetaData columnChunk = getColumnChunk(filterColumn.getColumnPath()); Statistics<T> stats = columnChunk.getStatistics(); if (stats.isEmpty()) { // we have no statistics available, we cannot drop any chunks return false; } if (isAllNulls(columnChunk)) { // we are looking for records where v > someValue // this chunk is all nulls, so we can drop it return true; } // drop if value >= max return value.compareTo(stats.genericGetMax()) >= 0; }
@Override public <T extends Comparable<T>> Boolean visit(Eq<T> eq) { Column<T> filterColumn = eq.getColumn(); T value = eq.getValue(); ColumnChunkMetaData columnChunk = getColumnChunk(filterColumn.getColumnPath()); Statistics<T> stats = columnChunk.getStatistics(); if (stats.isEmpty()) { // we have no statistics available, we cannot drop any chunks return false; } if (value == null) { // we are looking for records where v eq(null) // so drop if there are no nulls in this chunk return !hasNulls(columnChunk); } if (isAllNulls(columnChunk)) { // we are looking for records where v eq(someNonNull) // and this is a column of all nulls, so drop it return true; } // drop if value < min || value > max return value.compareTo(stats.genericGetMin()) < 0 || value.compareTo(stats.genericGetMax()) > 0; }
@Override public <T extends Comparable<T>> Boolean visit(NotEq<T> notEq) { Column<T> filterColumn = notEq.getColumn(); T value = notEq.getValue(); ColumnChunkMetaData columnChunk = getColumnChunk(filterColumn.getColumnPath()); Statistics<T> stats = columnChunk.getStatistics(); if (stats.isEmpty()) { // we have no statistics available, we cannot drop any chunks return false; } if (value == null) { // we are looking for records where v notEq(null) // so, if this is a column of all nulls, we can drop it return isAllNulls(columnChunk); } if (hasNulls(columnChunk)) { // we are looking for records where v notEq(someNonNull) // but this chunk contains nulls, we cannot drop it return false; } // drop if this is a column where min = max = value return value.compareTo(stats.genericGetMin()) == 0 && value.compareTo(stats.genericGetMax()) == 0; }
private <T extends Comparable<T>, U extends UserDefinedPredicate<T>> Boolean visit(UserDefined<T, U> ud, boolean inverted) { Column<T> filterColumn = ud.getColumn(); ColumnChunkMetaData columnChunk = getColumnChunk(filterColumn.getColumnPath()); U udp = ud.getUserDefinedPredicate(); Statistics<T> stats = columnChunk.getStatistics(); if (stats.isEmpty()) { // we have no statistics available, we cannot drop any chunks return false; } if (isAllNulls(columnChunk)) { // there is no min max, there is nothing // else we can say about this chunk, we // cannot drop it. return false; } parquet.filter2.predicate.Statistics<T> udpStats = new parquet.filter2.predicate.Statistics<T>(stats.genericGetMin(), stats.genericGetMax()); if (inverted) { return udp.inverseCanDrop(udpStats); } else { return udp.canDrop(udpStats); } }
if (statistics.genericGetMin() == null || statistics.genericGetMax() == null) { return Domain.create(ValueSet.all(type), hasNullValue);