private static ColumnStatistics doubleColumnStats(Long numberOfValues, Double minimum, Double maximum) { return new ColumnStatistics(numberOfValues, null, null, new DoubleStatistics(minimum, maximum), null, null); }
@Override protected boolean chunkMatchesStats(List<Double> chunk, ColumnStatistics columnStatistics) { assertNull(columnStatistics.getBooleanStatistics()); assertNull(columnStatistics.getIntegerStatistics()); assertNull(columnStatistics.getStringStatistics()); assertNull(columnStatistics.getDateStatistics()); // check basic statistics if (!super.chunkMatchesStats(chunk, columnStatistics)) { return false; } // statistics can be missing for any reason if (columnStatistics.getDoubleStatistics() != null) { // verify min if (!columnStatistics.getDoubleStatistics().getMin().equals(Ordering.natural().nullsLast().min(chunk))) { return false; } // verify max if (!columnStatistics.getDoubleStatistics().getMax().equals(Ordering.natural().nullsFirst().max(chunk))) { return false; } } return true; } }
private static DoubleStatistics toDoubleStatistics(OrcProto.DoubleStatistics doubleStatistics) { if (!doubleStatistics.hasMinimum() && !doubleStatistics.hasMaximum()) { return null; } // TODO remove this when double statistics are changed to correctly deal with NaNs // if either min, max, or sum is NaN, ignore the stat if ((doubleStatistics.hasMinimum() && Double.isNaN(doubleStatistics.getMinimum())) || (doubleStatistics.hasMaximum() && Double.isNaN(doubleStatistics.getMaximum())) || (doubleStatistics.hasSum() && Double.isNaN(doubleStatistics.getSum()))) { return null; } return new DoubleStatistics( doubleStatistics.hasMinimum() ? doubleStatistics.getMinimum() : null, doubleStatistics.hasMaximum() ? doubleStatistics.getMaximum() : null); }
private static DoubleStatistics toDoubleStatistics(OrcProto.DoubleStatistics doubleStatistics) { if (!doubleStatistics.hasMinimum() && !doubleStatistics.hasMaximum()) { return null; } // if either min, max, or sum is NaN, ignore the stat if ((doubleStatistics.hasMinimum() && Double.isNaN(doubleStatistics.getMinimum())) || (doubleStatistics.hasMaximum() && Double.isNaN(doubleStatistics.getMaximum())) || (doubleStatistics.hasSum() && Double.isNaN(doubleStatistics.getSum()))) { return null; } return new DoubleStatistics( doubleStatistics.hasMinimum() ? doubleStatistics.getMinimum() : null, doubleStatistics.hasMaximum() ? doubleStatistics.getMaximum() : null); }