public static org.apache.parquet.column.statistics.Statistics<?> readStats(Statistics statistics, PrimitiveTypeName type) { org.apache.parquet.column.statistics.Statistics<?> stats = org.apache.parquet.column.statistics.Statistics.getStatsBasedOnType(type); if (statistics != null) { if (statistics.isSetMax() && statistics.isSetMin()) { stats.setMinMaxFromBytes(statistics.min.array(), statistics.max.array()); } stats.setNumNulls(statistics.null_count); } return stats; }
public static org.apache.parquet.column.statistics.Statistics<?> readStats(Statistics statistics, PrimitiveTypeName type) { org.apache.parquet.column.statistics.Statistics<?> stats = org.apache.parquet.column.statistics.Statistics.getStatsBasedOnType(type); if (statistics != null) { if (statistics.isSetMax() && statistics.isSetMin()) { stats.setMinMaxFromBytes(statistics.min.array(), statistics.max.array()); } stats.setNumNulls(statistics.null_count); } return stats; }
@SuppressWarnings("unchecked") @Override public Statistics<T> visitTypedFieldExpr(TypedFieldExpr typedFieldExpr, Void value) throws RuntimeException { final ColumnStatistics<T> columnStatistics = columnStatMap.get(typedFieldExpr.getPath()); if (columnStatistics != null) { return columnStatistics.getStatistics(); } else if (typedFieldExpr.getMajorType().equals(Types.OPTIONAL_INT)) { // field does not exist. Statistics<T> statistics = Statistics.getStatsBasedOnType(PrimitiveType.PrimitiveTypeName.INT32); statistics.setNumNulls(rowCount); // all values are nulls return statistics; } return null; }
stat.setNumNulls(numNulls); case DATE: convertedStat = new LongStatistics(); convertedStat.setNumNulls(stat.getNumNulls()); long minMS = convertToDrillDateValue(Integer.parseInt(min.toString())); long maxMS = convertToDrillDateValue(Integer.parseInt(max.toString()));
TypeProtos.MajorType type = Types.required(TypeProtos.MinorType.VARCHAR); Statistics stat = new BinaryStatistics(); stat.setNumNulls(0); byte[] val = implicitColValues.get(columnName).getBytes(); stat.setMinMaxFromBytes(val, val);
@Test public void testMatchesWithStatistics() throws ParquetCorruptionException { String value = "Test"; ColumnDescriptor columnDescriptor = new ColumnDescriptor(new String[] {"path"}, BINARY, 0, 0); RichColumnDescriptor column = new RichColumnDescriptor(columnDescriptor, new PrimitiveType(OPTIONAL, BINARY, "Test column")); TupleDomain<ColumnDescriptor> effectivePredicate = getEffectivePredicate(column, createVarcharType(255), utf8Slice(value)); TupleDomainParquetPredicate parquetPredicate = new TupleDomainParquetPredicate(effectivePredicate, singletonList(column)); Statistics stats = getStatsBasedOnType(column.getType()); stats.setNumNulls(1L); stats.setMinMaxFromBytes(value.getBytes(), value.getBytes()); assertTrue(parquetPredicate.matches(2, singletonMap(column, stats), ID, true)); }
TypeProtos.MajorType type = Types.required(TypeProtos.MinorType.VARCHAR); Statistics stat = new BinaryStatistics(); stat.setNumNulls(0); byte[] val = implicitColValues.get(columnName).getBytes(); stat.setMinMaxFromBytes(val, val);
return null; statistics.setNumNulls(input.getNumNulls()); return statistics; } catch (Exception e) {