@Test public void testLongStatsToColumnStatistics() { LongColumnStatsData longColumnStatsData = new LongColumnStatsData(); longColumnStatsData.setLowValue(0); longColumnStatsData.setHighValue(100); longColumnStatsData.setNumNulls(1); longColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BIGINT_TYPE_NAME, longStats(longColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.of(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(100)))); assertEquals(actual.getDoubleStatistics(), Optional.empty()); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
longStats.setLowValue(Long.parseLong(value)); } else if (fName.equals("highValue")) { longStats.setHighValue(Long.parseLong(value)); } else { throw new SemanticException("Unknown stat");
private static void unpackLongStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumDVs(v); } else if (fName.equals("max")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setHighValue(v); } else if (fName.equals("min")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setLowValue(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getLongStats().setBitVectors(buf); ; } }
private void unpackLongStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumDVs(v); } else if (fName.equals("max")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setHighValue(v); } else if (fName.equals("min")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setLowValue(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getLongStats().setBitVectors(v);; } }
unsetHighValue(); } else { setHighValue((Long)value);
@Override ColumnStatisticsObj generate() { LongColumnStatsData longData = new LongColumnStatsData(genNumNulls(), genNumDvs()); longData.setLowValue(genLowVal()); longData.setHighValue(genHighVal()); ColumnStatisticsData data = new ColumnStatisticsData(); data.setLongStats(longData); return new ColumnStatisticsObj(colName, colType, data); }
private ColumnStatisticsObj getDummyLongColStat(String colName, int highVal, int lowVal, int numDVs, int numNulls) { ColumnStatisticsObj aggrColStats = new ColumnStatisticsObj(); aggrColStats.setColName(colName); aggrColStats.setColType("long"); LongColumnStatsData longStatsData = new LongColumnStatsData(); longStatsData.setHighValue(highVal); longStatsData.setLowValue(lowVal); longStatsData.setNumDVs(numDVs); longStatsData.setNumNulls(numNulls); ColumnStatisticsData aggrColStatsData = new ColumnStatisticsData(); aggrColStatsData.setLongStats(longStatsData); aggrColStats.setStatsData(aggrColStatsData); return aggrColStats; } }
ColumnStatisticsData data = new ColumnStatisticsData(); LongColumnStatsData dcsd = new LongColumnStatsData(); dcsd.setHighValue(1000 + i); dcsd.setLowValue(-1000 - i); dcsd.setNumNulls(i);
LongColumnStatsData newLongStatsData = newStatObj.getStatsData().getLongStats(); if (newLongStatsData.isSetHighValue()) { oldLongStatsData.setHighValue(newLongStatsData.getHighValue());
private void unpackLongStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumDVs(v); } else if (fName.equals("max")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setHighValue(v); } else if (fName.equals("min")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setLowValue(v); } }
@Test public void testLongStatsToColumnStatistics() { LongColumnStatsData longColumnStatsData = new LongColumnStatsData(); longColumnStatsData.setLowValue(0); longColumnStatsData.setHighValue(100); longColumnStatsData.setNumNulls(1); longColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BIGINT_TYPE_NAME, longStats(longColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.of(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(100)))); assertEquals(actual.getDoubleStatistics(), Optional.empty()); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
@Test public void testLongStatsToColumnStatistics() { LongColumnStatsData longColumnStatsData = new LongColumnStatsData(); longColumnStatsData.setLowValue(0); longColumnStatsData.setHighValue(100); longColumnStatsData.setNumNulls(1); longColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BIGINT_TYPE_NAME, longStats(longColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.of(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(100)))); assertEquals(actual.getDoubleStatistics(), Optional.empty()); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
@Test public void testLongStatsToColumnStatistics() { LongColumnStatsData longColumnStatsData = new LongColumnStatsData(); longColumnStatsData.setLowValue(0); longColumnStatsData.setHighValue(100); longColumnStatsData.setNumNulls(1); longColumnStatsData.setNumDVs(20); ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BIGINT_TYPE_NAME, longStats(longColumnStatsData)); HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000)); assertEquals(actual.getIntegerStatistics(), Optional.of(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(100)))); assertEquals(actual.getDoubleStatistics(), Optional.empty()); assertEquals(actual.getDecimalStatistics(), Optional.empty()); assertEquals(actual.getDateStatistics(), Optional.empty()); assertEquals(actual.getBooleanStatistics(), Optional.empty()); assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty()); assertEquals(actual.getNullsCount(), OptionalLong.of(1)); assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19)); }
unsetHighValue(); } else { setHighValue((Long)value);
unsetHighValue(); } else { setHighValue((Long)value);
} else if (colType.equals(ByteType.INSTANCE)) { LongColumnStatsData tinyint = new LongColumnStatsData(colStat.nullCount(), colStat.ndv()); tinyint.setHighValue(((Byte) colStat.max()).longValue()); tinyint.setLowValue(((Byte) colStat.min()).longValue()); } else if (colType.equals(ShortType.INSTANCE)) { LongColumnStatsData smallint = new LongColumnStatsData(colStat.nullCount(), colStat.ndv()); smallint.setHighValue(((Short) colStat.max()).longValue()); smallint.setLowValue(((Short) colStat.min()).longValue()); } else if (colType.equals(IntType.INSTANCE)) { LongColumnStatsData integer = new LongColumnStatsData(colStat.nullCount(), colStat.ndv()); integer.setHighValue(((Number) colStat.max()).longValue()); integer.setLowValue(((Number) colStat.min()).longValue()); } else if (colType.equals(LongType.INSTANCE)) { LongColumnStatsData longStats = new LongColumnStatsData(colStat.nullCount(), colStat.ndv()); longStats.setHighValue(((Long) colStat.max()).longValue()); longStats.setLowValue(((Long) colStat.min()).longValue()); } else if (colType instanceof TimestampType) { LongColumnStatsData timestampStats = new LongColumnStatsData(colStat.nullCount(), colStat.ndv()); timestampStats.setHighValue(((Timestamp) colStat.max()).getTime()); timestampStats.setLowValue(((Timestamp) colStat.min()).getTime());
unsetHighValue(); } else { setHighValue((Long)value);
longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); if (lhigh != null) { longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh));
LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(mStatsObj.getNumNulls()); longStats.setHighValue(mStatsObj.getLongHighValue()); longStats.setLowValue(mStatsObj.getLongLowValue()); longStats.setNumDVs(mStatsObj.getNumDVs());
LongColumnStatsData longStats = new LongColumnStatsData(); longStats.setNumNulls(mStatsObj.getNumNulls()); longStats.setHighValue(mStatsObj.getLongHighValue()); longStats.setLowValue(mStatsObj.getLongLowValue()); longStats.setNumDVs(mStatsObj.getNumDVs());