Factory basicStatsFactory = new BasicStats.Factory(); basicStatsFactory.addEnhancer(new BasicStats.DataSizeEstimator(conf)); basicStatsFactory.addEnhancer(new BasicStats.RowNumEstimator(estimateRowSizeFromSchema(conf, schema))); basicStatsFactory.addEnhancer(new BasicStats.SetMinRowNumber01()); BasicStats basicStats = basicStatsFactory.build(Partish.buildFor(table)); Factory basicStatsFactory = new Factory(); if (shouldEstimateStats) { basicStatsFactory.addEnhancer(new BasicStats.DataSizeEstimator(conf)); basicStatsFactory.addEnhancer(new BasicStats.RowNumEstimator(estimateRowSizeFromSchema(conf, schema))); BasicStats basicStats = basicStatsFactory.build(Partish.buildFor(table, p)); partStats.add(basicStats);
Factory basicStatsFactory = new BasicStats.Factory(); basicStatsFactory.addEnhancer(new BasicStats.DataSizeEstimator(conf)); basicStatsFactory.addEnhancer(new BasicStats.RowNumEstimator(estimateRowSizeFromSchema(conf, schema))); results = basicStatsFactory.buildAll(conf, inputs);
if (parts.size() <= 1) { for (Partish partish : parts) { ret.add(build(partish));
@Test public void mergeWithEmpty() { HiveConf conf = new HiveConf(); int avgRowSize = 100; int r0 = 13; int r1 = 15; int deserFactor = (int) conf.getFloatVar(ConfVars.HIVE_STATS_DESERIALIZATION_FACTOR); Partish p0 = new LocalPartishBuilder().numRows(r0).rawDataSize(avgRowSize * r0).buildPartition(); Partish p1 = new LocalPartishBuilder().totalSize(r1 * avgRowSize / deserFactor).buildPartition(); BasicStats.Factory factory = new BasicStats.Factory(new BasicStats.DataSizeEstimator(conf), new BasicStats.RowNumEstimator(avgRowSize)); BasicStats bs0 = factory.build(p0); BasicStats bs1 = factory.build(p1); BasicStats res = BasicStats.buildFrom(Lists.newArrayList(bs0, bs1)); assertEquals(r0 + r1, res.getNumRows()); assertEquals(avgRowSize * (r0 + r1), res.getDataSize()); }
@Test @Ignore("HIVE-18062 will fix this") public void mergedKeepsPartialStateEvenIfValuesAreSuccessfullyEstimated() { Partish p0 = new LocalPartishBuilder().numRows(10).rawDataSize(100).buildPartition(); Partish p1 = new LocalPartishBuilder().totalSize(10).buildPartition(); HiveConf conf = new HiveConf(); BasicStats.Factory factory = new BasicStats.Factory(new BasicStats.DataSizeEstimator(conf), new BasicStats.RowNumEstimator(10)); BasicStats bs0 = factory.build(p0); BasicStats bs1 = factory.build(p1); BasicStats res = BasicStats.buildFrom(Lists.newArrayList(bs0, bs1)); assertEquals(State.PARTIAL, res.getState()); }
@Override public BasicStats call() throws Exception { return build(part); } }));
@Test public void testDataSizeEstimator() { Partish p1 = new LocalPartishBuilder().totalSize(10).buildPartition(); HiveConf conf = new HiveConf(); conf.setFloatVar(ConfVars.HIVE_STATS_DESERIALIZATION_FACTOR, 13.0f); BasicStats.Factory factory = new BasicStats.Factory(new BasicStats.DataSizeEstimator(conf)); BasicStats res = factory.build(p1); assertEquals(130, res.getDataSize()); }