private long parseLong(String fieldName) { Map<String, String> params = partish.getPartParameters(); long result = -1; if (params != null) { try { result = Long.parseLong(params.get(fieldName)); } catch (NumberFormatException e) { result = -1; } } return result; }
public void collectFileStatus(Warehouse wh, HiveConf conf) throws MetaException, IOException { if (!partish.isTransactionalTable()) { partfileStatus = wh.getFileStatusesForSD(partish.getPartSd()); } else { Path path = new Path(partish.getPartSd().getLocation()); partfileStatus = AcidUtils.getAcidFilesForStats(partish.getTable(), path, conf, null); isMissingAcidState = true; } }
@Override public String apply(FooterStatCollector sc) { return String.format("%s#%s", sc.partish.getTable().getCompleteName(), sc.partish.getPartishType()); } };
Map<String, String> parameters = partish.getPartParameters(); try { long numRows = 0; if (partish.getTable() != null && AcidUtils.isTransactionalTable(partish.getTable())) { fileList = AcidUtils.getAcidFilesForStats(partish.getTable(), dir, jc, fs); } else { fileList = HiveStatsUtils.getFileStatusRecurse(dir, -1, fs); Utilities.FILE_OP_LOGGER.debug("Computing stats for {}", file); if (!file.isDirectory()) { InputFormat<?, ?> inputFormat = ReflectionUtil.newInstance(partish.getInputFormatClass(), jc); InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { partish.getLocation() }); if (file.getLen() == 0) { numFiles += 1; if (partish.getPartition() != null) { result = new Partition(partish.getTable(), partish.getPartition().getTPartition()); } else { result = new Table(partish.getTable().getTTable()); String msg = partish.getSimpleName() + " stats: [" + toString(parameters) + ']'; LOG.debug(msg); console.printInfo(msg); console.printInfo("[Warning] could not update stats for " + partish.getSimpleName() + ".", "Failed with exception " + e.getMessage() + "\n" + StringUtils.stringifyException(e));
public Object process(StatsAggregator statsAggregator) throws HiveException, MetaException { Partish p = partish; Map<String, String> parameters = p.getPartParameters(); if (work.isTargetRewritten()) { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); String spec = partish.getPartition() == null ? partish.getTable().getTableName() : partish.getPartition().getSpec().toString(); LOG.warn("Partition/partfiles is null for: " + spec); if (isMissingAcidState) { MetaStoreServerUtils.clearQuickStats(parameters); return p.getOutput(); if (StatsSetupConst.areBasicStatsUptoDate(parameters) || p.isTransactionalTable()) { String prefix = getAggregationPrefix(p.getTable(), p.getPartition()); updateStats(statsAggregator, parameters, prefix); return p.getOutput();
public Partish buildPartition() { Partition partition = Mockito.mock(Partition.class); org.apache.hadoop.hive.metastore.api.Partition tpartition = Mockito.mock(org.apache.hadoop.hive.metastore.api.Partition.class); doReturn(tpartition).when(partition).getTPartition(); doReturn(params).when(tpartition).getParameters(); return Partish.buildFor(null, partition); } }
console.printInfo("Table " + tableFullName + " stats: [" + toString(p.getPartParameters()) + ']'); LOG.info("Table " + tableFullName + " stats: [" + toString(p.getPartParameters()) + ']'); Object res = basicStatsProcessor.process(statsAggregator); if (res == null) { LOG.info("Partition " + basicStatsProcessor.partish.getPartition().getSpec() + " stats: [0]"); continue; console.printInfo("Partition " + basicStatsProcessor.partish.getPartition().getSpec() + " stats: [" + toString(basicStatsProcessor.partish.getPartParameters()) + ']'); LOG.info("Partition " + basicStatsProcessor.partish.getPartition().getSpec() + " stats: [" + toString(basicStatsProcessor.partish.getPartParameters()) + ']');
public void init(HiveConf conf, LogHelper console) throws IOException { this.console = console; dir = new Path(partish.getPartSd().getLocation()); fs = dir.getFileSystem(conf); }
for (FooterStatCollector statsCollection : scs) { if (statsCollection.result == null) { LOG.debug("Stats requested to be reliable. Empty stats found: {}", statsCollection.partish.getSimpleName()); return -1;
@Override public void apply(BasicStats stats) { long ds = stats.getRawDataSize(); if (ds <= 0) { ds = stats.getTotalSize(); // if data size is still 0 then get file size if (ds <= 0) { Path path = stats.partish.getPath(); try { ds = getFileSizeForPath(path); } catch (IOException e) { ds = 0L; } } ds = (long) (ds * deserFactor); stats.setDataSize(ds); } }
@Override public void apply(BasicStats stats) { // FIXME: there were different logic for part/table; merge these logics later if (stats.partish.getPartition() == null) { if (stats.getNumRows() < 0 && avgRowSize > 0) { stats.setNumRows(stats.getDataSize() / avgRowSize); } } else { if (avgRowSize > 0) { long rc = stats.getNumRows(); long s = stats.getDataSize(); if (rc <= 0 && s > 0) { rc = s / avgRowSize; stats.setNumRows(rc); } if (s <= 0 && rc > 0) { s = StatsUtils.safeMult(rc, avgRowSize); stats.setDataSize(s); } } } if (stats.getNumRows() > 0) { // FIXME: this promotion process should be removed later if (State.PARTIAL.morePreciseThan(stats.state)) { stats.state = State.PARTIAL; } } } }