scc.setStatsTmpDir(status.getPath().toString()); StatsAggregator statsAggregator = new FSStatsAggregator(); if (!statsAggregator.connect(scc)) { String value = statsAggregator.aggregateStats("", StatsSetupConst.RUN_TIME_ROW_COUNT); opIdToRuntimeNumRows.put(opId, Long.parseLong(value)); statsAggregator.closeConnection(scc);
/** * This method is static as it is called from the shutdown hook at the ExecDriver. */ public static void cleanUp(String jobID, Configuration config) { StatsAggregator statsAggregator; String statsImplementationClass = HiveConf.getVar(config, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory.setImplementation(statsImplementationClass, config); statsAggregator = StatsFactory.getStatsAggregator(); if (statsAggregator.connect(config)) { statsAggregator.cleanUp(jobID + Path.SEPARATOR); // Adding the path separator to avoid an Id being a prefix of another ID } } }
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String prefix, int maxPrefixLength, boolean atomic) throws HiveException { String aggKey = Utilities.getHashedStatsPrefix(prefix, maxPrefixLength); for (String statType : StatsSetupConst.statsRequireCompute) { String value = statsAggregator.aggregateStats(aggKey, statType); if (value != null && !value.isEmpty()) { long longValue = Long.parseLong(value); if (work.getLoadTableDesc() != null && !work.getLoadTableDesc().getReplace()) { String originalValue = parameters.get(statType); if (originalValue != null && !originalValue.equals("-1")) { longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid } } parameters.put(statType, String.valueOf(longValue)); } else { if (atomic) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); } } } statsAggregator.cleanUp(aggKey); }
private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(scc)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(scc)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
statsAggregator.closeConnection(scc);
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey) throws HiveException { for (String statType : StatsSetupConst.STATS_REQUIRE_COMPUTE) { String value = statsAggregator.aggregateStats(aggKey, statType); if (value != null && !value.isEmpty()) { long longValue = Long.parseLong(value); if (!work.isTargetRewritten()) { String originalValue = parameters.get(statType); if (originalValue != null) { longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid } } parameters.put(statType, String.valueOf(longValue)); } } }
private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(scc)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(scc)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
statsAggregator.closeConnection(scc);
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String prefix, boolean atomic) throws HiveException { String aggKey = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR; for (String statType : StatsSetupConst.statsRequireCompute) { String value = statsAggregator.aggregateStats(aggKey, statType); if (value != null && !value.isEmpty()) { long longValue = Long.parseLong(value); if (work.getLoadTableDesc() != null && !work.getLoadTableDesc().getReplace()) { String originalValue = parameters.get(statType); if (originalValue != null) { longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid } } parameters.put(statType, String.valueOf(longValue)); } else { if (atomic) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); } } } }
scc.setStatsTmpDir(status.getPath().toString()); StatsAggregator statsAggregator = new FSStatsAggregator(); if (!statsAggregator.connect(scc)) { String value = statsAggregator.aggregateStats("", StatsSetupConst.RUN_TIME_ROW_COUNT); opIdToRuntimeNumRows.put(opId, Long.parseLong(value)); statsAggregator.closeConnection(scc);
private StatsAggregator createStatsAggregator(HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(conf)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(conf, sourceTask)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
statsAggregator.closeConnection();
if (!statsAggregator.connect(conf)) { throw new HiveException("StatsAggregator connect failed " + statsImplementationClass); String rows = statsAggregator.aggregateStats(work.getAggKey(), StatsSetupConst.ROW_COUNT); if (rows != null) { tblStats.setNumRows(Long.parseLong(rows)); String rows = statsAggregator.aggregateStats(partitionID, StatsSetupConst.ROW_COUNT); if (rows != null) { newPartStats.setNumRows(Long.parseLong(rows)); + StringUtils.stringifyException(e)); } finally { statsAggregator.closeConnection();