private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } } }
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } }
private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(scc)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(scc)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(scc)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(scc)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
sCntxt.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, conf)); statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(sCntxt)) { // creating stats table if not exists if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
StatsCollectionContext sc = new StatsCollectionContext(job); sc.setStatsTmpDir(work.getStatsTmpDir()); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
sCntxt.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, conf)); statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(sCntxt)) { // creating stats table if not exists if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(jobConf)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } }
if (StatsFactory.setImplementation(statsImplementationClass, job)) { statsPublisher = StatsFactory.getStatsPublisher(); statsPublisher.init(job); // creating stats table if not exists
private StatsAggregator createStatsAggregator(HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(conf)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(conf, sourceTask)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
if (factory != null) { statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(conf)) { // creating stats table if not exists if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
if (factory != null) { statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(job)) { // creating stats table if not exists if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw
if (factory != null) { statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(job)) { // creating stats table if not exists if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw