for (FileStatus status : statuses) { if (status.isDir()) { StatsCollectionContext scc = new StatsCollectionContext(conf); String[] names = status.getPath().toString().split(Path.SEPARATOR); String opId = names[names.length - 1];
for (FileStatus status : statuses) { if (status.isDir()) { StatsCollectionContext scc = new StatsCollectionContext(conf); String[] names = status.getPath().toString().split(Path.SEPARATOR); String opId = names[names.length - 1];
StatsCollectionContext sc = new StatsCollectionContext(jc); sc.setStatsTmpDir(conf.getTmpStatsDir()); sc.setContextSuffix(getOperatorId());
StatsCollectionContext sc = new StatsCollectionContext(jc); sc.setStatsTmpDir(jc.get(StatsSetupConst.STATS_TMP_LOC, "")); if (!statsPublisher.connect(sc)) {
StatsCollectionContext sc = new StatsCollectionContext(jc); sc.setStatsTmpDir(conf.getTmpStatsDir()); if (!statsPublisher.connect(sc)) {
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
private void publishRunTimeStats() throws HiveException { StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setIndexForTezUnion(indexForTezUnion); sContext.setStatsTmpDir(conf.getRuntimeStatsTmpDir()); if (!statsPublisher.connect(sContext)) { LOG.error("StatsPublishing error: cannot connect to database"); throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg()); } String prefix = ""; Map<String, String> statsToPublish = new HashMap<String, String>(); statsToPublish.put(StatsSetupConst.RUN_TIME_ROW_COUNT, Long.toString(runTimeNumRows)); if (!statsPublisher.publishStat(prefix, statsToPublish)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg()); } if (!statsPublisher.closeConnection(sContext)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg()); } }
private void publishRunTimeStats() throws HiveException { StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getRuntimeStatsTmpDir()); sContext.setContextSuffix(getOperatorId()); if (!statsPublisher.connect(sContext)) { LOG.error("StatsPublishing error: cannot connect to database"); throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg()); } String prefix = ""; Map<String, String> statsToPublish = new HashMap<String, String>(); statsToPublish.put(StatsSetupConst.RUN_TIME_ROW_COUNT, Long.toString(runTimeNumRows)); if (!statsPublisher.publishStat(prefix, statsToPublish)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg()); } if (!statsPublisher.closeConnection(sContext)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg()); } }
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
private StatsCollectionContext getContext() throws HiveException { StatsCollectionContext scc = new StatsCollectionContext(conf); Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } scc.setTask(sourceTask); scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); return scc; }
StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getTmpStatsDir()); if (!statsPublisher.connect(sContext)) {
private StatsCollectionContext getContext() throws HiveException { StatsCollectionContext scc = new StatsCollectionContext(conf); Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } scc.setTask(sourceTask); scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); return scc; }
StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getStatsTmpDir()); if (!statsPublisher.connect(sContext)) {
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } } }
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } }
statsTmpDir.addAll(Utilities.getStatsTmpDirs(rWork, job)); StatsCollectionContext sc = new StatsCollectionContext(job); sc.setStatsTmpDirs(statsTmpDir); if (!statsPublisher.init(sc)) { // creating stats table if not exists
StatsFactory factory = StatsFactory.newFactory(conf); if (factory != null) { StatsCollectionContext sCntxt = new StatsCollectionContext(conf); sCntxt.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, conf)); statsPublisher = factory.getStatsPublisher();
statsTmpDir.addAll(Utilities.getStatsTmpDirs(rWork, job)); StatsCollectionContext sc = new StatsCollectionContext(job); sc.setStatsTmpDirs(statsTmpDir); if (!statsPublisher.init(sc)) { // creating stats table if not exists
if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(job); sc.setStatsTmpDir(work.getStatsTmpDir()); if (!statsPublisher.init(sc)) { // creating stats table if not exists
StatsFactory factory = StatsFactory.newFactory(conf); if (factory != null) { StatsCollectionContext sCntxt = new StatsCollectionContext(conf); sCntxt.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, conf)); statsPublisher = factory.getStatsPublisher();