for (FileStatus status : statuses) { if (status.isDir()) { StatsCollectionContext scc = new StatsCollectionContext(conf); String[] names = status.getPath().toString().split(Path.SEPARATOR); String opId = names[names.length - 1]; scc.setStatsTmpDir(status.getPath().toString()); StatsAggregator statsAggregator = new FSStatsAggregator(); if (!statsAggregator.connect(scc)) {
@Override public boolean init(StatsCollectionContext context) { try { for (String tmpDir : context.getStatsTmpDirs()) { Path statsDir = new Path(tmpDir); LOG.debug("Initing FSStatsPublisher with : " + statsDir); statsDir.getFileSystem(context.getHiveConf()).mkdirs(statsDir); LOG.info("created : " + statsDir); } return true; } catch (IOException e) { LOG.error("Failed to create dir", e); return false; } }
private StatsCollectionContext getContext() throws HiveException { StatsCollectionContext scc = new StatsCollectionContext(conf); Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } scc.setTask(sourceTask); scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); return scc; }
@Override public boolean closeConnection(StatsCollectionContext context) { List<String> statsDirs = context.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; if (context.getContextSuffix() == null) { throw new RuntimeException("ContextSuffix must be set before publishing!"); try { String suffix = Integer.toString(conf.getInt("mapred.task.partition", 0)); if (context.getContextSuffix() != null) { suffix += "_" + context.getContextSuffix();
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } } }
private void publishRunTimeStats() throws HiveException { StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getRuntimeStatsTmpDir()); sContext.setContextSuffix(getOperatorId()); if (!statsPublisher.connect(sContext)) { LOG.error("StatsPublishing error: cannot connect to database"); throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg()); } String prefix = ""; Map<String, String> statsToPublish = new HashMap<String, String>(); statsToPublish.put(StatsSetupConst.RUN_TIME_ROW_COUNT, Long.toString(runTimeNumRows)); if (!statsPublisher.publishStat(prefix, statsToPublish)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg()); } if (!statsPublisher.closeConnection(sContext)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg()); } }
private void publishRunTimeStats() throws HiveException { StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setIndexForTezUnion(indexForTezUnion); sContext.setStatsTmpDir(conf.getRuntimeStatsTmpDir()); if (!statsPublisher.connect(sContext)) { LOG.error("StatsPublishing error: cannot connect to database"); throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg()); } String prefix = ""; Map<String, String> statsToPublish = new HashMap<String, String>(); statsToPublish.put(StatsSetupConst.RUN_TIME_ROW_COUNT, Long.toString(runTimeNumRows)); if (!statsPublisher.publishStat(prefix, statsToPublish)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg()); } if (!statsPublisher.closeConnection(sContext)) { // The original exception is lost. // Not changing the interface to maintain backward compatibility throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg()); } }
@Override public boolean closeConnection(StatsCollectionContext context) { List<String> statsDirs = context.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; Path statsDir = new Path(statsDirs.get(0)); try { Path statsFile = null; if (context.getIndexForTezUnion() != -1) { statsFile = new Path(statsDir, StatsSetupConst.STATS_FILE_PREFIX + conf.getInt("mapred.task.partition", 0) + "_" + context.getIndexForTezUnion()); } else { statsFile = new Path(statsDir, StatsSetupConst.STATS_FILE_PREFIX
@Override public boolean closeConnection(StatsCollectionContext scc) { List<String> statsDirs = scc.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; Path statsDir = new Path(statsDirs.get(0)); LOG.debug("About to delete stats tmp dir :" + statsDir); try { fs.delete(statsDir,true); return true; } catch (IOException e) { LOG.error("Failed to delete stats dir", e); return true; } } }
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } }
StatsCollectionContext sc = new StatsCollectionContext(jc); sc.setStatsTmpDir(conf.getTmpStatsDir()); sc.setContextSuffix(getOperatorId()); if (!statsPublisher.connect(sc)) {
StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getStatsTmpDir()); if (!statsPublisher.connect(sContext)) { sContext.setIndexForTezUnion(this.getIndexForTezUnion()); if (!statsPublisher.closeConnection(sContext)) {
@Override public boolean closeConnection(StatsCollectionContext scc) { List<String> statsDirs = scc.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; Path statsDir = new Path(statsDirs.get(0)); LOG.debug("About to delete stats tmp dir :" + statsDir); try { fs.delete(statsDir,true); return true; } catch (IOException e) { LOG.error("Failed to delete stats dir", e); return true; } } }
for (FileStatus status : statuses) { if (status.isDir()) { StatsCollectionContext scc = new StatsCollectionContext(conf); String[] names = status.getPath().toString().split(Path.SEPARATOR); String opId = names[names.length - 1]; scc.setStatsTmpDir(status.getPath().toString()); StatsAggregator statsAggregator = new FSStatsAggregator(); if (!statsAggregator.connect(scc)) {
@Override public boolean connect(StatsCollectionContext context) { conf = context.getHiveConf(); List<String> statsDirs = context.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; Path statsDir = new Path(statsDirs.get(0)); LOG.debug("Connecting to : " + statsDir); statsMap = new HashMap<String, Map<String,String>>(); try { return statsDir.getFileSystem(conf).exists(statsDir); } catch (IOException e) { LOG.error("Failed to check if dir exists", e); return false; } }
StatsFactory factory = StatsFactory.newFactory(conf); if (factory != null) { StatsCollectionContext sCntxt = new StatsCollectionContext(conf); sCntxt.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, conf)); statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(sCntxt)) { // creating stats table if not exists
StatsCollectionContext sContext = new StatsCollectionContext(hconf); sContext.setStatsTmpDir(conf.getTmpStatsDir()); if (!statsPublisher.connect(sContext)) { sContext.setContextSuffix(getOperatorId());
private StatsCollectionContext getContext() throws HiveException { StatsCollectionContext scc = new StatsCollectionContext(conf); Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } scc.setTask(sourceTask); scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); return scc; }
StatsCollectionContext sc = new StatsCollectionContext(jc); sc.setStatsTmpDir(jc.get(StatsSetupConst.STATS_TMP_LOC, "")); if (!statsPublisher.connect(sc)) {
@Override public boolean connect(StatsCollectionContext context) { conf = context.getHiveConf(); List<String> statsDirs = context.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; Path statsDir = new Path(statsDirs.get(0)); LOG.debug("Connecting to : " + statsDir); statsMap = new HashMap<String, Map<String,String>>(); try { return statsDir.getFileSystem(conf).exists(statsDir); } catch (IOException e) { LOG.error("Failed to check if dir exists", e); return false; } }