private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(scc)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(scc)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
/** * Sets the paths of the implementation classes of publishing * and aggregation (IStatsPublisher and IStatsAggregator interfaces). * The paths are determined according to a configuration parameter which * is passed as the user input for choosing the implementation as MySQL, HBase, ... */ public static StatsFactory newFactory(String configurationParam, Configuration conf) { StatsFactory factory = new StatsFactory(conf); if (factory.initialize(configurationParam.toLowerCase())) { return factory; } return null; }
public static StatsPublisher getStatsPublisher(JobConf jc) { StatsFactory factory = StatsFactory.newFactory(jc); return factory == null ? null : factory.getStatsPublisher(); }
public static StatsPublisher getStatsPublisher(JobConf jc) { String statsImplementationClass = HiveConf.getVar(jc, HiveConf.ConfVars.HIVESTATSDBCLASS); if (StatsFactory.setImplementation(statsImplementationClass, jc)) { return StatsFactory.getStatsPublisher(); } else { return null; } }
/** * This method is static as it is called from the shutdown hook at the ExecDriver. */ public static void cleanUp(String jobID, Configuration config) { StatsAggregator statsAggregator; String statsImplementationClass = HiveConf.getVar(config, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory.setImplementation(statsImplementationClass, config); statsAggregator = StatsFactory.getStatsAggregator(); if (statsAggregator.connect(config)) { statsAggregator.cleanUp(jobID + Path.SEPARATOR); // Adding the path separator to avoid an Id being a prefix of another ID } } }
public static StatsFactory newFactory(Configuration conf) { return newFactory(HiveConf.getVar(conf, HIVESTATSDBCLASS), conf); }
private List<String> getRequiredCounterPrefix(StatsTask statsTask) throws HiveException, MetaException { List<String> prefixs = new LinkedList<String>(); StatsWork statsWork = statsTask.getWork(); String tablePrefix = getTablePrefix(statsWork); List<Map<String, String>> partitionSpecs = getPartitionSpecs(statsWork); int maxPrefixLength = StatsFactory.getMaxPrefixLength(conf); if (partitionSpecs == null) { prefixs.add(Utilities.getHashedStatsPrefix(tablePrefix, maxPrefixLength)); } else { for (Map<String, String> partitionSpec : partitionSpecs) { String prefixWithPartition = Utilities.join(tablePrefix, Warehouse.makePartPath(partitionSpec)); prefixs.add(Utilities.getHashedStatsPrefix(prefixWithPartition, maxPrefixLength)); } } return prefixs; }
public static StatsPublisher getStatsPublisher(JobConf jc) { StatsFactory factory = StatsFactory.newFactory(jc); return factory == null ? null : factory.getStatsPublisher(); }
if (StatsFactory.setImplementation(statsImplementationClass, job)) { statsPublisher = StatsFactory.getStatsPublisher();
private int aggregateStats() { StatsFactory.setImplementation(statsImplementationClass, conf); StatsAggregator statsAggregator = StatsFactory.getStatsAggregator();
public static StatsFactory newFactory(Configuration conf) { return newFactory(HiveConf.getVar(conf, HIVESTATSDBCLASS), conf); }
tsDesc.setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(conf));
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } } }
private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(scc)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(scc)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
/** * Sets the paths of the implementation classes of publishing * and aggregation (IStatsPublisher and IStatsAggregator interfaces). * The paths are determined according to a configuration parameter which * is passed as the user input for choosing the implementation as MySQL, HBase, ... */ public static StatsFactory newFactory(String configurationParam, Configuration conf) { StatsFactory factory = new StatsFactory(conf); if (factory.initialize(configurationParam.toLowerCase())) { return factory; } return null; }
public static StatsFactory newFactory(Configuration conf) { return newFactory(HiveConf.getVar(conf, HIVESTATSDBCLASS), conf); }
int maxPrefixLength = StatsFactory.getMaxPrefixLength(jc);
private void initStatsPublisher(BaseWork work) throws HiveException { // initialize stats publisher if necessary if (work.isGatheringStats()) { StatsPublisher statsPublisher; StatsFactory factory = StatsFactory.newFactory(jobConf); if (factory != null) { statsPublisher = factory.getStatsPublisher(); StatsCollectionContext sc = new StatsCollectionContext(jobConf); sc.setStatsTmpDirs(Utilities.getStatsTmpDirs(work, jobConf)); if (!statsPublisher.init(sc)) { // creating stats table if not exists if (HiveConf.getBoolVar(jobConf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { throw new HiveException( ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } } } } }
private StatsAggregator createStatsAggregator(HiveConf conf) throws HiveException { String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } // initialize stats publishing table for noscan which has only stats task // the rest of MR task following stats task initializes it in ExecDriver.java StatsPublisher statsPublisher = factory.getStatsPublisher(); if (!statsPublisher.init(conf)) { // creating stats table if not exists throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); } Task sourceTask = getWork().getSourceTask(); if (sourceTask == null) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); } // manufacture a StatsAggregator StatsAggregator statsAggregator = factory.getStatsAggregator(); if (!statsAggregator.connect(conf, sourceTask)) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); } return statsAggregator; }
/** * Sets the paths of the implementation classes of publishing * and aggregation (IStatsPublisher and IStatsAggregator interfaces). * The paths are determined according to a configuration parameter which * is passed as the user input for choosing the implementation as MySQL, HBase, ... */ public static StatsFactory newFactory(String configurationParam, Configuration conf) { StatsFactory factory = new StatsFactory(conf); if (factory.initialize(configurationParam.toLowerCase())) { return factory; } return null; }