/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ private static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance(clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
protected RetryingMetaStoreClient(Configuration conf, Class<?>[] constructorArgTypes, Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { this.ugi = getUGI(); if (this.ugi == null) { LOG.warn("RetryingMetaStoreClient unable to determine current user UGI."); } this.retryLimit = MetastoreConf.getIntVar(conf, ConfVars.THRIFT_FAILURE_RETRIES); this.retryDelaySeconds = MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; this.connectionLifeTimeInMillis = MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_LIFETIME, TimeUnit.MILLISECONDS); this.lastConnectionTime = System.currentTimeMillis(); String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS); localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = JavaUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs); LOG.info("RetryingMetaStoreClient proxy=" + msClientClass + " ugi=" + this.ugi + " retries=" + this.retryLimit + " delay=" + this.retryDelaySeconds + " lifetime=" + this.connectionLifeTimeInMillis); }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ public static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { @SuppressWarnings("unchecked") Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance( clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { if (e.getMessage().matches(".* class not found")) { // TODO MS-SPLIT For now if we cannot load the default PartitionExpressionForMetastore // class (since it's from ql) load the DefaultPartitionExpressionProxy, which just throws // UnsupportedOperationExceptions. This allows existing Hive instances to work but also // allows us to instantiate the metastore stand alone for testing. Not sure if this is // the best long term solution. return new DefaultPartitionExpressionProxy(); } LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
public static ExpressionTree makeExpressionTree(PartitionExpressionProxy expressionProxy, byte[] expr, String defaultPartitionName) throws MetaException { // We will try pushdown first, so make the filter. This will also validate the expression, // if serialization fails we will throw incompatible metastore error to the client. String filter = null; try { filter = expressionProxy.convertExprToFilter(expr, defaultPartitionName); } catch (MetaException ex) { // TODO MS-SPLIT - for now we have construct this by reflection because IMetaStoreClient // can't be // moved until after HiveMetaStore is moved, which can't be moved until this is moved. Class<? extends MetaException> exClass = JavaUtils.getClass( "org.apache.hadoop.hive.metastore.IMetaStoreClient$IncompatibleMetastoreException", MetaException.class); throw JavaUtils.newInstance(exClass, new Class<?>[]{String.class}, new Object[]{ex.getMessage()}); } // Make a tree out of the filter. // TODO: this is all pretty ugly. The only reason we need all these transformations // is to maintain support for simple filters for HCat users that query metastore. // If forcing everyone to use thick client is out of the question, maybe we could // parse the filter into standard hive expressions and not all this separate tree // Filter.g stuff. That way this method and ...ByFilter would just be merged. return PartFilterExprUtil.makeExpressionTree(filter); }
private static void startRemoteOnlyTasks(Configuration conf) throws Exception { if(!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_INITIATOR_ON)) { return; } ThreadPool.initialize(conf); Collection<String> taskNames = MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_REMOTE_ONLY); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS); ThreadPool.getPool().scheduleAtFixedRate(task, freq, freq, TimeUnit.MILLISECONDS); } }
for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS);
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ private static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance(clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ public static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { @SuppressWarnings("unchecked") Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance( clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { if (e.getMessage().matches(".* class not found")) { // TODO MS-SPLIT For now if we cannot load the default PartitionExpressionForMetastore // class (since it's from ql) load the DefaultPartitionExpressionProxy, which just throws // UnsupportedOperationExceptions. This allows existing Hive instances to work but also // allows us to instantiate the metastore stand alone for testing. Not sure if this is // the best long term solution. return new DefaultPartitionExpressionProxy(); } LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
public static ExpressionTree makeExpressionTree(PartitionExpressionProxy expressionProxy, byte[] expr) throws MetaException { // We will try pushdown first, so make the filter. This will also validate the expression, // if serialization fails we will throw incompatible metastore error to the client. String filter = null; try { filter = expressionProxy.convertExprToFilter(expr); } catch (MetaException ex) { // TODO MS-SPLIT - for now we have construct this by reflection because IMetaStoreClient // can't be // moved until after HiveMetaStore is moved, which can't be moved until this is moved. Class<? extends MetaException> exClass = JavaUtils.getClass( "org.apache.hadoop.hive.metastore.IMetaStoreClient$IncompatibleMetastoreException", MetaException.class); throw JavaUtils.newInstance(exClass, new Class<?>[]{String.class}, new Object[]{ex.getMessage()}); } // Make a tree out of the filter. // TODO: this is all pretty ugly. The only reason we need all these transformations // is to maintain support for simple filters for HCat users that query metastore. // If forcing everyone to use thick client is out of the question, maybe we could // parse the filter into standard hive expressions and not all this separate tree // Filter.g stuff. That way this method and ...ByFilter would just be merged. return PartFilterExprUtil.makeExpressionTree(filter); }
private static void startRemoteOnlyTasks(Configuration conf) throws Exception { if(!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_INITIATOR_ON)) { return; } ThreadPool.initialize(conf); Collection<String> taskNames = MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_REMOTE_ONLY); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS); ThreadPool.getPool().scheduleAtFixedRate(task, freq, freq, TimeUnit.MILLISECONDS); } }
for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS);