/** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf conf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(Configuration hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { @SuppressWarnings("unchecked") Class<? extends IMetaStoreClient> baseClass = JavaUtils.getClass(mscClassName, IMetaStoreClient.class); RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance( RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
public static RawStore getProxy(Configuration hiveConf, Configuration conf, String rawStoreClassName, int id) throws MetaException { Class<? extends RawStore> baseClass = JavaUtils.getClass(rawStoreClassName, RawStore.class); RawStoreProxy handler = new RawStoreProxy(hiveConf, conf, baseClass, id); // Look for interfaces on both the class and all base classes. return (RawStore) Proxy.newProxyInstance(RawStoreProxy.class.getClassLoader(), getAllInterfaces(baseClass), handler); }
CacheUpdateMasterWork(Configuration conf, boolean shouldRunPrewarm) { this.shouldRunPrewarm = shouldRunPrewarm; String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); try { rawStore = JavaUtils.getClass(rawStoreClassName, RawStore.class).newInstance(); rawStore.setConf(conf); } catch (InstantiationException | IllegalAccessException | MetaException e) { // MetaException here really means ClassNotFound (see the utility method). // So, if any of these happen, that means we can never succeed. throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); } }
private StorageSchemaReader getStorageSchemaReader() throws MetaException { if (storageSchemaReader == null) { String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL); Class<? extends StorageSchemaReader> readerClass = JavaUtils.getClass(className, StorageSchemaReader.class); try { storageSchemaReader = readerClass.newInstance(); } catch (InstantiationException|IllegalAccessException e) { LOG.error("Unable to instantiate class " + className, e); throw new MetaException(e.getMessage()); } } return storageSchemaReader; }
/** * Get an instance of the TxnStore that is appropriate for this store * @param conf configuration * @return txn store */ public static TxnStore getTxnStore(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.TXN_STORE_IMPL); try { TxnStore handler = JavaUtils.getClass(className, TxnStore.class).newInstance(); handler.setConf(conf); return handler; } catch (Exception e) { LOG.error("Unable to instantiate raw store directly in fastpath mode", e); throw new RuntimeException(e); } }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ private static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance(clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
private void setConfInternal(Configuration conf) { if (MetastoreConf.getBoolVar(conf, ConfVars.METASTORE_CACHE_CAN_USE_EVENT)) { canUseEvents = true; } else { canUseEvents = false; } LOG.info("canUseEvents is set to " + canUseEvents + " in cached Store"); String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); if (rawStore == null) { try { rawStore = (JavaUtils.getClass(rawStoreClassName, RawStore.class)).newInstance(); } catch (Exception e) { throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); } } rawStore.setConf(conf); Configuration oldConf = this.conf; this.conf = conf; this.areTxnStatsSupported = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_TXN_STATS_ENABLED); if (expressionProxy != null && conf != oldConf) { LOG.warn("Unexpected setConf when we were already configured"); } else { expressionProxy = PartFilterExprUtil.createExpressionProxy(conf); } }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ public static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { @SuppressWarnings("unchecked") Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance( clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { if (e.getMessage().matches(".* class not found")) { // TODO MS-SPLIT For now if we cannot load the default PartitionExpressionForMetastore // class (since it's from ql) load the DefaultPartitionExpressionProxy, which just throws // UnsupportedOperationExceptions. This allows existing Hive instances to work but also // allows us to instantiate the metastore stand alone for testing. Not sure if this is // the best long term solution. return new DefaultPartitionExpressionProxy(); } LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
public static ExpressionTree makeExpressionTree(PartitionExpressionProxy expressionProxy, byte[] expr, String defaultPartitionName) throws MetaException { // We will try pushdown first, so make the filter. This will also validate the expression, // if serialization fails we will throw incompatible metastore error to the client. String filter = null; try { filter = expressionProxy.convertExprToFilter(expr, defaultPartitionName); } catch (MetaException ex) { // TODO MS-SPLIT - for now we have construct this by reflection because IMetaStoreClient // can't be // moved until after HiveMetaStore is moved, which can't be moved until this is moved. Class<? extends MetaException> exClass = JavaUtils.getClass( "org.apache.hadoop.hive.metastore.IMetaStoreClient$IncompatibleMetastoreException", MetaException.class); throw JavaUtils.newInstance(exClass, new Class<?>[]{String.class}, new Object[]{ex.getMessage()}); } // Make a tree out of the filter. // TODO: this is all pretty ugly. The only reason we need all these transformations // is to maintain support for simple filters for HCat users that query metastore. // If forcing everyone to use thick client is out of the question, maybe we could // parse the filter into standard hive expressions and not all this separate tree // Filter.g stuff. That way this method and ...ByFilter would just be merged. return PartFilterExprUtil.makeExpressionTree(filter); }
private static void startRemoteOnlyTasks(Configuration conf) throws Exception { if(!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_INITIATOR_ON)) { return; } ThreadPool.initialize(conf); Collection<String> taskNames = MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_REMOTE_ONLY); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS); ThreadPool.getPool().scheduleAtFixedRate(task, freq, freq, TimeUnit.MILLISECONDS); } }
alterHandler = ReflectionUtils.newInstance(JavaUtils.getClass( alterHandlerName, AlterHandler.class), conf); wh = new Warehouse(conf); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS);
CacheUpdateMasterWork(Configuration conf, boolean shouldRunPrewarm) { this.shouldRunPrewarm = shouldRunPrewarm; String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); try { rawStore = JavaUtils.getClass(rawStoreClassName, RawStore.class).newInstance(); rawStore.setConf(conf); } catch (InstantiationException | IllegalAccessException | MetaException e) { // MetaException here really means ClassNotFound (see the utility method). // So, if any of these happen, that means we can never succeed. throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); } }
public static RawStore getProxy(Configuration hiveConf, Configuration conf, String rawStoreClassName, int id) throws MetaException { Class<? extends RawStore> baseClass = JavaUtils.getClass(rawStoreClassName, RawStore.class); RawStoreProxy handler = new RawStoreProxy(hiveConf, conf, baseClass, id); // Look for interfaces on both the class and all base classes. return (RawStore) Proxy.newProxyInstance(RawStoreProxy.class.getClassLoader(), getAllInterfaces(baseClass), handler); }
private StorageSchemaReader getStorageSchemaReader() throws MetaException { if (storageSchemaReader == null) { String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL); Class<? extends StorageSchemaReader> readerClass = JavaUtils.getClass(className, StorageSchemaReader.class); try { storageSchemaReader = readerClass.newInstance(); } catch (InstantiationException|IllegalAccessException e) { LOG.error("Unable to instantiate class " + className, e); throw new MetaException(e.getMessage()); } } return storageSchemaReader; }
/** * Get an instance of the TxnStore that is appropriate for this store * @param conf configuration * @return txn store */ public static TxnStore getTxnStore(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.TXN_STORE_IMPL); try { TxnStore handler = JavaUtils.getClass(className, TxnStore.class).newInstance(); handler.setConf(conf); return handler; } catch (Exception e) { LOG.error("Unable to instantiate raw store directly in fastpath mode", e); throw new RuntimeException(e); } }
private void setConfInternal(Configuration conf) { String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); if (rawStore == null) { try { rawStore = (JavaUtils.getClass(rawStoreClassName, RawStore.class)).newInstance(); } catch (Exception e) { throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); } } rawStore.setConf(conf); Configuration oldConf = this.conf; this.conf = conf; if (expressionProxy != null && conf != oldConf) { LOG.warn("Unexpected setConf when we were already configured"); } else { expressionProxy = PartFilterExprUtil.createExpressionProxy(conf); } }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ private static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance(clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ public static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { @SuppressWarnings("unchecked") Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance( clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { if (e.getMessage().matches(".* class not found")) { // TODO MS-SPLIT For now if we cannot load the default PartitionExpressionForMetastore // class (since it's from ql) load the DefaultPartitionExpressionProxy, which just throws // UnsupportedOperationExceptions. This allows existing Hive instances to work but also // allows us to instantiate the metastore stand alone for testing. Not sure if this is // the best long term solution. return new DefaultPartitionExpressionProxy(); } LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
public static ExpressionTree makeExpressionTree(PartitionExpressionProxy expressionProxy, byte[] expr) throws MetaException { // We will try pushdown first, so make the filter. This will also validate the expression, // if serialization fails we will throw incompatible metastore error to the client. String filter = null; try { filter = expressionProxy.convertExprToFilter(expr); } catch (MetaException ex) { // TODO MS-SPLIT - for now we have construct this by reflection because IMetaStoreClient // can't be // moved until after HiveMetaStore is moved, which can't be moved until this is moved. Class<? extends MetaException> exClass = JavaUtils.getClass( "org.apache.hadoop.hive.metastore.IMetaStoreClient$IncompatibleMetastoreException", MetaException.class); throw JavaUtils.newInstance(exClass, new Class<?>[]{String.class}, new Object[]{ex.getMessage()}); } // Make a tree out of the filter. // TODO: this is all pretty ugly. The only reason we need all these transformations // is to maintain support for simple filters for HCat users that query metastore. // If forcing everyone to use thick client is out of the question, maybe we could // parse the filter into standard hive expressions and not all this separate tree // Filter.g stuff. That way this method and ...ByFilter would just be merged. return PartFilterExprUtil.makeExpressionTree(filter); }
private static void startRemoteOnlyTasks(Configuration conf) throws Exception { if(!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_INITIATOR_ON)) { return; } ThreadPool.initialize(conf); Collection<String> taskNames = MetastoreConf.getStringCollection(conf, ConfVars.TASK_THREADS_REMOTE_ONLY); for (String taskName : taskNames) { MetastoreTaskThread task = JavaUtils.newInstance(JavaUtils.getClass(taskName, MetastoreTaskThread.class)); task.setConf(conf); long freq = task.runFrequency(TimeUnit.MILLISECONDS); ThreadPool.getPool().scheduleAtFixedRate(task, freq, freq, TimeUnit.MILLISECONDS); } }