@SuppressWarnings(value = "unchecked") public static <T> Class<? extends T> getClass(String className, Class<T> clazz) throws MetaException { try { return (Class<? extends T>) Class.forName(className, true, getClassLoader()); } catch (ClassNotFoundException e) { throw new MetaException(className + " class not found"); } }
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ private static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance(clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
private static void shouldNeverHappen(long txnid, long extLockId, long intLockId) { throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid) + " " + JavaUtils.lockIdToString(extLockId) + " " + intLockId); }
/** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf conf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(Configuration hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { @SuppressWarnings("unchecked") Class<? extends IMetaStoreClient> baseClass = JavaUtils.getClass(mscClassName, IMetaStoreClient.class); RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance( RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
protected RetryingMetaStoreClient(Configuration conf, Class<?>[] constructorArgTypes, Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { this.ugi = getUGI(); if (this.ugi == null) { LOG.warn("RetryingMetaStoreClient unable to determine current user UGI."); } this.retryLimit = MetastoreConf.getIntVar(conf, ConfVars.THRIFT_FAILURE_RETRIES); this.retryDelaySeconds = MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; this.connectionLifeTimeInMillis = MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_LIFETIME, TimeUnit.MILLISECONDS); this.lastConnectionTime = System.currentTimeMillis(); String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS); localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = JavaUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs); LOG.info("RetryingMetaStoreClient proxy=" + msClientClass + " ugi=" + this.ugi + " retries=" + this.retryLimit + " delay=" + this.retryDelaySeconds + " lifetime=" + this.connectionLifeTimeInMillis); }
private static void shouldNeverHappen(long txnid) { throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid)); } private static void shouldNeverHappen(long txnid, long extLockId, long intLockId) {
/** * Heartbeats on the lock table. This commits, so do not enter it with any state. * Should not be called on a lock that belongs to transaction. */ private void heartbeatLock(Connection dbConn, long extLockId) throws NoSuchLockException, SQLException, MetaException { // If the lock id is 0, then there are no locks in this heartbeat if (extLockId == 0) return; Statement stmt = null; try { stmt = dbConn.createStatement(); long now = getDbTime(dbConn); String s = "update HIVE_LOCKS set hl_last_heartbeat = " + now + " where hl_lock_ext_id = " + extLockId; LOG.debug("Going to execute update <" + s + ">"); int rc = stmt.executeUpdate(s); if (rc < 1) { LOG.debug("Going to rollback"); dbConn.rollback(); throw new NoSuchLockException("No such lock: " + JavaUtils.lockIdToString(extLockId)); } LOG.debug("Going to commit"); dbConn.commit(); } finally { closeStmt(stmt); } }
public static RawStore getProxy(Configuration hiveConf, Configuration conf, String rawStoreClassName, int id) throws MetaException { Class<? extends RawStore> baseClass = JavaUtils.getClass(rawStoreClassName, RawStore.class); RawStoreProxy handler = new RawStoreProxy(hiveConf, conf, baseClass, id); // Look for interfaces on both the class and all base classes. return (RawStore) Proxy.newProxyInstance(RawStoreProxy.class.getClassLoader(), getAllInterfaces(baseClass), handler); }
/** * Used to raise an informative error when the caller expected a txn in a particular TxnStatus * but found it in some other status */ private static void raiseTxnUnexpectedState(TxnStatus actualStatus, long txnid) throws NoSuchTxnException, TxnAbortedException { switch (actualStatus) { case ABORTED: throw new TxnAbortedException("Transaction " + JavaUtils.txnIdToString(txnid) + " already aborted"); case COMMITTED: throw new NoSuchTxnException("Transaction " + JavaUtils.txnIdToString(txnid) + " is already committed."); case UNKNOWN: throw new NoSuchTxnException("No such transaction " + JavaUtils.txnIdToString(txnid)); case OPEN: throw new NoSuchTxnException(JavaUtils.txnIdToString(txnid) + " is " + TxnStatus.OPEN); default: throw new IllegalArgumentException("Unknown TxnStatus " + actualStatus); } } /**
private List<LockInfo> getLockInfoFromLockId(Connection dbConn, long extLockId) throws NoSuchLockException, MetaException, SQLException { Statement stmt = null; try { stmt = dbConn.createStatement(); String s = "select hl_lock_ext_id, hl_lock_int_id, hl_db, hl_table, " + "hl_partition, hl_lock_state, hl_lock_type, hl_txnid from HIVE_LOCKS where " + "hl_lock_ext_id = " + extLockId; LOG.debug("Going to execute query <" + s + ">"); ResultSet rs = stmt.executeQuery(s); boolean sawAtLeastOne = false; List<LockInfo> ourLockInfo = new ArrayList<>(); while (rs.next()) { ourLockInfo.add(new LockInfo(rs)); sawAtLeastOne = true; } if (!sawAtLeastOne) { throw new MetaException("This should never happen! We already " + "checked the lock(" + JavaUtils.lockIdToString(extLockId) + ") existed but now we can't find it!"); } return ourLockInfo; } finally { closeStmt(stmt); } }
public static ExpressionTree makeExpressionTree(PartitionExpressionProxy expressionProxy, byte[] expr, String defaultPartitionName) throws MetaException { // We will try pushdown first, so make the filter. This will also validate the expression, // if serialization fails we will throw incompatible metastore error to the client. String filter = null; try { filter = expressionProxy.convertExprToFilter(expr, defaultPartitionName); } catch (MetaException ex) { // TODO MS-SPLIT - for now we have construct this by reflection because IMetaStoreClient // can't be // moved until after HiveMetaStore is moved, which can't be moved until this is moved. Class<? extends MetaException> exClass = JavaUtils.getClass( "org.apache.hadoop.hive.metastore.IMetaStoreClient$IncompatibleMetastoreException", MetaException.class); throw JavaUtils.newInstance(exClass, new Class<?>[]{String.class}, new Object[]{ex.getMessage()}); } // Make a tree out of the filter. // TODO: this is all pretty ugly. The only reason we need all these transformations // is to maintain support for simple filters for HCat users that query metastore. // If forcing everyone to use thick client is out of the question, maybe we could // parse the filter into standard hive expressions and not all this separate tree // Filter.g stuff. That way this method and ...ByFilter would just be merged. return PartFilterExprUtil.makeExpressionTree(filter); }
private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException { String errorMsg = "Unable to load filter hook at HMS server. "; String filterHookClassName = MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK); Preconditions.checkState(!isBlank(filterHookClassName)); try { return (MetaStoreFilterHook)Class.forName( filterHookClassName.trim(), true, JavaUtils.getClassLoader()).getConstructor( Configuration.class).newInstance(conf); } catch (Exception e) { LOG.error(errorMsg, e); throw new IllegalStateException(errorMsg + e.getMessage(), e); } }
CacheUpdateMasterWork(Configuration conf, boolean shouldRunPrewarm) { this.shouldRunPrewarm = shouldRunPrewarm; String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); try { rawStore = JavaUtils.getClass(rawStoreClassName, RawStore.class).newInstance(); rawStore.setConf(conf); } catch (InstantiationException | IllegalAccessException | MetaException e) { // MetaException here really means ClassNotFound (see the utility method). // So, if any of these happen, that means we can never succeed. throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); } }
@Override public String toString() { return JavaUtils.lockIdToString(extLockId) + " intLockId:" + intLockId + " " + JavaUtils.txnIdToString(txnId) + " db:" + db + " table:" + table + " partition:" + partition + " state:" + (state == null ? "null" : state.toString()) + " type:" + (type == null ? "null" : type.toString()); } private boolean isDbLock() {
private LockInfo getTxnIdFromLockId(Connection dbConn, long extLockId) throws NoSuchLockException, MetaException, SQLException { Statement stmt = null; ResultSet rs = null; try { stmt = dbConn.createStatement(); String s = "select hl_lock_ext_id, hl_lock_int_id, hl_db, hl_table, " + "hl_partition, hl_lock_state, hl_lock_type, hl_txnid from HIVE_LOCKS where " + "hl_lock_ext_id = " + extLockId; LOG.debug("Going to execute query <" + s + ">"); rs = stmt.executeQuery(s); if (!rs.next()) { return null; } LockInfo info = new LockInfo(rs); LOG.debug("getTxnIdFromLockId(" + extLockId + ") Return " + JavaUtils.txnIdToString(info.txnId)); return info; } finally { close(rs); closeStmt(stmt); } }
JavaUtils.lockIdToString(rqst.getLockid()) + ")"); checkRetryable(dbConn, e, "unlock(" + rqst + ")"); throw new MetaException("Unable to update transaction database " + JavaUtils.lockIdToString(extLockId) + " " + StringUtils.stringifyException(e)); } finally { closeStmt(stmt);
/** * Creates the proxy used to evaluate expressions. This is here to prevent circular * dependency - ql -> metastore client <-> metastore server -> ql. If server and * client are split, this can be removed. * @param conf Configuration. * @return The partition expression proxy. */ public static PartitionExpressionProxy createExpressionProxy(Configuration conf) { String className = MetastoreConf.getVar(conf, ConfVars.EXPRESSION_PROXY_CLASS); try { @SuppressWarnings("unchecked") Class<? extends PartitionExpressionProxy> clazz = JavaUtils.getClass(className, PartitionExpressionProxy.class); return JavaUtils.newInstance( clazz, new Class<?>[0], new Object[0]); } catch (MetaException e) { if (e.getMessage().matches(".* class not found")) { // TODO MS-SPLIT For now if we cannot load the default PartitionExpressionForMetastore // class (since it's from ql) load the DefaultPartitionExpressionProxy, which just throws // UnsupportedOperationExceptions. This allows existing Hive instances to work but also // allows us to instantiate the metastore stand alone for testing. Not sure if this is // the best long term solution. return new DefaultPartitionExpressionProxy(); } LOG.error("Error loading PartitionExpressionProxy", e); throw new RuntimeException("Error loading PartitionExpressionProxy: " + e.getMessage()); } }
synchronized private URIResolverHook loadUriResolverHook() throws IllegalStateException { String uriResolverClassName = MetastoreConf.getAsString(conf, ConfVars.URI_RESOLVER); if (uriResolverClassName.equals("")) { return null; } else { LOG.info("Loading uri resolver : " + uriResolverClassName); try { Class<?> uriResolverClass = Class.forName(uriResolverClassName, true, JavaUtils.getClassLoader()); return (URIResolverHook) ReflectionUtils.newInstance(uriResolverClass, null); } catch (Exception e) { LOG.error("Exception loading uri resolver hook" + e); return null; } } }
private StorageSchemaReader getStorageSchemaReader() throws MetaException { if (storageSchemaReader == null) { String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL); Class<? extends StorageSchemaReader> readerClass = JavaUtils.getClass(className, StorageSchemaReader.class); try { storageSchemaReader = readerClass.newInstance(); } catch (InstantiationException|IllegalAccessException e) { LOG.error("Unable to instantiate class " + className, e); throw new MetaException(e.getMessage()); } } return storageSchemaReader; }
dbConn.rollback(); StringBuilder sb = new StringBuilder("No such lock(s): (" + JavaUtils.lockIdToString(extLockId) + ":"); ResultSet rs = stmt.executeQuery("select hl_lock_int_id from HIVE_LOCKS where hl_lock_ext_id = " + extLockId); while(rs.next()) { sb.append(") ").append(JavaUtils.txnIdToString(txnId)); close(rs); throw new NoSuchLockException(sb.toString());