/** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf conf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(Configuration hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, String mscClassName) throws MetaException { return getProxy(hiveConf, constructorArgTypes, constructorArgs, null, mscClassName); }
/** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf conf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(Configuration hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { @SuppressWarnings("unchecked") Class<? extends IMetaStoreClient> baseClass = JavaUtils.getClass(mscClassName, IMetaStoreClient.class); RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance( RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
reloginExpiringKeytabUser(); if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) { if (this.ugi != null) { ret = method.invoke(base, args); long timeTaken = System.currentTimeMillis() - startTime; addMethodTime(method, timeTaken);
protected RetryingMetaStoreClient(Configuration conf, Class<?>[] constructorArgTypes, Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { this.ugi = getUGI(); if (this.ugi == null) { LOG.warn("RetryingMetaStoreClient unable to determine current user UGI."); } this.retryLimit = MetastoreConf.getIntVar(conf, ConfVars.THRIFT_FAILURE_RETRIES); this.retryDelaySeconds = MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; this.connectionLifeTimeInMillis = MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_LIFETIME, TimeUnit.MILLISECONDS); this.lastConnectionTime = System.currentTimeMillis(); String msUri = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS); localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = JavaUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs); LOG.info("RetryingMetaStoreClient proxy=" + msClientClass + " ugi=" + this.ugi + " retries=" + this.retryLimit + " delay=" + this.retryDelaySeconds + " lifetime=" + this.connectionLifeTimeInMillis); }
private void addMethodTime(Method method, long timeTaken) { String methodStr = getMethodString(method); while (true) { Long curTime = metaCallTimeMap.get(methodStr), newTime = timeTaken; if (curTime != null && metaCallTimeMap.replace(methodStr, curTime, newTime + curTime)) break; if (curTime == null && (null == metaCallTimeMap.putIfAbsent(methodStr, newTime))) break; } }
protected RetryingMetaStoreClient(HiveConf hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES); this.retryDelaySeconds = hiveConf.getTimeVar( HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; this.connectionLifeTimeInMillis = hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000; this.lastConnectionTime = System.currentTimeMillis(); String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS); localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = (IMetaStoreClient) MetaStoreUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs); }
private void addMethodTime(Method method, long timeTaken) { String methodStr = getMethodString(method); Long curTime = metaCallTimeMap.get(methodStr); if (curTime != null) { timeTaken += curTime; } metaCallTimeMap.put(methodStr, timeTaken); }
protected RetryingMetaStoreClient(HiveConf hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES); this.retryDelaySeconds = hiveConf.getTimeVar( HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; this.connectionLifeTimeInMillis = hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000; this.lastConnectionTime = System.currentTimeMillis(); String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS); localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = (IMetaStoreClient) MetaStoreUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs); }
public static IMetaStoreClient getNonCachedHiveMetastoreClient(HiveConf hiveConf) throws MetaException { return RetryingMetaStoreClient.getProxy(hiveConf, true); }
while (true) { try { reloginExpiringKeytabUser(); if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) { base.reconnect(); lastConnectionTime = System.currentTimeMillis(); ret = method.invoke(base, args); long timeTaken = System.currentTimeMillis() - startTime; addMethodTime(method, timeTaken);
public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader, String mscClassName) throws MetaException { Class<? extends IMetaStoreClient> baseClass = (Class<? extends IMetaStoreClient>) MetaStoreUtils.getClass(mscClassName); RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, hookLoader, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance(RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
private void addMethodTime(Method method, long timeTaken) { String methodStr = getMethodString(method); Long curTime = metaCallTimeMap.get(methodStr); if (curTime != null) { timeTaken += curTime; } metaCallTimeMap.put(methodStr, timeTaken); }
public static IMetaStoreClient getNonCachedHiveMetastoreClient(HiveConf hiveConf) throws MetaException { return RetryingMetaStoreClient.getProxy(hiveConf, true); }
while (true) { try { reloginExpiringKeytabUser(); if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) { base.reconnect(); lastConnectionTime = System.currentTimeMillis(); ret = method.invoke(base, args); long timeTaken = System.currentTimeMillis() - startTime; addMethodTime(method, timeTaken);
/** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { Class<? extends IMetaStoreClient> baseClass = (Class<? extends IMetaStoreClient>) MetaStoreUtils .getClass(mscClassName); RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance( RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
public static IMetaStoreClient getProxy(Configuration hiveConf, HiveMetaHookLoader hookLoader, ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName, boolean allowEmbedded) throws MetaException { return getProxy(hiveConf, new Class[] {Configuration.class, HiveMetaHookLoader.class, Boolean.class}, new Object[] {hiveConf, hookLoader, allowEmbedded}, metaCallTimeMap, mscClassName ); }
/** * This constructor is meant for Hive internal use only. * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes, Object[] constructorArgs, Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { Class<? extends IMetaStoreClient> baseClass = (Class<? extends IMetaStoreClient>) MetaStoreUtils .getClass(mscClassName); RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs, metaCallTimeMap, baseClass); return (IMetaStoreClient) Proxy.newProxyInstance( RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler); }
public static IMetaStoreClient getProxy( Configuration hiveConf, boolean allowEmbedded) throws MetaException { return getProxy(hiveConf, new Class[]{Configuration.class, HiveMetaHookLoader.class, Boolean.class}, new Object[]{hiveConf, null, allowEmbedded}, null, HiveMetaStoreClient.class.getName() ); }
private IMetaStoreClient createMetaStoreClient() throws MetaException { HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { @Override public HiveMetaHook getHook(Table tbl) throws MetaException { if (tbl == null) { return null; } try { HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(hiveConf, tbl.getParameters().get(META_TABLE_STORAGE)); return storageHandler == null ? null : storageHandler.getMetaHook(); } catch (HiveException e) { LOG.error(e.toString()); throw new MetaException("Failed to get storage handler: " + e); } } }; return RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName()); }
@Override public ICacheableMetaStoreClient call() throws MetaException { // This is called from HCat, so always allow embedded metastore (as was the default). return (ICacheableMetaStoreClient) RetryingMetaStoreClient.getProxy(cacheKey.getHiveConf(), new Class<?>[]{HiveConf.class, Integer.class, Boolean.class}, new Object[]{cacheKey.getHiveConf(), timeout, true}, CacheableHiveMetaStoreClient.class.getName()); } });