/** * Get or create a hive client depending on whether it exits in cache or not * @param hiveConf The hive configuration * @return the client * @throws MetaException When HiveMetaStoreClient couldn't be created * @throws IOException */ public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf) throws MetaException, IOException { if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){ // If cache is disabled, don't use it. return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf); } // Singleton behaviour: create the cache instance if required. if (hiveClientCache == null) { synchronized (IMetaStoreClient.class) { if (hiveClientCache == null) { hiveClientCache = new HiveClientCache(hiveConf); } } } try { return hiveClientCache.get(hiveConf); } catch (LoginException e) { throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e); } }
@Test public void testCacheMiss() throws IOException, MetaException, LoginException { HiveClientCache cache = new HiveClientCache(1000); IMetaStoreClient client = cache.get(hiveConf); assertNotNull(client); // Set different uri as it is one of the criteria deciding whether to return the same client or not hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, " "); // URIs are checked for string equivalence, even spaces make them different IMetaStoreClient client2 = cache.get(hiveConf); assertNotNull(client2); assertNotSame(client, client2); }
/** * Get or create a hive client depending on whether it exits in cache or not * @param hiveConf The hive configuration * @return the client * @throws MetaException When HiveMetaStoreClient couldn't be created * @throws IOException */ public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf) throws MetaException, IOException { if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){ // If cache is disabled, don't use it. return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf); } // Singleton behaviour: create the cache instance if required. if (hiveClientCache == null) { synchronized (IMetaStoreClient.class) { if (hiveClientCache == null) { hiveClientCache = new HiveClientCache(hiveConf); } } } try { return hiveClientCache.get(hiveConf); } catch (LoginException e) { throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e); } }
/** * Check that a new client is returned for the same configuration after the expiry time. * Also verify that the expiry time configuration is honoured */ @Test public void testCacheExpiry() throws IOException, MetaException, LoginException, InterruptedException { HiveClientCache cache = new HiveClientCache(1); HiveClientCache.ICacheableMetaStoreClient client = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); assertNotNull(client); Thread.sleep(2500); HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); client.close(); assertTrue(client.isClosed()); // close() after *expiry time* and *a cache access* should have tore down the client assertNotNull(client2); assertNotSame(client, client2); }
/** * Get or create a hive client depending on whether it exits in cache or not * @param hiveConf The hive configuration * @return the client * @throws MetaException When HiveMetaStoreClient couldn't be created * @throws IOException */ public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf) throws MetaException, IOException { if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){ // If cache is disabled, don't use it. return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf); } // Singleton behaviour: create the cache instance if required. if (hiveClientCache == null) { synchronized (IMetaStoreClient.class) { if (hiveClientCache == null) { hiveClientCache = new HiveClientCache(hiveConf); } } } try { return hiveClientCache.get(hiveConf); } catch (LoginException e) { throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e); } }
@Test public void testCloseAllClients() throws IOException, MetaException, LoginException { final HiveClientCache cache = new HiveClientCache(1000); HiveClientCache.ICacheableMetaStoreClient client1 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, " "); // URIs are checked for string equivalence, even spaces make them different HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); cache.closeAllClientsQuietly(); assertTrue(client1.isClosed()); assertTrue(client2.isClosed()); }
/** * Get or create a hive client depending on whether it exits in cache or not * @param hiveConf The hive configuration * @return the client * @throws MetaException When HiveMetaStoreClient couldn't be created * @throws IOException */ public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf) throws MetaException, IOException { if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){ // If cache is disabled, don't use it. return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf); } // Singleton behaviour: create the cache instance if required. if (hiveClientCache == null) { synchronized (IMetaStoreClient.class) { if (hiveClientCache == null) { hiveClientCache = new HiveClientCache(hiveConf); } } } try { return hiveClientCache.get(hiveConf); } catch (LoginException e) { throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e); } }
@Test public void testCacheHit() throws IOException, MetaException, LoginException { HiveClientCache cache = new HiveClientCache(1000); HiveClientCache.ICacheableMetaStoreClient client = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); assertNotNull(client); client.close(); // close shouldn't matter // Setting a non important configuration should return the same client only hiveConf.setIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS, 10); HiveClientCache.ICacheableMetaStoreClient client2 = (HiveClientCache.ICacheableMetaStoreClient) cache.get(hiveConf); assertNotNull(client2); assertSame(client, client2); assertEquals(client.getUsers(), client2.getUsers()); client2.close(); }
/** * Get or create a hive client depending on whether it exits in cache or not * @param hiveConf The hive configuration * @return the client * @throws MetaException When HiveMetaStoreClient couldn't be created * @throws IOException */ public static IMetaStoreClient getHiveMetastoreClient(HiveConf hiveConf) throws MetaException, IOException { if (hiveConf.getBoolean(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, false)){ // If cache is disabled, don't use it. return HiveClientCache.getNonCachedHiveMetastoreClient(hiveConf); } // Singleton behaviour: create the cache instance if required. if (hiveClientCache == null) { synchronized (IMetaStoreClient.class) { if (hiveClientCache == null) { hiveClientCache = new HiveClientCache(hiveConf); } } } try { return hiveClientCache.get(hiveConf); } catch (LoginException e) { throw new IOException("Couldn't create hiveMetaStoreClient, Error getting UGI for user", e); } }
(HiveClientCache.CacheableHiveMetaStoreClient) cache.get(metaServer.getHiveConf());