/** Helper method which gets tables in a database. Retries once if the first call to fetch the metadata fails */ protected static List<String> getTableNamesHelper(final IMetaStoreClient mClient, final String dbName) throws TException { try { return mClient.getAllTables(dbName); } catch (MetaException e) { /* HiveMetaStoreClient is encapsulating both the MetaException/TExceptions inside MetaException. Since we don't have good way to differentiate, we will close older connection and retry once. This is only applicable for getAllTables and getAllDatabases method since other methods are properly throwing correct exceptions. */ logger.warn("Failure while attempting to get hive tables. Retries once.", e); try { mClient.close(); } catch (Exception ex) { logger.warn("Failure while attempting to close existing hive metastore connection. May leak connection.", ex); } mClient.reconnect(); return mClient.getAllTables(dbName); } }
@Test(expected = MetaException.class) public void testGetAllTablesInvalidData() throws Exception { client.getAllTables(null); }
/** * Get all tables in db with given table pattern. */ public Collection<DbAndTable> getTables() throws IOException { List<DbAndTable> tables = Lists.newArrayList(); try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Iterable<String> databases = Iterables.filter(client.get().getAllDatabases(), new Predicate<String>() { @Override public boolean apply(String db) { return HiveDatasetFinder.this.whitelistBlacklist.acceptDb(db); } }); for (final String db : databases) { Iterable<String> tableNames = Iterables.filter(client.get().getAllTables(db), new Predicate<String>() { @Override public boolean apply(String table) { return HiveDatasetFinder.this.whitelistBlacklist.acceptTable(db, table); } }); for (String tableName : tableNames) { tables.add(new DbAndTable(db, tableName)); } } } catch (Exception exc) { throw new IOException(exc); } return tables; }
@Test public void testGetAllTablesCaseInsensitive() throws Exception { // Check case insensitive search List<String> tables = client.getAllTables("dEFauLt"); Assert.assertEquals("Found tables size", 5, tables.size()); }
List<String> tables = hms.getAllTables(dbName); LOG.debug("found " + tables.size() + " tables in " + dbName); for(String tableName : tables) {
for (String dbName : getMSC().getAllDatabases()) { List<String> tables = getMSC().getAllTables(dbName);
@Test public void getAllTablesInBogusCatalog() throws TException { List<String> names = client.getAllTables("nosuch", testTables[0].getDbName()); Assert.assertTrue(names.isEmpty()); }
LOG.debug("processing " + dbName); for (String tblName : hiveClient.getAllTables(dbName)) { numTbl++; LOG.debug("processing " + dbName + "." + tblName);
/** * Get all tables for the specified database. * @param dbName * @return List of table names * @throws HiveException */ public List<Table> getAllTableObjects(String dbName) throws HiveException { try { return Lists.transform(getMSC().getTableObjectsByName(dbName, getMSC().getAllTables(dbName)), new hive.com.google.common.base.Function<org.apache.hadoop.hive.metastore.api.Table, Table>() { @Override public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { return new Table(table); } } ); } catch (Exception e) { throw new HiveException(e); } }
@Test public void testGetAllTables() throws Exception { List<String> tables = client.getAllTables(DEFAULT_DATABASE); Assert.assertEquals("All tables size", 5, tables.size()); for(Table table : testTables) { if (table.getDbName().equals(DEFAULT_DATABASE)) { Assert.assertTrue("Checking table names", tables.contains(table.getTableName())); } } // Drop one table, see what remains client.dropTable(testTables[1].getCatName(), testTables[1].getDbName(), testTables[1] .getTableName()); tables = client.getAllTables(DEFAULT_DATABASE); Assert.assertEquals("All tables size", 4, tables.size()); for(Table table : testTables) { if (table.getDbName().equals(DEFAULT_DATABASE) && !table.getTableName().equals(testTables[1].getTableName())) { Assert.assertTrue("Checking table names", tables.contains(table.getTableName())); } } // No such database tables = client.getAllTables("no_such_database"); Assert.assertEquals("All tables size", 0, tables.size()); }
private HiveMetastoreClientPool getTestPool(List<HiveDatasetFinder.DbAndTable> dbAndTables) throws Exception { SetMultimap<String, String> entities = HashMultimap.create(); for (HiveDatasetFinder.DbAndTable dbAndTable : dbAndTables) { entities.put(dbAndTable.getDb(), dbAndTable.getTable()); } HiveMetastoreClientPool pool = Mockito.mock(HiveMetastoreClientPool.class); IMetaStoreClient client = Mockito.mock(IMetaStoreClient.class); Mockito.when(client.getAllDatabases()).thenReturn(Lists.newArrayList(entities.keySet())); for (String db : entities.keySet()) { Mockito.doReturn(Lists.newArrayList(entities.get(db))).when(client).getAllTables(db); } for (HiveDatasetFinder.DbAndTable dbAndTable : dbAndTables) { Table table = new Table(); table.setDbName(dbAndTable.getDb()); table.setTableName(dbAndTable.getTable()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation("/tmp/test"); table.setSd(sd); Mockito.doReturn(table).when(client).getTable(dbAndTable.getDb(), dbAndTable.getTable()); } @SuppressWarnings("unchecked") AutoReturnableObject<IMetaStoreClient> aro = Mockito.mock(AutoReturnableObject.class); Mockito.when(aro.get()).thenReturn(client); Mockito.when(pool.getHiveRegProps()).thenReturn(null); Mockito.when(pool.getClient()).thenReturn(aro); return pool; }
for(String tableName : client.getAllTables(DEFAULT_DATABASE)) { client.dropTable(DEFAULT_DATABASE, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) { client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) { client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) { client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) { client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) { client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE_NAME)) { client.dropTable(DEFAULT_DATABASE_NAME, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE)) { client.dropTable(DEFAULT_DATABASE, tableName, true, true, true);
for(String tableName : client.getAllTables(DEFAULT_DATABASE)) { client.dropTable(DEFAULT_DATABASE, tableName, true, true, true);