public static CachingHiveMetastore memoizeMetastore(ExtendedHiveMetastore delegate, long maximumSize) { return new CachingHiveMetastore( delegate, newDirectExecutorService(), OptionalLong.empty(), OptionalLong.empty(), maximumSize); }
@Override public Optional<Database> getDatabase(String databaseName) { return get(databaseCache, databaseName); }
@Override public Map<String, Optional<Partition>> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { Iterable<HivePartitionName> names = transform(partitionNames, name -> HivePartitionName.hivePartitionName(databaseName, tableName, name)); Map<HivePartitionName, Optional<Partition>> all = getAll(partitionCache, names); ImmutableMap.Builder<String, Optional<Partition>> partitionsByName = ImmutableMap.builder(); for (Entry<HivePartitionName, Optional<Partition>> entry : all.entrySet()) { partitionsByName.put(entry.getKey().getPartitionName().get(), entry.getValue()); } return partitionsByName.build(); }
@Test public void testGetAllDatabases() { assertEquals(mockClient.getAccessCount(), 0); assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); assertEquals(metastore.getAllDatabases(), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 2); }
@Test public void testGetAllTable() { assertEquals(mockClient.getAccessCount(), 0); assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); assertEquals(metastore.getAllTables(TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); }
@Test public void testGetTable() { assertEquals(mockClient.getAccessCount(), 0); assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); assertNotNull(metastore.getTable(TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); }
retry() .stopOn(InvalidOperationException.class, MetaException.class) .stopOnIllegalExceptions() .run("alterTable", stats.getAlterTable().wrap(() -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { Optional<Table> source = loadTable(new HiveTableName(databaseName, tableName)); if (!source.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); invalidateTable(databaseName, tableName); invalidateTable(table.getDbName(), table.getTableName());
@BeforeMethod public void setUp() throws Exception { mockClient = new MockHiveMetastoreClient(); MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient); ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s"))); metastore = new CachingHiveMetastore(mockHiveCluster, executor, new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES)); stats = ((CachingHiveMetastore) metastore).getStats(); }
@Override public void renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName) { try { delegate.renameTable(databaseName, tableName, newDatabaseName, newTableName); } finally { invalidateTable(databaseName, tableName); invalidateTable(newDatabaseName, newTableName); } }
@Override public void dropPartition(String databaseName, String tableName, List<String> parts) { try { retry() .stopOn(NoSuchObjectException.class, MetaException.class) .stopOnIllegalExceptions() .run("dropPartition", stats.getDropPartition().wrap(() -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { client.dropPartition(databaseName, tableName, parts, true); } return null; })); } catch (NoSuchObjectException e) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } throw Throwables.propagate(e); } finally { invalidatePartitionCache(databaseName, tableName); } }
@Override public void alterPartition(String databaseName, String tableName, PartitionWithStatistics partition) { try { delegate.alterPartition(databaseName, tableName, partition); } finally { invalidatePartitionCache(databaseName, tableName); } }
@Test public void testGetPartitionsByNames() { assertEquals(mockClient.getAccessCount(), 0); metastore.getTable(TEST_DATABASE, TEST_TABLE); assertEquals(mockClient.getAccessCount(), 1); // Select half of the available partitions and load them into the cache assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).size(), 1); assertEquals(mockClient.getAccessCount(), 2); // Now select all of the partitions assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2); // There should be one more access to fetch the remaining partition assertEquals(mockClient.getAccessCount(), 3); // Now if we fetch any or both of them, they should not hit the client assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1)).size(), 1); assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION2)).size(), 1); assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2); assertEquals(mockClient.getAccessCount(), 3); metastore.flushCache(); // Fetching both should only result in one batched access assertEquals(metastore.getPartitionsByNames(TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2)).size(), 2); assertEquals(mockClient.getAccessCount(), 4); }
@Override public void dropTable(String databaseName, String tableName) { try { retry() .stopOn(NoSuchObjectException.class) .stopOnIllegalExceptions() .run("dropTable", stats.getDropTable().wrap(() -> { try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { client.dropTable(databaseName, tableName, true); } return null; })); } catch (NoSuchObjectException e) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } throw Throwables.propagate(e); } finally { invalidateTable(databaseName, tableName); } }
@Override public Optional<Database> getDatabase(String databaseName) { return super.getDatabase(databaseName) .map(database -> Database.builder(database) .setLocation(Optional.of(basePath.toString())) .build()); }
@Override public void createTable(Table table, PrincipalPrivileges privileges) { // hack to work around the metastore not being configured for S3 or other FS Table.Builder tableBuilder = Table.builder(table); tableBuilder.getStorageBuilder().setLocation("/"); super.createTable(tableBuilder.build(), privileges); }
@Test public void testGetPartitionNames() { ImmutableList<String> expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); assertEquals(metastore.getPartitionNames(TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); }
@Test public void testGetPartitionNamesByParts() { ImmutableList<String> parts = ImmutableList.of(); ImmutableList<String> expectedPartitions = ImmutableList.of(TEST_PARTITION1, TEST_PARTITION2); assertEquals(mockClient.getAccessCount(), 0); assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); metastore.flushCache(); assertEquals(metastore.getPartitionNamesByParts(TEST_DATABASE, TEST_TABLE, parts).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); }
@Test public void testNoCacheExceptions() { // Throw exceptions on usage mockClient.setThrowException(true); try { metastore.getAllDatabases(); } catch (RuntimeException ignored) { } assertEquals(mockClient.getAccessCount(), 1); // Second try should hit the client again try { metastore.getAllDatabases(); } catch (RuntimeException ignored) { } assertEquals(mockClient.getAccessCount(), 2); }
@Override public void dropTable(String databaseName, String tableName, boolean deleteData) { try { delegate.dropTable(databaseName, tableName, deleteData); } finally { invalidateTable(databaseName, tableName); } }
retry() .stopOn(NoSuchObjectException.class, MetaException.class) .stopOnIllegalExceptions() invalidatePartitionCache(databaseName, tableName);