@Override public NamespaceService get(String userName) { Preconditions.checkNotNull(userName, "requires userName"); // per method contract return new NamespaceServiceImpl(kvStoreProvider); } }
private static void deleteSplitOrphans(LocalKVStoreProvider provider) { System.out.print("Deleting split orphans... "); NamespaceServiceImpl service = new NamespaceServiceImpl(provider); // Since the system is offline, it is okay to delete split orphans to only keep current versions. System.out.println(String.format("Completed. Deleted %d orphans.", service.deleteSplitOrphans(DatasetSplitId.SplitOrphansRetentionPolicy.KEEP_CURRENT_VERSION_ONLY))); }
@Override public void upgrade(UpgradeContext context) { namespace = new NamespaceServiceImpl(context.getKVStoreProvider()); store = new ExternalReflectionStore(DirectProvider.wrap(context.getKVStoreProvider())); final Iterable<ExternalReflection> reflections = store.getExternalReflections(); StreamSupport.stream(reflections.spliterator(), false) .forEach(this::update); }
public static int pruneOrphans(LocalKVStoreProvider kvStoreProvider) { AtomicInteger results = new AtomicInteger(); final NamespaceServiceImpl namespaceService = new NamespaceServiceImpl(kvStoreProvider); // check tags for orphans final CollaborationTagStore tagsStore = new CollaborationTagStore(kvStoreProvider); StreamSupport.stream(tagsStore.find().spliterator(), false) .filter(entry -> { // if item is not in the namespace, delete the entry final String entityId = entry.getValue().getEntityId(); return namespaceService.findDatasetByUUID(entityId) == null; }) .forEach(entry -> { results.getAndIncrement(); tagsStore.delete(entry.getKey()); }); // check wikis for orphans final CollaborationWikiStore wikiStore = new CollaborationWikiStore(kvStoreProvider); StreamSupport.stream(wikiStore.find().spliterator(), false) .filter(entry -> { // if item is not in the namespace, delete the entry final String entityId = entry.getValue().getEntityId(); return namespaceService.findDatasetByUUID(entityId) == null; }) .forEach(entry -> { results.getAndIncrement(); wikiStore.delete(entry.getKey()); }); return results.get(); }
@Override public void upgrade(UpgradeContext context) { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { final NamespaceKey key = new DatasetPath(ImmutableList.of("sys", "materializations")).toNamespaceKey(); final DatasetConfig dataset = namespaceService.getDataset(key); namespaceService.deleteDataset(key, dataset.getTag()); } catch (NamespaceNotFoundException e) { // no metadata was found for sys.materializations // most likely the table was never queried // nothing more to do System.out.println(" 'sys.materializations' metadata not found...skipping"); } catch (NamespaceException e) { throw new RuntimeException("Failed to delete metadata for 'sys.materialization'", e); } } }
@Bootstrap @POST @Path("create") public void createTestDataset() throws Exception { refreshNow("cp"); // TODO: Clean up this mess SampleDataPopulator.addDefaultFirstUser(userService, new NamespaceServiceImpl(provider)); NamespaceService nsWithAuth = context.getNamespaceService(DEFAULT_USER_NAME); DatasetVersionMutator ds = newDS(nsWithAuth); // Closing sdp means remove the temporary directory @SuppressWarnings("resource") SampleDataPopulator sdp = new SampleDataPopulator(context, newSourceService(nsWithAuth, ds), ds, userService, nsWithAuth, DEFAULT_USER_NAME); sdp.populateInitialData(); }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { // Pre-1.5, the config object for internal sources was null. Deleting the internal sources without a config will // ensure that they get recreated on startup. if (sourceConfig.getConfig() == null) { System.out.printf(" deleting '%s'%n", sourceConfig.getName()); // following may throw an exception, we let it propagate to fail the upgrade namespaceService.deleteSource(sourceConfig.getKey(), sourceConfig.getTag()); } } }
private void migrateGoals(UpgradeContext context) { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); final Provider<KVStoreProvider> provider = DirectProvider.wrap(context.getKVStoreProvider()); final ReflectionGoalsStore reflectionGoalsStore = new ReflectionGoalsStore(provider); final ReflectionEntriesStore reflectionEntriesStore = new ReflectionEntriesStore(provider); final MaterializationStore materializationStore = new MaterializationStore(provider); for (ReflectionGoal reflectionGoal : reflectionGoalsStore.getAll()) { try { migrateGoal(namespaceService, reflectionGoalsStore, reflectionEntriesStore, materializationStore, reflectionGoal); } catch (Exception e) { System.out.println(String.format(" failed migrating reflection [%s] for dataset [%s]: %s", reflectionGoal.getId(), reflectionGoal.getDatasetId(), e)); } } }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sourceConfig); if (connectionConf instanceof S3PluginConfig) { S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf; if ((s3PluginConfig.credentialType == AWSAuthenticationType.ACCESS_KEY) && isNullOrEmpty(s3PluginConfig.accessKey)) { s3PluginConfig.credentialType = AWSAuthenticationType.NONE; sourceConfig.setConnectionConf(s3PluginConfig); namespaceService.addOrUpdateSource(sourceConfig.getKey(), sourceConfig); } } } }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue; } System.out.printf(" Handling Hive source %s%n", source.getName()); for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath); if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; } System.out.printf(" Clearing read definition of table %s%n", datasetPath.getSchemaPath()); datasetConfig.setReadDefinition(null); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig); } } } catch (NamespaceException e) { throw new RuntimeException("Hive121BasedInputSplits failed", e); } }
@Override public void upgrade(UpgradeContext context) { final NamespaceService namespace = new NamespaceServiceImpl(context.getKVStoreProvider()); for (SourceConfig sourceConfig : namespace.getSources()) { MetadataPolicy metadataPolicy = sourceConfig.getMetadataPolicy(); if (metadataPolicy != null) { long originalTTL = metadataPolicy.getDatasetDefinitionTtlMs(); metadataPolicy.setDatasetDefinitionRefreshAfterMs(originalTTL); if (originalTTL == ORIGINAL_DEFAULT_REFRESH_MILLIS) { metadataPolicy.setDatasetDefinitionExpireAfterMs(CatalogService.DEFAULT_EXPIRE_MILLIS); } else { metadataPolicy.setDatasetDefinitionExpireAfterMs(3 * originalTTL); } sourceConfig.setMetadataPolicy(metadataPolicy); System.out.println(" Updating source " + sourceConfig.getName()); try { namespace.addOrUpdateSource(new NamespaceKey(sourceConfig.getName()), sourceConfig); } catch (NamespaceException e) { Throwables.propagate(e); } } } }
@Test public void testDeleteEntityNotFound() throws Exception { try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) { kvstore.start(); final NamespaceServiceImpl ns = new NamespaceServiceImpl(kvstore); try { ns.deleteEntity(new NamespaceKey(Arrays.asList("does", "not", "exist")), NameSpaceContainer.Type.FOLDER, "123", true); fail("deleteEntity should have failed."); } catch(NamespaceNotFoundException e) { // Expected } } }
kvstore.start(); namespaceService = new NamespaceServiceImpl(kvstore); namespaceStore = kvstore.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class); splitsStore = kvstore.getStore(NamespaceServiceImpl.DatasetSplitCreator.class);
@Test public void testDatasetUnderFolderOrSpace() throws Exception { try ( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); addSpace(ns, "a"); addFolder(ns, "a.foo"); addFolder(ns, "a.foo.bar1"); addFolder(ns, "a.foo.bar2"); addFolder(ns, "a.foo.bar1.bar3"); addDS(ns, "a.ds0"); addDS(ns, "a.foo.ds1"); addDS(ns, "a.foo.ds2"); addDS(ns, "a.foo.bar1.ds3"); addDS(ns, "a.foo.bar2.ds4"); addDS(ns, "a.foo.bar1.bar3.ds5"); addDS(ns, "a.foo.bar1.bar3.ds6"); assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a"))))); assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo"))))); assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1"))))); assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar2"))))); assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("a", "foo", "bar1", "bar3"))))); } }
@Test public void testDatasetsUnderHome() throws Exception { try ( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); addHome(ns, "a"); addFolder(ns, "@a.foo"); addFolder(ns, "@a.foo.bar1"); addFolder(ns, "@a.foo.bar2"); addFolder(ns, "@a.foo.bar1.bar3"); addDS(ns, "@a.ds0"); addDS(ns, "@a.foo.ds1"); addDS(ns, "@a.foo.ds2"); addDS(ns, "@a.foo.bar1.ds3"); addDS(ns, "@a.foo.bar2.ds4"); addDS(ns, "@a.foo.bar1.bar3.ds5"); addDS(ns, "@a.foo.bar1.bar3.ds6"); assertEquals(7, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a"))))); assertEquals(6, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo"))))); assertEquals(1, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar2"))))); assertEquals(3, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1"))))); assertEquals(2, Iterables.size(ns.getAllDatasets(new NamespaceKey(asList("@a", "foo", "bar1", "bar3"))))); } }
@Test public void insertingDifferentEntityTypesAtSamePath() throws Exception { try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); addSpace(ns, "a"); thrown.expect(ConcurrentModificationException.class); addSource(ns, "a"); addFolder(ns, "a.foo"); // Try to add dataset with path "a.foo" try { addDS(ns, "a.foo"); fail("Expected the above call to fail"); } catch (UserException ex) { assertTrue(ex.getMessage().contains("There already exists an entity of type [FOLDER] at given path [a.foo]")); } // Try to add folder with path "a.foo". There already a folder at "a.foo" try { addFolder(ns, "a.foo"); fail("Expected the above call to fail"); } catch (UserException ex) { assertTrue(ex.getMessage().contains("There already exists an entity of type [FOLDER] at given path [a.foo]")); } } }
private void checkUpdateHelper(S3PluginConfig s3OldPluginConfig, AWSAuthenticationType authenticationType) throws Exception { try (final KVStoreProvider kvStoreProvider = new LocalKVStoreProvider(CLASSPATH_SCAN_RESULT, null, true, false)) { kvStoreProvider.start(); KVStore<byte[], NameSpaceContainer> namespace = kvStoreProvider.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class); newS3Source(namespace, "s3 plugin config", s3OldPluginConfig); // Performing upgrade UpdateS3CredentialType task = new UpdateS3CredentialType(); final LogicalPlanPersistence lpPersistence = new LogicalPlanPersistence(DEFAULT_SABOT_CONFIG, CLASSPATH_SCAN_RESULT); final ConnectionReader connectionReader = ConnectionReader.of(CLASSPATH_SCAN_RESULT, DEFAULT_SABOT_CONFIG); UpgradeContext context = new UpgradeContext(kvStoreProvider, lpPersistence, connectionReader); task.upgrade(context); final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); assertEquals(1, sources.size()); ConnectionConf<?, ?> connectionConf = context.getConnectionReader().getConnectionConf(sources.get(0)); assertTrue(connectionConf instanceof S3PluginConfig); S3PluginConfig s3PluginConfig = (S3PluginConfig) connectionConf; assertEquals(authenticationType, s3PluginConfig.credentialType); } }
@Test public void testDataSetSchema() throws Exception { try( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); Field field1 = new Field("a", true, new Int(32, true), null); Field child1 = new Field("c", true, Utf8.INSTANCE, null); Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1)); Schema schema = new Schema(ImmutableList.of(field1, field2)); FlatBufferBuilder builder = new FlatBufferBuilder(); schema.getSchema(builder); builder.finish(schema.getSchema(builder)); addSource(ns, "s"); addPhysicalDS(ns, "s.foo", builder.sizedByteArray()); ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray()); Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb)); assertEquals(schema, returnedSchema); } }
@Before public void setup() throws Exception { { SampleDataPopulator.addDefaultFirstUser(l(UserService.class), new NamespaceServiceImpl(l(KVStoreProvider.class))); final HDFSConf hdfsConfig = new HDFSConf(); hdfsConfig.hostname = host; hdfsConfig.port = port; SourceConfig source = new SourceConfig(); source.setName(SOURCE_NAME); source.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE); source.setConnectionConf(hdfsConfig); source.setId(new EntityId(SOURCE_ID)); source.setDescription(SOURCE_DESC); ((CatalogServiceImpl)l(CatalogService.class)).getSystemUserCatalog().createSource(source); } }
@Test public void testGetDatasetCount() throws Exception { try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); // create some nested datasets addSource(ns, "a"); for (int i = 0; i < 10; i++) { addDS(ns, "a.foo" + i); } for (int i = 0; i < 50; i++) { addDS(ns, "a.foo0.bar" + i); } for (int i = 0; i < 50; i++) { addDS(ns, "a.baz" + i); } // test count bound BoundedDatasetCount boundedDatasetCount = ns.getDatasetCount(new NamespaceKey("a"), 5000, 30); assertTrue(boundedDatasetCount.isCountBound()); assertEquals(boundedDatasetCount.getCount(), 30); // test time bound - the code checks every 50 children visited to see if the time bound has been hit and we give it 0 ms boundedDatasetCount = ns.getDatasetCount(new NamespaceKey("a"), 0, 1000); assertTrue(boundedDatasetCount.isTimeBound()); } }