@Override public NamespaceService get(String userName) { Preconditions.checkNotNull(userName, "requires userName"); // per method contract return new NamespaceServiceImpl(kvStoreProvider); } }
@Override public void deleteSpace(final NamespaceKey spacePath, String version) throws NamespaceException { deleteEntity(spacePath, SPACE, version, true); }
private static void deleteSplitOrphans(LocalKVStoreProvider provider) { System.out.print("Deleting split orphans... "); NamespaceServiceImpl service = new NamespaceServiceImpl(provider); // Since the system is offline, it is okay to delete split orphans to only keep current versions. System.out.println(String.format("Completed. Deleted %d orphans.", service.deleteSplitOrphans(DatasetSplitId.SplitOrphansRetentionPolicy.KEEP_CURRENT_VERSION_ONLY))); }
public static int pruneOrphans(LocalKVStoreProvider kvStoreProvider) { AtomicInteger results = new AtomicInteger(); final NamespaceServiceImpl namespaceService = new NamespaceServiceImpl(kvStoreProvider); // check tags for orphans final CollaborationTagStore tagsStore = new CollaborationTagStore(kvStoreProvider); StreamSupport.stream(tagsStore.find().spliterator(), false) .filter(entry -> { // if item is not in the namespace, delete the entry final String entityId = entry.getValue().getEntityId(); return namespaceService.findDatasetByUUID(entityId) == null; }) .forEach(entry -> { results.getAndIncrement(); tagsStore.delete(entry.getKey()); }); // check wikis for orphans final CollaborationWikiStore wikiStore = new CollaborationWikiStore(kvStoreProvider); StreamSupport.stream(wikiStore.find().spliterator(), false) .filter(entry -> { // if item is not in the namespace, delete the entry final String entityId = entry.getValue().getEntityId(); return namespaceService.findDatasetByUUID(entityId) == null; }) .forEach(entry -> { results.getAndIncrement(); wikiStore.delete(entry.getKey()); }); return results.get(); }
@Override public void deleteDataset(final NamespaceKey datasetPath, String version) throws NamespaceException { NameSpaceContainer container = deleteEntity(datasetPath, DATASET, version, true); if (container.getDataset().getType() == PHYSICAL_DATASET_SOURCE_FOLDER) { // create a folder so that any existing datasets under the folder are now visible addOrUpdateFolder(datasetPath, new FolderConfig() .setFullPathList(datasetPath.getPathComponents()) .setName(datasetPath.getName()) ); } }
@Test public void testDeleteEntityNotFound() throws Exception { try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) { kvstore.start(); final NamespaceServiceImpl ns = new NamespaceServiceImpl(kvstore); try { ns.deleteEntity(new NamespaceKey(Arrays.asList("does", "not", "exist")), NameSpaceContainer.Type.FOLDER, "123", true); fail("deleteEntity should have failed."); } catch(NamespaceNotFoundException e) { // Expected } } }
try (final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); Long lastSplitVersion = System.currentTimeMillis(); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); Long newSplitVersion = datasetConfig.getReadDefinition().getSplitVersion(); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); assertEquals(newSplitVersion, datasetConfig.getReadDefinition().getSplitVersion()); ns.addOrUpdateDataset(new NamespaceKey(datasetConfig.getFullPathList()), datasetConfig, splits); assertEquals(10, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); newSplitVersion = datasetConfig.getReadDefinition().getSplitVersion(); ns.addOrUpdateDataset(new NamespaceKey(datasetConfig.getFullPathList()), datasetConfig, splits); assertEquals(9, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); newSplitVersion = datasetConfig.getReadDefinition().getSplitVersion(); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); newSplitVersion = datasetConfig.getReadDefinition().getSplitVersion();
final NamespaceKey namespaceKey = new NamespaceKey(nameSpaceContainer.getFullPathList()); searchIndex.index(NamespaceServiceImpl.getKey(namespaceKey), searchEntity); indexCount.getAndIncrement(); });
!compareSplits(dataset, splits, splitsStore.find(DatasetSplitId.getSplitsRange(dataset)))) { addOrUpdateDataset(datasetPath, dataset, attributes); return; while (true) { try { addOrUpdateDataset(datasetPath, dataset, attributes); break; } catch (ConcurrentModificationException cme) { final DatasetConfig existingDatasetConfig = getDataset(datasetPath); if (existingDatasetConfig.getReadDefinition() != null && existingDatasetConfig.getReadDefinition().getSplitVersion() != null && deleteSplits(splitIds); break;
addOrUpdateDataset(datasetPath, datasetConfig, attributes); } catch (ConcurrentModificationException e) { logger.warn("Failure while updating dataset " + datasetPath, e);
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
@Override public void upgrade(UpgradeContext context) { namespace = new NamespaceServiceImpl(context.getKVStoreProvider()); store = new ExternalReflectionStore(DirectProvider.wrap(context.getKVStoreProvider())); final Iterable<ExternalReflection> reflections = store.getExternalReflections(); StreamSupport.stream(reflections.spliterator(), false) .forEach(this::update); }
private void addHome(String name) throws Exception { final HomeConfig home = new HomeConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setOwner(name); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.HOME) .setFullPathList(Arrays.asList(name)) .setHome(home); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey("@" + name)), container); }
@Override public void deleteSourceChildren(final NamespaceKey sourcePath, String version) throws NamespaceException { deleteEntity(sourcePath, SOURCE, version, false); }
@Override public void upgrade(UpgradeContext context) { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { final NamespaceKey key = new DatasetPath(ImmutableList.of("sys", "materializations")).toNamespaceKey(); final DatasetConfig dataset = namespaceService.getDataset(key); namespaceService.deleteDataset(key, dataset.getTag()); } catch (NamespaceNotFoundException e) { // no metadata was found for sys.materializations // most likely the table was never queried // nothing more to do System.out.println(" 'sys.materializations' metadata not found...skipping"); } catch (NamespaceException e) { throw new RuntimeException("Failed to delete metadata for 'sys.materialization'", e); } } }
private void newS3Source(KVStore<byte[], NameSpaceContainer> namespace, String path, S3PluginConfig s3PluginConfig) { final List<String> fullPathList = Arrays.asList(path); final SourceConfig config = new SourceConfig() .setId(new EntityId(UUID.randomUUID().toString())) .setName(path) .setConnectionConf(s3PluginConfig); namespace.put(NamespaceServiceImpl.getKey(new NamespaceKey(fullPathList)), new NameSpaceContainer() .setFullPathList(fullPathList) .setType(NameSpaceContainer.Type.SOURCE) .setSource(config)); } }
@Override public void deleteSource(final NamespaceKey sourcePath, String version) throws NamespaceException { deleteEntity(sourcePath, SOURCE, version, true); }
final NamespaceService namespace = new NamespaceServiceImpl(kvstore);
private void addSource(String name) throws Exception { final SourceConfig source = new SourceConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setName(name) .setType("test") .setCtime(100L) .setAccelerationRefreshPeriod(REFRESH_PERIOD_MS) .setAccelerationGracePeriod(GRACE_PERIOD_MS) .setMetadataPolicy(new MetadataPolicy().setDatasetDefinitionExpireAfterMs(TimeUnit.DAYS.toMillis(1))); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.SOURCE) .setFullPathList(Arrays.asList(name)) .setSource(source); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey(name)), container); }
@Override public void deleteFolder(final NamespaceKey folderPath, String version) throws NamespaceException { deleteEntity(folderPath, FOLDER, version, true); }