final NamespaceKey namespaceKey = new NamespaceKey(nameSpaceContainer.getFullPathList()); searchIndex.index(NamespaceServiceImpl.getKey(namespaceKey), searchEntity); indexCount.getAndIncrement(); });
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
private void addHome(String name) throws Exception { final HomeConfig home = new HomeConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setOwner(name); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.HOME) .setFullPathList(Arrays.asList(name)) .setHome(home); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey("@" + name)), container); }
private void newS3Source(KVStore<byte[], NameSpaceContainer> namespace, String path, S3PluginConfig s3PluginConfig) { final List<String> fullPathList = Arrays.asList(path); final SourceConfig config = new SourceConfig() .setId(new EntityId(UUID.randomUUID().toString())) .setName(path) .setConnectionConf(s3PluginConfig); namespace.put(NamespaceServiceImpl.getKey(new NamespaceKey(fullPathList)), new NameSpaceContainer() .setFullPathList(fullPathList) .setType(NameSpaceContainer.Type.SOURCE) .setSource(config)); } }
private void addSource(String name) throws Exception { final SourceConfig source = new SourceConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setName(name) .setType("test") .setCtime(100L) .setAccelerationRefreshPeriod(REFRESH_PERIOD_MS) .setAccelerationGracePeriod(GRACE_PERIOD_MS) .setMetadataPolicy(new MetadataPolicy().setDatasetDefinitionExpireAfterMs(TimeUnit.DAYS.toMillis(1))); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.SOURCE) .setFullPathList(Arrays.asList(name)) .setSource(source); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey(name)), container); }
private DatasetConfig addDataset(IndexedStore<byte[], NameSpaceContainer> namespace, IndexedStore<DatasetSplitId, DatasetSplit> splitsStore, String id, List<String> path, int splits) { DatasetConfig ds = new DatasetConfig() .setId(new EntityId(id)) .setName(last(path)) .setFullPathList(path) .setType(DatasetType.PHYSICAL_DATASET) .setReadDefinition(new ReadDefinition().setSplitVersion(42L)); namespace.put( NamespaceServiceImpl.getKey(new NamespaceKey(path)), new NameSpaceContainer().setType(NameSpaceContainer.Type.DATASET).setFullPathList(path).setDataset(ds)); for(int i = 0; i < splits; i++) { final String splitKey = Integer.toString(i); DatasetSplit split = new DatasetSplit() .setSplitVersion(42L) .setSplitKey(splitKey); // Generate an older dataset split id DatasetSplitId splitId = UnsafeDatasetSplitIdHelper.of(ds, splitKey); splitsStore.put(splitId, split); } return ds; }