@Override public void upgrade(UpgradeContext context) { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { final NamespaceKey key = new DatasetPath(ImmutableList.of("sys", "materializations")).toNamespaceKey(); final DatasetConfig dataset = namespaceService.getDataset(key); namespaceService.deleteDataset(key, dataset.getTag()); } catch (NamespaceNotFoundException e) { // no metadata was found for sys.materializations // most likely the table was never queried // nothing more to do System.out.println(" 'sys.materializations' metadata not found...skipping"); } catch (NamespaceException e) { throw new RuntimeException("Failed to delete metadata for 'sys.materialization'", e); } } }
public static String getVersion(NamespaceKey namespaceKey, NamespaceService namespaceService) throws NamespaceException { NameSpaceContainer container = namespaceService.getEntities(Arrays.asList(namespaceKey)).get(0); switch (container.getType()) { case SOURCE: return container.getSource().getTag(); case SPACE: return container.getSpace().getTag(); case HOME: return container.getHome().getTag(); case FOLDER: return container.getFolder().getTag(); case DATASET: return container.getDataset().getTag(); default: throw new RuntimeException("Invalid container type"); } } }
private void deleteDataset(List<String> path) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); try { DatasetConfig oldDataset = namespaceService.getDataset(key); namespaceService.deleteDataset(key, oldDataset.getTag()); } catch (NamespaceNotFoundException e) { // Ignore } } }
private void loadIfNecessary() { if(datasetPointer != null){ return; } SplitsPointer splitsPointer; if(datasetConfig.getReadDefinition() != null) { splitsPointer = DatasetSplitsPointer.of(ns, datasetConfig); } else { try{ final DatasetConfig newDatasetConfig = datasetAccessor.getDataset(); newDatasetConfig.setId(datasetConfig.getId()); newDatasetConfig.setTag(datasetConfig.getTag()); List<DatasetSplit> splits = datasetAccessor.getSplits(); ns.addOrUpdateDataset(getName(), newDatasetConfig, splits); datasetConfig = newDatasetConfig; splitsPointer = MaterializedSplitsPointer.of(splits, splits.size()); } catch (Exception e) { throw new RuntimeException(e); } } datasetPointer = new TableMetadataImpl(pluginId, datasetConfig, user, splitsPointer); }
public static CatalogItem fromDatasetConfig(DatasetConfig datasetConfig, CollaborationTag tags) { DatasetSubType datasetType = DatasetSubType.PROMOTED; if (datasetConfig.getType() == DatasetType.PHYSICAL_DATASET) { datasetType = DatasetSubType.DIRECT; } else if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { datasetType = DatasetSubType.VIRTUAL; } return new Builder() .setId(datasetConfig.getId().getId()) .setPath(Lists.newArrayList(datasetConfig.getFullPathList())) .setTag(String.valueOf(datasetConfig.getTag())) .setType(CatalogItemType.DATASET) .setDatasetType(datasetType) .setTags(tags) .build(); }
public VirtualDatasetUI get(DatasetPath path, DatasetVersion version) throws DatasetNotFoundException, NamespaceException { try { final DatasetConfig datasetConfig = namespaceService.getDataset(path.toNamespaceKey()); final VirtualDatasetVersion virtualDatasetVersion = datasetVersions.get(new VersionDatasetKey(path, version)); if (virtualDatasetVersion == null) { throw new DatasetNotFoundException(path, format("Missing version %s.", version.toString())); } final VirtualDatasetUI virtualDatasetUI = toVirtualDatasetUI(virtualDatasetVersion) .setId(datasetConfig.getId().getId()) .setSavedTag(datasetConfig.getTag()); return virtualDatasetUI; } catch (NamespaceNotFoundException e) { throw new DatasetNotFoundException(path, format("Some path not found while looking for dataset %s, version %s.", path.toPathString(), version.toString()), e); } }
@Override public String getTag(NameSpaceContainer value) { switch (value.getType()) { case DATASET: return value.getDataset().getTag(); case FOLDER: return value.getFolder().getTag(); case HOME: return value.getHome().getTag(); case SOURCE: return value.getSource().getTag(); case SPACE: return value.getSpace().getTag(); default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
.setSavedTag(datasetConfig.getTag()); } catch (final NamespaceException ex) { logger.debug("dataset error for {}", path, ex);
public static PhysicalDatasetConfig toPhysicalDatasetConfig(DatasetConfig datasetConfig) { checkNotNull(datasetConfig.getPhysicalDataset()); final com.dremio.service.namespace.dataset.proto.PhysicalDataset physicalDataset = datasetConfig.getPhysicalDataset(); final PhysicalDatasetConfig physicalDatasetConfig = new PhysicalDatasetConfig(); physicalDatasetConfig.setFormatSettings(physicalDataset.getFormatSettings()); physicalDatasetConfig.setFullPathList(datasetConfig.getFullPathList()); physicalDatasetConfig.setType(datasetConfig.getType()); physicalDatasetConfig.setName(datasetConfig.getName()); physicalDatasetConfig.setTag(datasetConfig.getTag()); physicalDatasetConfig.setId(datasetConfig.getId().getId()); return physicalDatasetConfig; }
protected void doTraverseAndDeleteChildren(final NameSpaceContainer child) throws NamespaceException { final NamespaceInternalKey childKey = new NamespaceInternalKey(new NamespaceKey(child.getFullPathList()), keyNormalization); traverseAndDeleteChildren(childKey, child); switch (child.getType()) { case FOLDER: namespace.delete(childKey.getKey(), child.getFolder().getTag()); break; case DATASET: namespace.delete(childKey.getKey(), child.getDataset().getTag()); break; default: // Only leaf level or intermediate namespace container types are expected here. throw new RuntimeException("Unexpected namespace container type: " + child.getType()); } }
public void deleteDataset(DatasetPath ds) { NamespaceKey key = ds.toNamespaceKey(); DatasetConfig config; try { config = getNamespaceService().getDataset(key); if (config != null) { getNamespaceService().deleteDataset(key, config.getTag()); } }catch (NamespaceException e) { } }
public static FileConfig toFileConfig(DatasetConfig datasetConfig) { final FileConfig fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); fileConfig.setCtime(datasetConfig.getCreatedAt()); fileConfig.setTag(datasetConfig.getTag()); fileConfig.setOwner(datasetConfig.getOwner()); fileConfig.setFullPathList(datasetConfig.getFullPathList()); fileConfig.setName(datasetConfig.getName()); return fileConfig; }
public void put(VirtualDatasetUI ds, NamespaceAttribute... attributes) throws DatasetNotFoundException, NamespaceException { DatasetPath path = new DatasetPath(ds.getFullPathList()); validatePath(path); validate(path, ds); DatasetConfig datasetConfig = toVirtualDatasetVersion(ds).getDataset(); namespaceService.addOrUpdateDataset(path.toNamespaceKey(), datasetConfig, attributes); ds.setId(datasetConfig.getId().getId()); ds.setSavedTag(datasetConfig.getTag()); // Update this version of dataset with new occ version of dataset config from namespace. putVersion(ds); }
@Test public void testDeleteDataset() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.VIRTUAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, "sql", null, null, null ); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setId(new EntityId(dataset.getId())); datasetConfig.setType(VIRTUAL_DATASET); datasetConfig.setFullPathList(dataset.getPath()); datasetConfig.setTag(dataset.getTag()); namespaceContainer.setDataset(datasetConfig); when(namespaceService.getEntityById(dataset.getId())).thenReturn(namespaceContainer); DremioTable dremioTable = mock(DremioTable.class); when(dremioTable.getDatasetConfig()).thenReturn(datasetConfig); when(catalog.getTable(any(String.class))).thenReturn(dremioTable); catalogServiceHelper.deleteCatalogItem(dataset.getId(), "1"); verify(namespaceService, times(1)).deleteDataset(new NamespaceKey(dataset.getPath()), datasetConfig.getTag()); }
/** * @param path * the id of the dataset * @return the latest saved version of the corresponding dataset * @throws DatasetNotFoundException * if the path was not found */ public VirtualDatasetUI get(DatasetPath path) throws DatasetNotFoundException, NamespaceException { try { final DatasetConfig datasetConfig = namespaceService.getDataset(path.toNamespaceKey()); final VirtualDatasetVersion virtualDatasetVersion = datasetVersions.get(new VersionDatasetKey(path, datasetConfig.getVirtualDataset().getVersion())); if (virtualDatasetVersion == null) { throw new DatasetNotFoundException(path, format("Missing version %s.", datasetConfig.getVirtualDataset().getVersion().toString())); } final VirtualDatasetUI virtualDatasetUI = toVirtualDatasetUI(virtualDatasetVersion) .setId(datasetConfig.getId().getId()) .setSavedTag(datasetConfig.getTag()); return virtualDatasetUI; } catch (NamespaceNotFoundException nsnf) { throw new DatasetNotFoundException(path, nsnf); } }
/** * Carry over few properties from old dataset config to new one * @param oldConfig old dataset config from namespace * @param newConfig new dataset config thats about to be saved in namespace */ public static void copyFromOldConfig(DatasetConfig oldConfig, DatasetConfig newConfig) { if(oldConfig == null) { return; } newConfig.setId(oldConfig.getId()); newConfig.setTag(oldConfig.getTag()); newConfig.setCreatedAt(oldConfig.getCreatedAt()); newConfig.setType(oldConfig.getType()); newConfig.setFullPathList(oldConfig.getFullPathList()); newConfig.setOwner(oldConfig.getOwner()); // make sure to copy the acceleration settings from old to new config // newConfig may contain upgrade fileFormat physical settings if (oldConfig.getPhysicalDataset() != null) { if (newConfig.getPhysicalDataset() == null) { newConfig.setPhysicalDataset(new PhysicalDataset()); } } }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
@Test public void testCreateDatasets() throws Exception { expectSuccess(getBuilder(getAPIv2().path("space/spaceCreateDataset")).buildPut(Entity.json(new Space(null, "spaceCreateDataset", null, null, null, 0, null))), Space.class); DatasetPath datasetPath = new DatasetPath("spaceCreateDataset.ds1"); DatasetUI ds1 = createDatasetFromSQLAndSave(datasetPath, "select s.s_name from cp.\"tpch/supplier.parquet\" s", asList("cp")); DatasetConfig dataset = l(NamespaceService.class).getDataset(datasetPath.toNamespaceKey()); assertEquals(ds1.getVersion(), dataset.getTag()); getDataset(getDatasetPath(ds1)); DatasetUI ds2 = createDatasetFromParentAndSave(new DatasetPath("spaceCreateDataset.ds3"), getDatasetPath(ds1).toPathString()); getDataset(getDatasetPath(ds2)); }