void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
@Test public void testQuoteInPathAndDatasetName() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("test dimension", "VARCHAR"), new ViewFieldType("test \" measure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("@dremio", "fol\"der.ext", "foo", "bar")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); // make sure everything is escaped correctly assertTrue(script.contains(" DIMENSION \"test dimension\"")); assertTrue(script.contains(" MEASURE \"test \"\" measure\"")); assertTrue(script.contains(" FROM \"@dremio\".\"fol\"\"der.ext\".\"foo\".\"bar\"")); }
@Test public void testQuoting() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("testdimension", "VARCHAR"), new ViewFieldType("testmeasure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("space", "folder.ext", "UNTITLED")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains(" DIMENSION \"testdimension\"")); assertTrue(script.contains(" MEASURE \"testmeasure\"")); assertTrue(script.contains(" FROM \"space\".\"folder.ext\".\"UNTITLED\"")); }
@Test public void testSanitizing() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("testdetail2", "STRUCTURED"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("evil /!@ *-=+{}<>,~ characters") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(DatasetTool.TMP_DATASET_PATH.toPathList()) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains("evil_characters: DIRECT QUERY")); } }
public static DatasetConfig toDatasetConfig(PhysicalDatasetConfig physicalDatasetConfig, String owner) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(physicalDatasetConfig.getFullPathList()); datasetConfig.setName(physicalDatasetConfig.getName()); datasetConfig.setType(physicalDatasetConfig.getType()); datasetConfig.setTag(physicalDatasetConfig.getTag()); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings( physicalDatasetConfig.getFormatSettings())); return datasetConfig; }
@Override public DatasetConfig getDataset() throws Exception { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
@Override public DatasetConfig getDataset() { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
private void createSpaceAndVDS(NamespaceKey spacePath, List<String> vdsPath) throws NamespaceException { // create space SpaceConfig spaceConfig = new SpaceConfig(); spaceConfig.setName(spacePath.getRoot()); newNamespaceService().addOrUpdateSpace(spacePath, spaceConfig); // create vds VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql("select * from sys.version"); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(vdsPath.get(vdsPath.size() - 1)); datasetConfig.setFullPathList(vdsPath); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setVirtualDataset(virtualDataset); getSabotContext().getViewCreator(SystemUser.SYSTEM_USERNAME).createView(vdsPath, "select * from sys.version", null); } }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
public static void addPhysicalDS(NamespaceService ns, String filePath, DatasetType type, byte[] datasetSchema) throws Exception { NamespaceKey datasetPath = new NamespaceKey(PathUtils.parseFullPath(filePath)); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); final PhysicalDataset physicalDataset = new PhysicalDataset(); if (datasetSchema != null) { datasetConfig.setRecordSchema(io.protostuff.ByteString.copyFrom(datasetSchema)); } datasetConfig.setSchemaVersion(DatasetHelper.CURRENT_VERSION); datasetConfig.setPhysicalDataset(physicalDataset); ns.tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public static void addDS(NamespaceService ns, String name) throws Exception { final NamespaceKey dsPath = new NamespaceKey(PathUtils.parseFullPath(name)); final DatasetConfig ds = new DatasetConfig(); final VirtualDataset vds = new VirtualDataset(); vds.setVersion(DatasetVersion.newVersion()); ds.setType(DatasetType.VIRTUAL_DATASET); ds.setVirtualDataset(vds); ds.setFullPathList(dsPath.getPathComponents()); ds.setName(dsPath.getName()); ns.addOrUpdateDataset(dsPath, ds); }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }