public void saveVirtualDataset(List<String> path) throws NamespaceException { final VirtualDataset virtualDataset = new VirtualDataset().setSql("SELECT 1"); saveDataset(path, DatasetType.VIRTUAL_DATASET, config -> config.setVirtualDataset(virtualDataset)); }
@Test public void testQuoteInPathAndDatasetName() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("test dimension", "VARCHAR"), new ViewFieldType("test \" measure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("@dremio", "fol\"der.ext", "foo", "bar")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); // make sure everything is escaped correctly assertTrue(script.contains(" DIMENSION \"test dimension\"")); assertTrue(script.contains(" MEASURE \"test \"\" measure\"")); assertTrue(script.contains(" FROM \"@dremio\".\"fol\"\"der.ext\".\"foo\".\"bar\"")); }
@Test public void testQuoting() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("testdimension", "VARCHAR"), new ViewFieldType("testmeasure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("space", "folder.ext", "UNTITLED")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains(" DIMENSION \"testdimension\"")); assertTrue(script.contains(" MEASURE \"testmeasure\"")); assertTrue(script.contains(" FROM \"space\".\"folder.ext\".\"UNTITLED\"")); }
@Test public void testFieldNamesWithSpaceQuoting() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("test dimension", "VARCHAR"), new ViewFieldType("test \" measure", "INTEGER"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(Arrays.asList("space", "folder.ext", "UNTITLED")) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); // make sure everything is escaped correctly assertTrue(script.contains(" DIMENSION \"test dimension\"")); assertTrue(script.contains(" MEASURE \"test \"\" measure\"")); assertTrue(script.contains(" FROM \"space\".\"folder.ext\".\"UNTITLED\"")); }
@Override public void updateView(List<String> path, String sql, List<String> sqlContext, NamespaceAttribute... attributes) { SecurityContext securityContext = getSecurityContext(); QueryExecutor executor = new QueryExecutor(jobsService, null, securityContext); DatasetVersion version = DatasetVersion.newVersion(); NamespaceKey namespaceKey = new NamespaceKey(path); try { DatasetConfig dataset = namespaceService.getDataset(namespaceKey); DatasetPath datasetPath = new DatasetPath(path); final VirtualDatasetUI virtualDataset = datasetService.get(datasetPath); Transformer transformer = new Transformer(contextService.get().get(), namespaceService, datasetService, executor, securityContext); TransformUpdateSQL transformUpdateSQL = new TransformUpdateSQL(); transformUpdateSQL.setSql(sql); transformUpdateSQL.setSqlContextList(sqlContext); VirtualDatasetUI virtualDatasetUI = transformer.transformWithExtract(version, datasetPath, virtualDataset, transformUpdateSQL); // copy over VirtualDatasetUI values to the pre-existing DatasetConfig dataset.setVirtualDataset(DatasetsUtil.toVirtualDataset(virtualDatasetUI)); dataset.setRecordSchema(virtualDatasetUI.getRecordSchema()); namespaceService.addOrUpdateDataset(namespaceKey, dataset, attributes); } catch (Exception e) { throw Throwables.propagate(e); } }
@Test public void testGetDatasetCatalogEntityById() throws Exception { DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setId(new EntityId("dataset-id")); datasetConfig.setFullPathList(Collections.singletonList("path")); datasetConfig.setType(VIRTUAL_DATASET); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(""); datasetConfig.setVirtualDataset(virtualDataset); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); namespaceContainer.setDataset(datasetConfig); when(namespaceService.getEntityById(datasetConfig.getId().getId())).thenReturn(namespaceContainer); ReflectionSettings reflectionSettings = mock(ReflectionSettings.class); when(reflectionSettings.getStoredReflectionSettings(any(NamespaceKey.class))).thenReturn(Optional.<AccelerationSettings>absent()); when(reflectionServiceHelper.getReflectionSettings()).thenReturn(reflectionSettings); DremioTable dremioTable = mock(DremioTable.class); when(dremioTable.getDatasetConfig()).thenReturn(datasetConfig); when(catalog.getTable(any(String.class))).thenReturn(dremioTable); Optional<CatalogEntity> entity = catalogServiceHelper.getCatalogEntityById(datasetConfig.getId().getId()); assertTrue(entity.isPresent()); CatalogEntity catalogEntity = entity.get(); assertTrue(catalogEntity instanceof Dataset); Dataset dataset = (Dataset) catalogEntity; assertEquals(dataset.getId(), datasetConfig.getId().getId()); }
@Test public void testSanitizing() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("testdetail2", "STRUCTURED"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("evil /!@ *-=+{}<>,~ characters") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(DatasetTool.TMP_DATASET_PATH.toPathList()) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains("evil_characters: DIRECT QUERY")); } }
virtualDataset.setContextList(dataset.getSqlContext()); config.setType(VIRTUAL_DATASET); config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config);
@Test public void testGeneratedOutput() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList( new ViewFieldType("testdimension", "VARCHAR"), new ViewFieldType("testdimension2", "CHAR"), new ViewFieldType("testmeasure", "INTEGER"), new ViewFieldType("testmeasure2", "REAL"), new ViewFieldType("testdetail", "MAP"), new ViewFieldType("testdetail2", "STRUCTURED"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("UNTITLED") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(DatasetTool.TMP_DATASET_PATH.toPathList()) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains(" DIMENSION \"testdimension\", \"testdimension2\"")); assertTrue(script.contains(" MEASURE \"testmeasure\", \"testmeasure2\"")); assertTrue(script.contains(" DETAIL \"testdetail\", \"testdetail2\"")); assertTrue(script.contains(" FROM \"tmp\".\"UNTITLED\"")); }
currentDatasetConfig.setVirtualDataset(virtualDataset);
virtualDataset.setContextList(dataset.getSqlContext()); config.setType(PHYSICAL_DATASET); config.setVirtualDataset(virtualDataset);
private void createSpaceAndVDS(NamespaceKey spacePath, List<String> vdsPath) throws NamespaceException { // create space SpaceConfig spaceConfig = new SpaceConfig(); spaceConfig.setName(spacePath.getRoot()); newNamespaceService().addOrUpdateSpace(spacePath, spaceConfig); // create vds VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql("select * from sys.version"); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(vdsPath.get(vdsPath.size() - 1)); datasetConfig.setFullPathList(vdsPath); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setVirtualDataset(virtualDataset); getSabotContext().getViewCreator(SystemUser.SYSTEM_USERNAME).createView(vdsPath, "select * from sys.version", null); } }
public static void addDS(NamespaceService ns, String name) throws Exception { final NamespaceKey dsPath = new NamespaceKey(PathUtils.parseFullPath(name)); final DatasetConfig ds = new DatasetConfig(); final VirtualDataset vds = new VirtualDataset(); vds.setVersion(DatasetVersion.newVersion()); ds.setType(DatasetType.VIRTUAL_DATASET); ds.setVirtualDataset(vds); ds.setFullPathList(dsPath.getPathComponents()); ds.setName(dsPath.getName()); ns.addOrUpdateDataset(dsPath, ds); }
@Test public void testUpdatePDSWithChangedTypeShouldFail() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.PHYSICAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, null, null, null, null ); DatasetConfig config = new DatasetConfig(); config.setId(new EntityId(dataset.getId())); config.setFullPathList(dataset.getPath()); config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(dataset.getSql()); virtualDataset.setContextList(dataset.getSqlContext()); config.setType(VIRTUAL_DATASET); config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config); thrown.expect(IllegalArgumentException.class); catalogServiceHelper.updateCatalogItem(dataset, dataset.getId()); }
public static VirtualDatasetVersion toVirtualDatasetVersion(VirtualDatasetUI virtualDatasetUI) { final VirtualDatasetVersion vvds = new VirtualDatasetVersion(); final DatasetConfig datasetConfig = new DatasetConfig(); final VirtualDataset virtualDataset = toVirtualDataset(virtualDatasetUI); vvds.setLastTransform(virtualDatasetUI.getLastTransform()); vvds.setState(virtualDatasetUI.getState()); vvds.setPreviousVersion(virtualDatasetUI.getPreviousVersion()); vvds.setNamed(virtualDatasetUI.getIsNamed()); vvds.setDerivation(virtualDatasetUI.getDerivation()); datasetConfig.setName(virtualDatasetUI.getName()); datasetConfig.setOwner(virtualDatasetUI.getOwner()); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setCreatedAt(virtualDatasetUI.getCreatedAt()); datasetConfig.setFullPathList(virtualDatasetUI.getFullPathList()); datasetConfig.setTag(virtualDatasetUI.getSavedTag()); datasetConfig.setVirtualDataset(virtualDataset); datasetConfig.setRecordSchema(virtualDatasetUI.getRecordSchema()); if (virtualDatasetUI.getId() != null) { datasetConfig.setId(new EntityId(virtualDatasetUI.getId())); } vvds.setDataset(datasetConfig); return vvds; }