/** * Registers a dataset given into the <code>resource</code> definition. * * @param resource The dataset definition resource. */ protected ElasticSearchDataSetDef _registerDataSet(String resource) throws Exception { // Register the SQL data set URL fileURL = Thread.currentThread().getContextClassLoader().getResource(resource); String json = IOUtils.toString(fileURL); ElasticSearchDataSetDef def = (ElasticSearchDataSetDef) jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); return def; }
@Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { try { checkNotNull("file", file); checkNotNull("attrs", attrs); if (file.getFileName().toString().endsWith(DATASET_EXT) && attrs.isRegularFile()) { String json = ioService.readAllString(file); DataSetDef def = getDataSetDefJsonMarshaller().fromJson(json); result.add(def); } } catch (final Exception e) { log.error("Data set definition read error: " + file.getFileName(), e); return FileVisitResult.TERMINATE; } return FileVisitResult.CONTINUE; } });
@Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { try { checkNotNull("file", file); checkNotNull("attrs", attrs); if (file.getFileName().toString().endsWith(DATASET_EXT) && attrs.isRegularFile()) { String json = ioService.readAllString(file); DataSetDef def = getDataSetDefJsonMarshaller().fromJson(json); result.add(def); } } catch (final Exception e) { log.error("Data set definition read error: " + file.getFileName(), e); return FileVisitResult.TERMINATE; } return FileVisitResult.CONTINUE; } });
@Test public void testAllColumns() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("expenseReports_allcolumns.dset"); String json = IOUtils.toString(fileURL, StandardCharsets.UTF_8); SQLDataSetDef def = (SQLDataSetDef) jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); DataSetMetadata metadata = dataSetManager.getDataSetMetadata("expense_reports_allcolumns"); assertThat(metadata.getNumberOfColumns()).isEqualTo(6); assertThat(metadata.getEstimatedSize()).isEqualTo(6350); }
private void assertDataSetDef(final String def1, final String def2) throws Exception { if (def1 == null && def2 != null) Assert.assertTrue("JSON string for Def1 is null and for Def2 is not null", false); if (def1 != null && def2 == null) Assert.assertTrue("JSON string for Def1 is not null and for Def2 is null", false); if (def1 == null) Assert.assertTrue("JSON string for both definitions is null", false); DataSetDef def1Object = jsonMarshaller.fromJson(def1); DataSetDef def2Object = jsonMarshaller.fromJson(def2); Assert.assertEquals(def1Object, def2Object); }
private void assertDataSetDef(final String def1, final String def2) throws Exception { if (def1 == null && def2 != null) Assert.assertTrue("JSON string for Def1 is null and for Def2 is not null", false); if (def1 != null && def2 == null) Assert.assertTrue("JSON string for Def1 is not null and for Def2 is null", false); if (def1 == null) Assert.assertTrue("JSON string for both definitions is null", false); DataSetDef def1Object = jsonMarshaller.fromJson(def1); DataSetDef def2Object = jsonMarshaller.fromJson(def2); Assert.assertEquals(def1Object, def2Object); }
public DataSetDef loadDataSetDef(org.uberfire.backend.vfs.Path path) { Path nioPath = convert(path); if (ioService.exists(nioPath)) { try { String json = ioService.readAllString(nioPath); DataSetDef def = getDataSetDefJsonMarshaller().fromJson(json); return def; } catch (Exception e) { String msg = "Error parsing data set JSON definition: " + path.getFileName(); throw exceptionManager.handleException(new Exception(msg, e)); } } return null; }
public DataSetDef loadDataSetDef(org.uberfire.backend.vfs.Path path) { Path nioPath = convert(path); if (ioService.exists(nioPath)) { try { String json = ioService.readAllString(nioPath); DataSetDef def = getDataSetDefJsonMarshaller().fromJson(json); return def; } catch (Exception e) { String msg = "Error parsing data set JSON definition: " + path.getFileName(); throw exceptionManager.handleException(new Exception(msg, e)); } } return null; }
@Test public void testGenerateDataSet() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("salesPerYear.dset"); String json = IOUtils.toString(fileURL); DataSetDef def = jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); DataSet result = dataSetManager.getDataSet("salesPerYear"); //printDataSet(result); assertDataSetValues(result, dataSetFormatter, new String[][]{ {"JANUARY", "1,000.00", "2,000.00", "3,000.00"}, {"FEBRUARY", "1,400.00", "2,300.00", "2,000.00"}, {"MARCH", "1,300.00", "2,000.00", "1,400.00"}, {"APRIL", "900.00", "2,100.00", "1,500.00"}, {"MAY", "1,300.00", "2,300.00", "1,600.00"}, {"JUNE", "1,010.00", "2,000.00", "1,500.00"}, {"JULY", "1,050.00", "2,400.00", "3,000.00"}, {"AUGUST", "2,300.00", "2,000.00", "3,200.00"}, {"SEPTEMBER", "1,900.00", "2,700.00", "3,000.00"}, {"OCTOBER", "1,200.00", "2,200.00", "3,100.00"}, {"NOVEMBER", "1,400.00", "2,100.00", "3,100.00"}, {"DECEMBER", "1,100.00", "2,100.00", "4,200.00"} }, 0); }
@Test public void testGenerateDataSetAdjusted() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("salesPerYearAdjusted.dset"); String json = IOUtils.toString(fileURL); DataSetDef def = jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); DataSet result = dataSetManager.getDataSet("salesPerYearAdjusted"); //printDataSet(result); assertDataSetValues(result, dataSetFormatter, new String[][] { {"JANUARY", "900.00", "1,800.00", "2,700.00"}, {"FEBRUARY", "1,260.00", "2,070.00", "1,800.00"}, {"MARCH", "1,170.00", "1,800.00", "1,260.00"}, {"APRIL", "810.00", "1,890.00", "1,350.00"}, {"MAY", "1,170.00", "2,070.00", "1,440.00"}, {"JUNE", "909.00", "1,800.00", "1,350.00"}, {"JULY", "945.00", "2,160.00", "2,700.00"}, {"AUGUST", "2,070.00", "1,800.00", "2,880.00"}, {"SEPTEMBER", "1,710.00", "2,430.00", "2,700.00"}, {"OCTOBER", "1,080.00", "1,980.00", "2,790.00"}, {"NOVEMBER", "1,260.00", "1,890.00", "2,790.00"}, {"DECEMBER", "990.00", "1,890.00", "3,780.00"} }, 0); }
@Test public void testGenerateDataSet() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("salesPerYear.dset"); String json = IOUtils.toString(fileURL, StandardCharsets.UTF_8); DataSetDef def = jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); DataSet result = dataSetManager.getDataSet("salesPerYear"); //printDataSet(result); assertDataSetValues(result, dataSetFormatter, new String[][]{ {"JANUARY", "1,000.00", "2,000.00", "3,000.00"}, {"FEBRUARY", "1,400.00", "2,300.00", "2,000.00"}, {"MARCH", "1,300.00", "2,000.00", "1,400.00"}, {"APRIL", "900.00", "2,100.00", "1,500.00"}, {"MAY", "1,300.00", "2,300.00", "1,600.00"}, {"JUNE", "1,010.00", "2,000.00", "1,500.00"}, {"JULY", "1,050.00", "2,400.00", "3,000.00"}, {"AUGUST", "2,300.00", "2,000.00", "3,200.00"}, {"SEPTEMBER", "1,900.00", "2,700.00", "3,000.00"}, {"OCTOBER", "1,200.00", "2,200.00", "3,100.00"}, {"NOVEMBER", "1,400.00", "2,100.00", "3,100.00"}, {"DECEMBER", "1,100.00", "2,100.00", "4,200.00"} }, 0); }
@Test public void testAllColumns() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("expenseReports_allcolumns.dset"); String json = IOUtils.toString(fileURL); SQLDataSetDef def = (SQLDataSetDef) jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); DataSetMetadata metadata = dataSetManager.getDataSetMetadata("expense_reports_allcolumns"); assertThat(metadata.getNumberOfColumns()).isEqualTo(6); assertThat(metadata.getEstimatedSize()).isEqualTo(6350); }
@Test public void testProviderRegistryInit() throws Exception { bootstrap.init(); DataSetProviderRegistry dataSetProviderRegistry = DataSetCore.get().getDataSetProviderRegistry(); assertEquals(dataSetProviderRegistry, providerRegistryCDI); assertEquals(dataSetProviderRegistry.getAvailableTypes().size(), 6); DataSetDefJSONMarshaller jsonMarshaller = DataSetCore.get().getDataSetDefJSONMarshaller(); jsonMarshaller.fromJson(CSV_JSON); // No exception }
@Test public void testProviderRegistryInit() throws Exception { bootstrap.init(); DataSetProviderRegistry dataSetProviderRegistry = DataSetCore.get().getDataSetProviderRegistry(); assertEquals(dataSetProviderRegistry, providerRegistryCDI); assertEquals(dataSetProviderRegistry.getAvailableTypes().size(), 6); DataSetDefJSONMarshaller jsonMarshaller = DataSetCore.get().getDataSetDefJSONMarshaller(); jsonMarshaller.fromJson(CSV_JSON); // No exception }
@Test public void testColumnAlias() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("expenseReports_columnalias.dset"); String json = IOUtils.toString(fileURL, StandardCharsets.UTF_8); SQLDataSetDef def = (SQLDataSetDef) jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); String uuid = "expense_reports_columnalias"; DataSetMetadata metadata = dataSetManager.getDataSetMetadata(uuid); assertThat(metadata.getNumberOfColumns()).isEqualTo(3); assertThat(metadata.getColumnId(0)).isEqualTo("Id"); assertThat(metadata.getColumnId(1)).isEqualTo("Employee"); assertThat(metadata.getColumnId(2)).isEqualTo("Amount"); DataSet dataSet = dataSetManager.lookupDataSet( DataSetLookupFactory.newDataSetLookupBuilder() .dataset(uuid) .filter("id", FilterFactory.notNull()) .filter("amount", OR(greaterThan(0), lowerThan(1000))) .group("employee") .column("EMPLOYEE", "employee") .column("id", AggregateFunctionType.COUNT, "id") .column("AMOUNT", AggregateFunctionType.SUM, "amount") .sort("id", SortOrder.ASCENDING) .buildLookup()); assertThat(dataSet.getColumns().size()).isEqualTo(3); assertNotNull(dataSet.getColumnById("ID")); assertNotNull(dataSet.getColumnById("EMPLOYEE")); assertNotNull(dataSet.getColumnById("AMOUNT")); }
protected void _testDataSetCache(String scenario, int rows) throws Exception { // Register the data set definition URL fileURL = Thread.currentThread().getContextClassLoader().getResource("expenseReports_" + scenario + ".dset"); String json = IOUtils.toString(fileURL); SQLDataSetDef def = (SQLDataSetDef) jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); // Lookup the dataset (forces the caches to initialize) dataSetManager.lookupDataSet( DataSetLookupFactory.newDataSetLookupBuilder() .dataset("expense_reports_" + scenario) .buildLookup()); // Insert some extra rows into the database populateDbTable(); // Ensure the data set is now outdated assertThat(sqlDataSetProvider.isDataSetOutdated(def)).isEqualTo(true); // Lookup the last database content DataSet result = dataSetManager.lookupDataSet( DataSetLookupFactory.newDataSetLookupBuilder() .dataset("expense_reports_" + scenario) .buildLookup()); assertThat(result.getRowCount()).isEqualTo(rows); } }
@Test public void testColumnSet() throws Exception { URL fileURL = Thread.currentThread().getContextClassLoader().getResource("expenseReports_columnset.dset"); String json = IOUtils.toString(fileURL); SQLDataSetDef def = (SQLDataSetDef) jsonMarshaller.fromJson(json); dataSetDefRegistry.registerDataSetDef(def); DataSetMetadata metadata = dataSetManager.getDataSetMetadata("expense_reports_columnset"); assertThat(metadata.getNumberOfColumns()).isEqualTo(4); if (!testSettings.isMonetDB()) { assertThat(metadata.getEstimatedSize()).isEqualTo(4300); } final String uuid = "expense_reports_columnset"; DataSet dataSet = dataSetManager.lookupDataSet( DataSetLookupFactory.newDataSetLookupBuilder() .dataset(uuid) .buildLookup()); assertThat(dataSet.getColumns().size()).isEqualTo(4); assertThat(dataSet.getValueAt(0, 0)).isEqualTo("Engineering"); assertThat(dataSet.getValueAt(0, 1)).isEqualTo("Roxie Foraker"); assertThat(dataSet.getValueAt(0, 2)).isEqualTo(120.35d); assertThat(dataSetFormatter.formatValueAt(dataSet, 0, 3)).isEqualTo("12/11/15 12:00"); assertDataSetDefinition(dataSet, uuid); }
@Test public void testCustom() throws Exception { final DataSetDef dataSetDef = new DataSetDef(); dataSetDef.setName("custom data set name"); dataSetDef.setUUID("custom-test-uuid"); dataSetDef.setProvider(CUSTOM_PROVIDER_TYPE); dataSetDef.setCacheEnabled(false); dataSetDef.setCacheMaxRows(100); dataSetDef.setPublic(true); dataSetDef.setPushEnabled(false); dataSetDef.setPushMaxSize(10); dataSetDef.setRefreshAlways(false); dataSetDef.setRefreshTime("1second"); dataSetDef.setProperty("prop1", "Hello"); String json = jsonMarshaller.toJsonString(dataSetDef); String customJSONContent = getFileAsString(CUSTOM_DEF_PATH); assertDataSetDef(json, customJSONContent); DataSetDef fromJson = jsonMarshaller.fromJson(customJSONContent); assertEquals(dataSetDef, fromJson); }
@Test public void testCustom() throws Exception { final DataSetDef dataSetDef = new DataSetDef(); dataSetDef.setName("custom data set name"); dataSetDef.setUUID("custom-test-uuid"); dataSetDef.setProvider(CUSTOM_PROVIDER_TYPE); dataSetDef.setCacheEnabled(false); dataSetDef.setCacheMaxRows(100); dataSetDef.setPublic(true); dataSetDef.setPushEnabled(false); dataSetDef.setPushMaxSize(10); dataSetDef.setRefreshAlways(false); dataSetDef.setRefreshTime("1second"); dataSetDef.setProperty("prop1", "Hello"); String json = jsonMarshaller.toJsonString(dataSetDef); String customJSONContent = getFileAsString(CUSTOM_DEF_PATH); assertDataSetDef(json, customJSONContent); DataSetDef fromJson = jsonMarshaller.fromJson(customJSONContent); assertEquals(dataSetDef, fromJson); }
@Test public void testColumns() throws Exception { String json = getFileAsString(EXPENSES_DEF_PATH); DataSetDef def = jsonMarshaller.fromJson(json); assertEquals(def.getColumns().size(), 6); DataColumnDef column1 = def.getColumnById("EXPENSES_ID"); DataColumnDef column2 = def.getColumnById("DEPARTMENT"); DataColumnDef column3 = def.getColumnById("AMOUNT"); DataColumnDef column4 = def.getColumnById("CREATION_DATE"); DataColumnDef column5 = def.getColumnById("EMPLOYEE"); DataColumnDef column6 = def.getColumnById("CITY"); assertNotNull(column1); assertNotNull(column2); assertNotNull(column3); assertNotNull(column4); assertNotNull(column5); assertNotNull(column6); assertEquals(column1.getColumnType(), ColumnType.NUMBER); assertEquals(column2.getColumnType(), ColumnType.LABEL); assertEquals(column3.getColumnType(), ColumnType.NUMBER); assertEquals(column4.getColumnType(), ColumnType.DATE); assertEquals(column5.getColumnType(), ColumnType.LABEL); assertEquals(column6.getColumnType(), ColumnType.LABEL); }