private DDFManager getDDFManager() throws DDFException { if (mManager == null) mManager = DDFManager.get(DDFManager.EngineType.BASIC); return mManager; }
@Test public void testLoadConfig() throws Exception { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); // this will trigger a configuration loading Assert.assertEquals("basic", manager.getEngine()); Assert.assertNotNull(Config.getConfigHandler()); Assert.assertNotNull(Config.getConfigHandler().loadConfig()); }
public static DDFManager get(String engineType) throws DDFException { return DDFManager.get(EngineType.fromString(engineType)); }
@Test public void testReadConfig() throws Exception { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); // this will trigger a configuration loading Assert.assertEquals("basic", manager.getEngine()); Configuration config = Config.getConfigHandler().getConfig(); Assert.assertNotNull(config); Map<String, Section> sections = config.getSections(); Assert.assertNotNull(sections); for (String sectionName : sections.keySet()) { Map<String, String> settings = config.getSettings(sectionName); Assert.assertNotNull(settings); } }
@BeforeClass public static void startServer() throws Exception { Thread.sleep(1000); LOG = LoggerFactory.getLogger(BaseTest.class); manager = DDFManager.get(DDFManager.EngineType.SPARK); }
private void initialize(String name, T[] data, String engineName) throws DDFException { if (data == null || data.length == 0) throw new DDFException("Cannot initialize a null or zero-length Vector"); DDF newDDF = DDFManager.get(DDFManager.EngineType.fromString(engineName)) // .newDDF(null, (Object) data, new Class[] { Array.class, this.getParameterizedType() },name, // new Schema(name, String.format("%s %s", name, this.getParameterizedType().getSimpleName()))); this.initialize(newDDF, name); }
public static IPersistible doLoad(PersistenceUri uri) throws DDFException { if (uri == null) throw new DDFException("URI cannot be null"); if (Strings.isNullOrEmpty(uri.getEngine())) throw new DDFException("Engine/Protocol in URI cannot be missing"); return DDFManager.get(EngineType.fromString(uri.getEngine())).load(uri); }
public static void main(String[] args) throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.SPARK); manager.sql("drop table if exists airline", false); manager.sql("create table airline (Year int,Month int,DayofMonth int," + "DayOfWeek int,DepTime int,CRSDepTime int,ArrTime int," + "CRSArrTime int,UniqueCarrier string, FlightNum int, " + "TailNum string, ActualElapsedTime int, CRSElapsedTime int, " + "AirTime int, ArrDelay int, DepDelay int, Origin string, " + "Dest string, Distance int, TaxiIn int, TaxiOut int, Cancelled int, " + "CancellationCode string, Diverted string, CarrierDelay int, " + "WeatherDelay int, NASDelay int, SecurityDelay int, LateAircraftDelay int ) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','", false); manager.sql("load data local inpath 'resources/test/airline.csv' into table airline", false); DDF ddf = manager.sql2ddf("SELECT * FROM AIRLINE", false); long nrow = ddf.getNumRows(); int ncol = ddf.getNumColumns(); System.out.println("Number of data row is " + nrow); System.out.println("Number of data columns is " + ncol); } }
@BeforeClass public static void startServer() throws Exception { Thread.sleep(1000); // LOG = LoggerFactory.getLogger(BaseTest.class); // manager = DDFManager.get("spark"); /* manager = DDFManager.get("jdbc", new JDBCDataSourceDescriptor(new DataSourceURI("jdbc:mysql://localhost/testdb"), new JDBCDataSourceDescriptor.JDBCDataSourceCredentials("pauser", "papwd"), null)); DataSourceDescriptor ds = manager.getDataSourceDescriptor(); if (ds instanceof JDBCDataSourceDescriptor) { System.out.println("hello"); } DDF ret = manager.sql2ddf("select * from testtable", "jdbc");*/ // Add 2 test ddfs. manager = DDFManager.get(DDFManager.EngineType.SPARK); parser = new CCJSqlParserManager(); }
@Test public void testRegister() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf1 = manager.newDDF(); Collection<IGloballyAddressable> objs = manager.REGISTRY.getObjects(); Assert.assertEquals("Initial registry must be empty", 0, objs.size()); // TODO /** manager.REGISTRY.register(ddf1); Assert.assertEquals("Registry must have exactly 1 object", 1, objs.size()); manager.REGISTRY.register(ddf1); Assert.assertEquals("Registry must have exactly 1 object", 1, objs.size()); DDF ddf2 = (DDF) manager.REGISTRY.retrieve(ddf1.getNamespace(), ddf1.getName()); Assert.assertEquals("Stored object must equal retrieved object", ddf1, ddf2); manager.REGISTRY.unregisterAll(); objs = manager.REGISTRY.getObjects(); Assert.assertEquals("Registry must be empty after unregisterAll", 0, objs.size()); */ } }
@Ignore public void testPersistDDF() throws Exception { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); PersistenceUri uri = ddf.persist(); Assert.assertEquals("PersistenceUri must have '" + ConfigConstant.ENGINE_NAME_BASIC + "' for engine/protocol", ConfigConstant.ENGINE_NAME_BASIC.toString(), uri.getEngine()); Assert.assertTrue("Persisted file must exist: " + uri, new File(uri.getPath()).exists()); ddf.unpersist(); } /*
@Test public void testNewRepHandler() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = new DummyRepresentationHandler(ddf); Double[] data = new Double[] { 0.0, 2.0, 1.0, 3.0, 4.0, 10.0, 11.0 }; handler.add(data, Double[].class); Object[] obj1 = (Object[]) handler.get(Object[].class); Integer[] obj2 = (Integer[]) handler.get(Integer[].class); Assert.assertNotNull(obj1); Assert.assertNotNull(obj2); Assert.assertEquals(handler.getAllRepresentations().size(), 3); String[] obj3 = (String[]) handler.get(String[].class); Assert.assertNotNull(obj3); Assert.assertEquals(handler.getAllRepresentations().size(), 4); } }
@Test public void testPersistenceDir() throws IOException, DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); List<String> namespaces = ddf.getPersistenceHandler().listNamespaces(); Assert.assertNotNull(namespaces); for (String namespace : namespaces) { List<String> ddfs = ddf.getPersistenceHandler().listItems(namespace); Assert.assertNotNull(ddfs); } }
@Test public void testRepresentDDF() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = ddf.getRepresentationHandler(); handler.reset(); Assert.assertNull("There should not be any existing representations", handler.get(list.get(0).getClass())); handler.set(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.add(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.remove(List.class, String.class); Assert.assertNull("There should now be no representation of type <List,String>", handler.get(List.class, String.class)); }
@Test public void testConfusionMatrix() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.SPARK); try { manager.sql("drop table if exists airline", false);
@Ignore public void testModelSerialize2DDF() throws DDFException { DummyModel dummyModel = new DummyModel(20, "dummymodel2"); Model model = new Model(dummyModel); DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = model.serialize2DDF(manager); Object obj = ddf.getRepresentationHandler().get(List.class, String.class); List<Schema.Column> cols = ddf.getSchema().getColumns(); List<String> lsString = (List<String>) obj; Assert.assertTrue(obj != null); Assert.assertTrue(obj instanceof List); Assert.assertTrue(ddf != null); APersistenceHandler.PersistenceUri uri = ddf.persist(); PersistenceHandler pHandler = new PersistenceHandler(null); DDF ddf2 = (DDF) pHandler.load(uri); Model model2 = Model.deserializeFromDDF((BasicDDF) ddf2); Assert.assertTrue(ddf2 != null); Assert.assertTrue(model2 != null); Assert.assertTrue(model2.getRawModel() instanceof DummyModel); } }