protected DDF getDummyDDF() throws DDFException { if (mDummyDDF == null) mDummyDDF = this.newDDF(this); return mDummyDDF; }
public DDF newDDF(DDFManager manager, Object data, Class<?>[] typeSpecs, String name, Schema schema) throws DDFException { DDF ddf = this.newDDF(new Class<?>[] { DDFManager.class, Object.class, Class[].class, String.class, Schema.class }, new Object[] { manager, data, typeSpecs, name, schema }); return ddf; }
public DDF newDDF(Object data, Class<?>[] typeSpecs, String name, Schema schema) throws DDFException { // @formatter:off DDF ddf = this.newDDF(new Class<?>[] { DDFManager.class, Object.class, Class[].class, String.class, Schema.class }, new Object[] { this, data, typeSpecs, name, schema }); return ddf; }
@Override public DDF newDDF(DDFManager manager, Object data, Class<?>[] typeSpecs, String name, Schema schema) throws DDFException { DDF ddf = super.newDDF(manager, data, typeSpecs, name, schema); ((SparkDDF) ddf).saveAsTable(); return ddf; }
@Override public DDF newDDF(Object data, Class<?>[] typeSpecs, String name, Schema schema) throws DDFException { DDF ddf = super.newDDF(data, typeSpecs, name, schema); ((SparkDDF) ddf).saveAsTable(); return ddf; }
/** * Instantiates a new DDF of the type specified in ddf.ini as "DDF", passing in this DDFManager as the sole argument * * @return the newly instantiated DDF * @throws DDFException */ public DDF newDDF() throws DDFException { DDF ddf = this.newDDF(new Class<?>[] { DDFManager.class }, new Object[] { this }); ddf.getPersistenceHandler().setPersistable(false); return ddf; }
/** * Instantiates a new DDF of the type specified in ddf.ini as "DDF", using the constructor that requires only * {@link DDFManager} as an argument. * * @param manager the {@link DDFManager} to assign * @return the newly instantiated DDF * @throws DDFException */ public DDF newDDF(DDFManager manager) throws DDFException { DDF ddf = this.newDDF(new Class<?>[] { DDFManager.class }, new Object[] { manager }); ddf.getPersistenceHandler().setPersistable(false); return ddf; }
private void initialize(String name, T[] data, String engineName) throws DDFException { if (data == null || data.length == 0) throw new DDFException("Cannot initialize a null or zero-length Vector"); DDF newDDF = DDFManager.get(DDFManager.EngineType.fromString(engineName)) // .newDDF(null, (Object) data, new Class[] { Array.class, this.getParameterizedType() },name, // new Schema(name, String.format("%s %s", name, this.getParameterizedType().getSimpleName()))); this.initialize(newDDF, name); }
@Override public DDF residuals() throws DDFException { SparkDDF predictionDDF = (SparkDDF) this.getDDF(); JavaRDD<double[]> predictionRDD = predictionDDF.getJavaRDD(double[].class); JavaRDD<double[]> result = predictionRDD.map(new MetricsMapperResiduals()); if (result == null) mLog.error(">> javaRDD result of MetricMapper residuals is null"); if (predictionDDF.getManager() == null) mLog.error(">> predictionDDF.getManager() is null"); if (result.rdd() == null) mLog.error(">> result.rdd() is null"); if (predictionDDF.getSchema() == null) mLog.error(">> predictionDDF.getSchema() is null"); if (predictionDDF.getName() == null) mLog.error(">> predictionDDF.getName() is null"); Schema schema = new Schema("residuals double"); DDFManager manager = this.getDDF().getManager(); DDF residualDDF = manager .newDDF(manager, result.rdd(), new Class<?>[] { RDD.class, double[].class }, null, schema); if (residualDDF == null) mLog.error(">>>>>>>>>>>.residualDDF is null"); return residualDDF; }
@Test public void testCreateDDF() throws DDFException { Assert.assertNotNull("DDFManager cannot be null", this.getDDFManager()); DDF ddf = this.getDDFManager().newDDF(); Assert.assertNotNull("DDF cannot be null", ddf); Assert.assertNotNull(ddf.getUUID()); DDF ddf2 = this.getTestDDF(); Assert.assertNotNull("DDF cannot be null", ddf2); }
@Test public void testRegister() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf1 = manager.newDDF(); Collection<IGloballyAddressable> objs = manager.REGISTRY.getObjects(); Assert.assertEquals("Initial registry must be empty", 0, objs.size()); // TODO /** manager.REGISTRY.register(ddf1); Assert.assertEquals("Registry must have exactly 1 object", 1, objs.size()); manager.REGISTRY.register(ddf1); Assert.assertEquals("Registry must have exactly 1 object", 1, objs.size()); DDF ddf2 = (DDF) manager.REGISTRY.retrieve(ddf1.getNamespace(), ddf1.getName()); Assert.assertEquals("Stored object must equal retrieved object", ddf1, ddf2); manager.REGISTRY.unregisterAll(); objs = manager.REGISTRY.getObjects(); Assert.assertEquals("Registry must be empty after unregisterAll", 0, objs.size()); */ } }
@Ignore public void testPersistDDF() throws Exception { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); PersistenceUri uri = ddf.persist(); Assert.assertEquals("PersistenceUri must have '" + ConfigConstant.ENGINE_NAME_BASIC + "' for engine/protocol", ConfigConstant.ENGINE_NAME_BASIC.toString(), uri.getEngine()); Assert.assertTrue("Persisted file must exist: " + uri, new File(uri.getPath()).exists()); ddf.unpersist(); } /*
@Test public void testNewRepHandler() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = new DummyRepresentationHandler(ddf); Double[] data = new Double[] { 0.0, 2.0, 1.0, 3.0, 4.0, 10.0, 11.0 }; handler.add(data, Double[].class); Object[] obj1 = (Object[]) handler.get(Object[].class); Integer[] obj2 = (Integer[]) handler.get(Integer[].class); Assert.assertNotNull(obj1); Assert.assertNotNull(obj2); Assert.assertEquals(handler.getAllRepresentations().size(), 3); String[] obj3 = (String[]) handler.get(String[].class); Assert.assertNotNull(obj3); Assert.assertEquals(handler.getAllRepresentations().size(), 4); } }
@Override public DDF merge(DDF anotherDDF) throws DDFException { DataFrame rdd1 = ((DataFrame) this.getDDF().getRepresentationHandler().get(DataFrame.class)); DataFrame rdd2 = ((DataFrame) anotherDDF.getRepresentationHandler().get(DataFrame.class)); DataFrame newRDD = rdd1.unionAll(rdd2); Schema schema = SparkUtils.schemaFromDataFrame(newRDD); return this.getManager().newDDF(newRDD, new Class<?>[]{DataFrame.class}, null, schema); } }
@Test public void testPersistenceDir() throws IOException, DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); List<String> namespaces = ddf.getPersistenceHandler().listNamespaces(); Assert.assertNotNull(namespaces); for (String namespace : namespaces) { List<String> ddfs = ddf.getPersistenceHandler().listItems(namespace); Assert.assertNotNull(ddfs); } }
@Override public DDF sql2ddf(String command, Schema schema, DataSourceDescriptor dataSource, DataFormat dataFormat) throws DDFException { // TableRDD tableRdd = null; // RDD<Row> rddRow = null; DataFrame rdd = this.getHiveContext().sql(command); if (schema == null) schema = SchemaHandler.getSchemaFromDataFrame(rdd); DDF ddf = this.getManager().newDDF(this.getManager(), rdd, new Class<?>[] {DataFrame.class}, null, schema); ddf.getRepresentationHandler().cache(false); ddf.getRepresentationHandler().get(new Class<?>[]{RDD.class, Row.class}); return ddf; }
@Test public void testRepresentDDF() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = ddf.getRepresentationHandler(); handler.reset(); Assert.assertNull("There should not be any existing representations", handler.get(list.get(0).getClass())); handler.set(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.add(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.remove(List.class, String.class); Assert.assertNull("There should now be no representation of type <List,String>", handler.get(List.class, String.class)); }