public void removeDDF(DDF ddf) throws DDFException { ddf.getRepresentationHandler().uncacheAll(); ddf.getRepresentationHandler().reset(); mDDFCache.removeDDF(ddf); }
public double rmse(DDF predictedDDF, boolean implicitPrefs) throws DDFException { RDD<Rating> predictions = (RDD<Rating>) predictedDDF.getRepresentationHandler().get(RDD.class, Rating.class); RDD<Rating> ratings = (RDD<Rating>) this.getDDF().getRepresentationHandler().get(RDD.class, Rating.class); return new ROCComputer().computeRmse(ratings, predictions, false); }
@Override /* * input expected RDD[double[][]] * (non-Javadoc) * @see io.ddf.ml.AMLMetricsSupporter#roc(io.ddf.DDF, int) */ public RocMetric roc(DDF predictionDDF, int alpha_length) throws DDFException { RDD<LabeledPoint> rddLabeledPoint = (RDD<LabeledPoint>) predictionDDF.getRepresentationHandler() .get(RDD.class, LabeledPoint.class); ROCComputer rc = new ROCComputer(); return (rc.ROC(rddLabeledPoint, alpha_length)); }
@Override public DDF updateInplace(DDF newddf) throws DDFException { //copy content of newddf to this ddf DDF curDDF = this.getDDF(); curDDF.getRepresentationHandler().reset(); curDDF.getRepresentationHandler().setRepresentations(newddf.getRepresentationHandler().getAllRepresentations()); newddf.getMetaDataHandler().copyFactor(this.getDDF()); curDDF.getSchemaHandler().setSchema(newddf.getSchema()); return curDDF; } }
@Override public DDF merge(DDF anotherDDF) throws DDFException { DataFrame rdd1 = ((DataFrame) this.getDDF().getRepresentationHandler().get(DataFrame.class)); DataFrame rdd2 = ((DataFrame) anotherDDF.getRepresentationHandler().get(DataFrame.class)); DataFrame newRDD = rdd1.unionAll(rdd2); Schema schema = SparkUtils.schemaFromDataFrame(newRDD); return this.getManager().newDDF(newRDD, new Class<?>[]{DataFrame.class}, null, schema); } }
@Override public DDF sql2ddf(String command, Schema schema, DataSourceDescriptor dataSource, DataFormat dataFormat) throws DDFException { // TableRDD tableRdd = null; // RDD<Row> rddRow = null; DataFrame rdd = this.getHiveContext().sql(command); if (schema == null) schema = SchemaHandler.getSchemaFromDataFrame(rdd); DDF ddf = this.getManager().newDDF(this.getManager(), rdd, new Class<?>[] {DataFrame.class}, null, schema); ddf.getRepresentationHandler().cache(false); ddf.getRepresentationHandler().get(new Class<?>[]{RDD.class, Row.class}); return ddf; }
@Test public void testDDFRepresentations() throws DDFException { DDF ddf = this.getTestDDF(); Assert.assertNotNull("DDF cannot be null", ddf); Object repr = ddf.getRepresentationHandler().getDefault(); Assert.assertNotNull("Representation cannot be null", repr); @SuppressWarnings("unchecked") List<Object[]> list = (List<Object[]>) repr; Assert.assertNotNull("List cannot be null", list); for (Object[] row : list) { Assert.assertNotNull("Row in list cannot be null", row); } }
@Override public Schema generateSchema() throws DDFException { if (this.getSchema() != null) return this.getSchema(); // Try to infer from the DDF's data Object data = this.getDDF().getRepresentationHandler().getDefault(); // TODO: for now, we'll just support the "null" case if (data == null) return new Schema(null, "null BLOB"); return null; }
rdd = (RDD<LabeledPoint>) this.getDDF().getRepresentationHandler().get(RDD.class, LabeledPoint.class); rdd = (RDD<Vector>) this.getDDF().getRepresentationHandler().get(RDD.class, Vector.class); } else if (paramInfo.paramMatches(double[].class)) { rdd = (RDD<double[]>) this.getDDF().getRepresentationHandler().get(RDD.class, double[].class); } else if (paramInfo.paramMatches(io.ddf.types.Vector.class)) { rdd = (RDD<io.ddf.types.Vector>) this.getDDF().getRepresentationHandler() .get(RDD.class, io.ddf.types.Vector.class); } else if (paramInfo.paramMatches(TupleMatrixVector.class)) { rdd = (RDD<TupleMatrixVector>) this.getDDF().getRepresentationHandler().get(RDD.class, TupleMatrixVector.class); } else if (paramInfo.paramMatches(Rating.class)) { rdd = (RDD<Rating>) this.getDDF().getRepresentationHandler().get(RDD.class, Rating.class); rdd = (RDD<Object[]>) this.getDDF().getRepresentationHandler().get(RDD.class, Object[].class);
@Override public DDF loadFromJDBC(JDBCDataSourceDescriptor dataSource) throws DDFException { SparkDDFManager sparkDDFManager = (SparkDDFManager)mDDFManager; HiveContext sqlContext = sparkDDFManager.getHiveContext(); JDBCDataSourceCredentials cred = (JDBCDataSourceCredentials)dataSource.getDataSourceCredentials(); String fullURL = dataSource.getDataSourceUri().getUri().toString(); if (cred.getUsername() != null && !cred.getUsername().equals("")) { fullURL += String.format("?user=%s&password=%s", cred.getUsername(), cred.getPassword()); } Map<String, String> options = new HashMap<String, String>(); options.put("url", fullURL); options.put("dbtable", dataSource.getDbTable()); DataFrame df = sqlContext.load("jdbc", options); DDF ddf = sparkDDFManager.newDDF(sparkDDFManager, df, new Class<?>[]{DataFrame.class}, null, SparkUtils.schemaFromDataFrame(df)); // TODO? ddf.getRepresentationHandler().get(RDD.class, Row.class); ddf.getMetaDataHandler().setDataSourceDescriptor(dataSource); return ddf; }
@SuppressWarnings("unchecked") @Override public Summary[] getSummaryImpl() throws DDFException { RDD<Object[]> rdd = (RDD<Object[]>) this.getDDF().getRepresentationHandler().get(RDD.class, Object[].class); JavaRDD<Object[]> data = rdd.toJavaRDD(); Summary[] stats = data.map(new GetSummaryMapper()).reduce(new GetSummaryReducer()); return stats; }
@Test public void testRepresentDDF() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = ddf.getRepresentationHandler(); handler.reset(); Assert.assertNull("There should not be any existing representations", handler.get(list.get(0).getClass())); handler.set(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.add(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.remove(List.class, String.class); Assert.assertNull("There should now be no representation of type <List,String>", handler.get(List.class, String.class)); }
IHandleRepresentations repHandler = this.getDDF().getRepresentationHandler(); if (columnIndexes.size() > 0) { try {
@Ignore public void testModelSerialize2DDF() throws DDFException { DummyModel dummyModel = new DummyModel(20, "dummymodel2"); Model model = new Model(dummyModel); DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = model.serialize2DDF(manager); Object obj = ddf.getRepresentationHandler().get(List.class, String.class); List<Schema.Column> cols = ddf.getSchema().getColumns(); List<String> lsString = (List<String>) obj; Assert.assertTrue(obj != null); Assert.assertTrue(obj instanceof List); Assert.assertTrue(ddf != null); APersistenceHandler.PersistenceUri uri = ddf.persist(); PersistenceHandler pHandler = new PersistenceHandler(null); DDF ddf2 = (DDF) pHandler.load(uri); Model model2 = Model.deserializeFromDDF((BasicDDF) ddf2); Assert.assertTrue(ddf2 != null); Assert.assertTrue(model2 != null); Assert.assertTrue(model2.getRawModel() instanceof DummyModel); } }
/** * Initialization to be done after constructor assignments, such as setting of the all-important DDFManager. */ protected void initialize(DDFManager manager, Object data, Class<?>[] typeSpecs, String name, Schema schema) throws DDFException { this.validateSchema(schema); this.setManager(manager); // this must be done first in case later stuff needs a manager if (typeSpecs != null) { this.getRepresentationHandler().set(data, typeSpecs); } this.getSchemaHandler().setSchema(schema); if(schema!= null && schema.getTableName() == null) { String tableName = this.getSchemaHandler().newTableName(); schema.setTableName(tableName); } manager.setDDFUUID(this, UUID.randomUUID()); if(!Strings.isNullOrEmpty(name)) manager.setDDFName(this, name); // Facades this.ML = new MLFacade(this, this.getMLSupporter()); this.VIEWS = new ViewsFacade(this, this.getViewHandler()); this.Transform = new TransformFacade(this, this.getTransformationHandler()); this.R = new RFacade(this, this.getAggregationHandler()); }