@SuppressWarnings("unchecked") public <T> List<T> getList(Class<T> rowType) throws DDFException { return (List<T>) this.getRepresentationHandler().get(List.class, rowType); }
@Test public void testNewRepHandler() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = new DummyRepresentationHandler(ddf); Double[] data = new Double[] { 0.0, 2.0, 1.0, 3.0, 4.0, 10.0, 11.0 }; handler.add(data, Double[].class); Object[] obj1 = (Object[]) handler.get(Object[].class); Integer[] obj2 = (Integer[]) handler.get(Integer[].class); Assert.assertNotNull(obj1); Assert.assertNotNull(obj2); Assert.assertEquals(handler.getAllRepresentations().size(), 3); String[] obj3 = (String[]) handler.get(String[].class); Assert.assertNotNull(obj3); Assert.assertEquals(handler.getAllRepresentations().size(), 4); } }
public IGetResult getRDD(Class<?>... acceptableUnitTypes) throws DDFException { if (acceptableUnitTypes == null || acceptableUnitTypes.length == 0) { throw new DDFException("Acceptable Unit Types must be specified"); } // Compile a list of acceptableTypeSpecs List<Class<?>[]> acceptableTypeSpecs = new ArrayList<Class<?>[]>(); for (Class<?> unitType : acceptableUnitTypes) { acceptableTypeSpecs.add(new Class<?>[] { RDD.class, unitType }); } return this.getRepresentationHandler().get(acceptableTypeSpecs.toArray(new Class<?>[0][])); }
@SuppressWarnings("unchecked") public <T> RDD<T> getRDD(Class<T> unitType) throws DDFException { Object obj = this.getRepresentationHandler().get(RDD.class, unitType); if (obj instanceof RDD<?>) return (RDD<T>) obj; else throw new DDFException("Unable to get RDD with unit type " + unitType); }
public void saveAsTable() throws DDFException { if (!this.isTable()) { DataFrame rdd = (DataFrame) this.getRepresentationHandler().get(DataFrame.class); if (rdd == null) { mLog.info("Could not create SchemaRDD for ddf"); mLog.info(String.format("Could not save ddf %s as table", this.getUUID().toString())); } else { mLog.info(String.format(">>>> register %s as table", this.getTableName())); rdd.registerTempTable(this.getTableName()); } } }
public double rmse(DDF predictedDDF, boolean implicitPrefs) throws DDFException { RDD<Rating> predictions = (RDD<Rating>) predictedDDF.getRepresentationHandler().get(RDD.class, Rating.class); RDD<Rating> ratings = (RDD<Rating>) this.getDDF().getRepresentationHandler().get(RDD.class, Rating.class); return new ROCComputer().computeRmse(ratings, predictions, false); }
@Override /* * input expected RDD[double[][]] * (non-Javadoc) * @see io.ddf.ml.AMLMetricsSupporter#roc(io.ddf.DDF, int) */ public RocMetric roc(DDF predictionDDF, int alpha_length) throws DDFException { RDD<LabeledPoint> rddLabeledPoint = (RDD<LabeledPoint>) predictionDDF.getRepresentationHandler() .get(RDD.class, LabeledPoint.class); ROCComputer rc = new ROCComputer(); return (rc.ROC(rddLabeledPoint, alpha_length)); }
@Override public DDF merge(DDF anotherDDF) throws DDFException { DataFrame rdd1 = ((DataFrame) this.getDDF().getRepresentationHandler().get(DataFrame.class)); DataFrame rdd2 = ((DataFrame) anotherDDF.getRepresentationHandler().get(DataFrame.class)); DataFrame newRDD = rdd1.unionAll(rdd2); Schema schema = SparkUtils.schemaFromDataFrame(newRDD); return this.getManager().newDDF(newRDD, new Class<?>[]{DataFrame.class}, null, schema); } }
@Test public void testRepresentDDF() throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = manager.newDDF(); IHandleRepresentations handler = ddf.getRepresentationHandler(); handler.reset(); Assert.assertNull("There should not be any existing representations", handler.get(list.get(0).getClass())); handler.set(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.add(list, List.class, String.class); Assert.assertNotNull("There should now be a representation of type <List,String>", handler.get(List.class, String.class)); handler.remove(List.class, String.class); Assert.assertNull("There should now be no representation of type <List,String>", handler.get(List.class, String.class)); }
rdd = (RDD<LabeledPoint>) this.getDDF().getRepresentationHandler().get(RDD.class, LabeledPoint.class); rdd = (RDD<Vector>) this.getDDF().getRepresentationHandler().get(RDD.class, Vector.class); } else if (paramInfo.paramMatches(double[].class)) { rdd = (RDD<double[]>) this.getDDF().getRepresentationHandler().get(RDD.class, double[].class); } else if (paramInfo.paramMatches(io.ddf.types.Vector.class)) { rdd = (RDD<io.ddf.types.Vector>) this.getDDF().getRepresentationHandler() .get(RDD.class, io.ddf.types.Vector.class); } else if (paramInfo.paramMatches(TupleMatrixVector.class)) { rdd = (RDD<TupleMatrixVector>) this.getDDF().getRepresentationHandler().get(RDD.class, TupleMatrixVector.class); } else if (paramInfo.paramMatches(Rating.class)) { rdd = (RDD<Rating>) this.getDDF().getRepresentationHandler().get(RDD.class, Rating.class); rdd = (RDD<Object[]>) this.getDDF().getRepresentationHandler().get(RDD.class, Object[].class);
@Override public DDF loadFromJDBC(JDBCDataSourceDescriptor dataSource) throws DDFException { SparkDDFManager sparkDDFManager = (SparkDDFManager)mDDFManager; HiveContext sqlContext = sparkDDFManager.getHiveContext(); JDBCDataSourceCredentials cred = (JDBCDataSourceCredentials)dataSource.getDataSourceCredentials(); String fullURL = dataSource.getDataSourceUri().getUri().toString(); if (cred.getUsername() != null && !cred.getUsername().equals("")) { fullURL += String.format("?user=%s&password=%s", cred.getUsername(), cred.getPassword()); } Map<String, String> options = new HashMap<String, String>(); options.put("url", fullURL); options.put("dbtable", dataSource.getDbTable()); DataFrame df = sqlContext.load("jdbc", options); DDF ddf = sparkDDFManager.newDDF(sparkDDFManager, df, new Class<?>[]{DataFrame.class}, null, SparkUtils.schemaFromDataFrame(df)); // TODO? ddf.getRepresentationHandler().get(RDD.class, Row.class); ddf.getMetaDataHandler().setDataSourceDescriptor(dataSource); return ddf; }
@SuppressWarnings("unchecked") @Override public Summary[] getSummaryImpl() throws DDFException { RDD<Object[]> rdd = (RDD<Object[]>) this.getDDF().getRepresentationHandler().get(RDD.class, Object[].class); JavaRDD<Object[]> data = rdd.toJavaRDD(); Summary[] stats = data.map(new GetSummaryMapper()).reduce(new GetSummaryReducer()); return stats; }
@Override public DDF sql2ddf(String command, Schema schema, DataSourceDescriptor dataSource, DataFormat dataFormat) throws DDFException { // TableRDD tableRdd = null; // RDD<Row> rddRow = null; DataFrame rdd = this.getHiveContext().sql(command); if (schema == null) schema = SchemaHandler.getSchemaFromDataFrame(rdd); DDF ddf = this.getManager().newDDF(this.getManager(), rdd, new Class<?>[] {DataFrame.class}, null, schema); ddf.getRepresentationHandler().cache(false); ddf.getRepresentationHandler().get(new Class<?>[]{RDD.class, Row.class}); return ddf; }
@Ignore public void testModelSerialize2DDF() throws DDFException { DummyModel dummyModel = new DummyModel(20, "dummymodel2"); Model model = new Model(dummyModel); DDFManager manager = DDFManager.get(DDFManager.EngineType.BASIC); DDF ddf = model.serialize2DDF(manager); Object obj = ddf.getRepresentationHandler().get(List.class, String.class); List<Schema.Column> cols = ddf.getSchema().getColumns(); List<String> lsString = (List<String>) obj; Assert.assertTrue(obj != null); Assert.assertTrue(obj instanceof List); Assert.assertTrue(ddf != null); APersistenceHandler.PersistenceUri uri = ddf.persist(); PersistenceHandler pHandler = new PersistenceHandler(null); DDF ddf2 = (DDF) pHandler.load(uri); Model model2 = Model.deserializeFromDDF((BasicDDF) ddf2); Assert.assertTrue(ddf2 != null); Assert.assertTrue(model2 != null); Assert.assertTrue(model2.getRawModel() instanceof DummyModel); } }