public Dataset<Row> resetDataFrame(GeoWaveRDD pairRDD) { rowRDD = null; dataFrame = null; return getDataFrame(pairRDD); } }
private void loadStoresAndViews() throws IOException { final Collection<InputStoreInfo> addStores = inputStores.values(); for (final InputStoreInfo storeInfo : addStores) { final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().addTypeName(storeInfo.typeName).build()); storeInfo.rdd = GeoWaveRDDLoader.loadRDD(session.sparkContext(), storeInfo.storeOptions, rddOpts); // Create a DataFrame from the Left RDD final SimpleFeatureDataFrame dataFrame = new SimpleFeatureDataFrame(session); if (!dataFrame.init(storeInfo.storeOptions, storeInfo.typeName)) { LOGGER.error("Failed to initialize dataframe"); return; } LOGGER.debug(dataFrame.getSchema().json()); final Dataset<Row> dfTemp = dataFrame.getDataFrame(storeInfo.rdd); dfTemp.createOrReplaceTempView(storeInfo.viewName); } }
final SimpleFeatureDataFrame leftResultFrame = new SimpleFeatureDataFrame(session); final SimpleFeatureDataFrame rightResultFrame = new SimpleFeatureDataFrame(session); leftResultFrame.init(leftStore.storeOptions, leftStore.typeName); rightResultFrame.init(rightStore.storeOptions, rightStore.typeName); final Dataset<Row> leftFrame = leftResultFrame.getDataFrame(joinRunner.getLeftResults()); final Dataset<Row> rightFrame = rightResultFrame.getDataFrame(joinRunner.getRightResults()); leftFrame.createOrReplaceTempView(leftStore.viewName); rightFrame.createOrReplaceTempView(rightStore.viewName);