private void writeResultsToNewAdapter() throws IOException { if (outputStore != null) { final Index[] leftIndices = getIndicesForAdapter( leftStore, leftAdapterTypeName, leftInternalAdapterStore, leftIndexStore); final FeatureDataAdapter newLeftAdapter = createOutputAdapter(leftStore, leftAdapterTypeName, leftIndices, outLeftAdapterTypeName); final Index[] rightIndices = getIndicesForAdapter( rightStore, rightAdapterTypeName, rightInternalAdapterStore, rightIndexStore); final FeatureDataAdapter newRightAdapter = createOutputAdapter( rightStore, rightAdapterTypeName, rightIndices, outRightAdapterTypeName); // Write each feature set to new adapter and store using original // indexing methods. RDDUtils.writeRDDToGeoWave(sc, leftIndices, outputStore, newLeftAdapter, getLeftResults()); RDDUtils.writeRDDToGeoWave(sc, rightIndices, outputStore, newRightAdapter, getRightResults()); } }
private void loadDatasets() throws IOException { if (leftStore != null) { if (leftRDD == null) { leftRDD = createRDDFromOptions( leftStore, leftAdapterTypeName, leftInternalAdapterStore, leftIndexStore); } } if (rightStore != null) { if (rightRDD == null) { rightRDD = createRDDFromOptions( rightStore, rightAdapterTypeName, rightInternalAdapterStore, rightIndexStore); } } }
final SpatialJoinRunner runner = new SpatialJoinRunner(); runner.setAppName(spatialJoinOptions.getAppName()); runner.setMaster(spatialJoinOptions.getMaster()); runner.setHost(spatialJoinOptions.getHost()); runner.setPartCount(spatialJoinOptions.getPartCount()); runner.setPredicate(predicate); runner.setLeftStore(leftDataStore); if (spatialJoinOptions.getLeftAdapterTypeName() != null) { runner.setLeftAdapterTypeName(spatialJoinOptions.getLeftAdapterTypeName()); runner.setRightStore(rightDataStore); if (spatialJoinOptions.getRightAdapterTypeName() != null) { runner.setRightAdapterTypeName(spatialJoinOptions.getRightAdapterTypeName()); runner.setOutputStore(outputDataStore); if (spatialJoinOptions.getOutputLeftAdapterTypeName() != null) { runner.setOutputLeftAdapterTypeName(spatialJoinOptions.getOutputLeftAdapterTypeName()); runner.setOutputRightAdapterTypeName(spatialJoinOptions.getOutputRightAdapterTypeName()); runner.run(); runner.close(); return null;
final SpatialJoinRunner joinRunner = new SpatialJoinRunner(session); joinRunner.setNegativeTest(negativePredicate); rightStrat = rightIndices[0].getIndexStrategy(); joinRunner.setLeftRDD( GeoWaveRDDLoader.loadIndexedRDD(session.sparkContext(), leftStore.rdd, leftStrat)); joinRunner.setRightRDD( GeoWaveRDDLoader.loadIndexedRDD(session.sparkContext(), rightStore.rdd, rightStrat)); joinRunner.setPredicate(pred.predicate); joinRunner.setLeftStore(leftStore.storeOptions); joinRunner.setRightStore(rightStore.storeOptions); joinRunner.run(); rightResultFrame.init(rightStore.storeOptions, rightStore.typeName); final Dataset<Row> leftFrame = leftResultFrame.getDataFrame(joinRunner.getLeftResults()); final Dataset<Row> rightFrame = rightResultFrame.getDataFrame(joinRunner.getRightResults()); leftFrame.createOrReplaceTempView(leftStore.viewName); rightFrame.createOrReplaceTempView(rightStore.viewName);
public void run() throws InterruptedException, ExecutionException, IOException { leftInternalAdapterStore = leftStore.createInternalAdapterStore(); rightInternalAdapterStore = rightStore.createInternalAdapterStore(); leftIndexStore = leftStore.createIndexStore(); rightIndexStore = rightStore.createIndexStore(); // Init context initContext(); // Load RDDs loadDatasets(); // Verify CRS match/transform possible verifyCRS(); // Run join joinStrategy.getJoinOptions().setNegativePredicate(negativeTest); joinStrategy.join(session, leftRDD, rightRDD, predicate); writeResultsToNewAdapter(); }
private FeatureDataAdapter createOutputAdapter( final DataStorePluginOptions originalOptions, final String originalTypeName, final Index[] indices, String outputTypeName) { if (outputTypeName == null) { outputTypeName = createDefaultAdapterTypeName(originalTypeName, originalOptions); } final FeatureDataAdapter newAdapter = FeatureDataUtils.cloneFeatureDataAdapter(originalOptions, originalTypeName, outputTypeName); newAdapter.init(indices); return newAdapter; }
getIndicesForAdapter(storeOptions, adapterTypeName, internalAdapterStore, indexStore); if (rddIndices.length > 0) { rddStrategy = rddIndices[0].getIndexStrategy();