public static GeoWaveRDD loadRDD(final SparkContext sc, final DataStorePluginOptions storeOptions) throws IOException { final RDDOptions defaultOptions = new RDDOptions(); return GeoWaveRDDLoader.loadRDD(sc, storeOptions, defaultOptions); }
public static GeoWaveRDD loadRDD( final SparkContext sc, final DataStorePluginOptions storeOptions, final RDDOptions rddOpts) throws IOException { final JavaPairRDD<GeoWaveInputKey, SimpleFeature> rawRDD = GeoWaveRDDLoader.loadRawRDD(sc, storeOptions, rddOpts); return new GeoWaveRDD(rawRDD); }
public static GeoWaveIndexedRDD loadIndexedRDD( final SparkContext sc, final GeoWaveRDD inputRDD, final NumericIndexStrategy indexStrategy) throws IOException { if ((inputRDD == null) || !inputRDD.isLoaded()) { return null; } // Index strategy can be expensive so we will broadcast it and store it Broadcast<NumericIndexStrategy> broadcastStrategy = null; if (indexStrategy != null) { broadcastStrategy = (Broadcast<NumericIndexStrategy>) RDDUtils.broadcastIndexStrategy(sc, indexStrategy); } final GeoWaveIndexedRDD returnRDD = new GeoWaveIndexedRDD(inputRDD, broadcastStrategy); return returnRDD; }
private void loadStoresAndViews() throws IOException { final Collection<InputStoreInfo> addStores = inputStores.values(); for (final InputStoreInfo storeInfo : addStores) { final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().addTypeName(storeInfo.typeName).build()); storeInfo.rdd = GeoWaveRDDLoader.loadRDD(session.sparkContext(), storeInfo.storeOptions, rddOpts); // Create a DataFrame from the Left RDD final SimpleFeatureDataFrame dataFrame = new SimpleFeatureDataFrame(session); if (!dataFrame.init(storeInfo.storeOptions, storeInfo.typeName)) { LOGGER.error("Failed to initialize dataframe"); return; } LOGGER.debug(dataFrame.getSchema().json()); final Dataset<Row> dfTemp = dataFrame.getDataFrame(storeInfo.rdd); dfTemp.createOrReplaceTempView(storeInfo.viewName); } }
/** * Translate a set of objects in a JavaRDD to SimpleFeatures and push to GeoWave * * @throws IOException */ public static void writeRDDToGeoWave( SparkContext sc, Index index, DataStorePluginOptions outputStoreOptions, FeatureDataAdapter adapter, GeoWaveRDD inputRDD) throws IOException { if (!inputRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return; } writeToGeoWave(sc, index, outputStoreOptions, adapter, inputRDD.getRawRDD().values()); }
public static GeoWaveIndexedRDD loadIndexedRDD( final SparkContext sc, final DataStorePluginOptions storeOptions, final RDDOptions rddOpts, final NumericIndexStrategy indexStrategy) throws IOException { final GeoWaveRDD wrappedRDD = GeoWaveRDDLoader.loadRDD(sc, storeOptions, rddOpts); if (wrappedRDD == null) { return null; } // Index strategy can be expensive so we will broadcast it and store it Broadcast<NumericIndexStrategy> broadcastStrategy = null; if (indexStrategy != null) { broadcastStrategy = (Broadcast<NumericIndexStrategy>) RDDUtils.broadcastIndexStrategy(sc, indexStrategy); } final GeoWaveIndexedRDD returnRDD = new GeoWaveIndexedRDD(wrappedRDD, broadcastStrategy); return returnRDD; }
public JavaPairRDD<ByteArray, Tuple2<GeoWaveInputKey, SimpleFeature>> getIndexedFeatureRDD( final double bufferAmount) { verifyParameters(); if (!geowaveRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return null; geowaveRDD.getRawRDD().flatMapToPair( new PairFlatMapFunction<Tuple2<GeoWaveInputKey, SimpleFeature>, ByteArray, Tuple2<GeoWaveInputKey, SimpleFeature>>() {
private void writeResultsToNewAdapter() throws IOException { if (outputStore != null) { final Index[] leftIndices = getIndicesForAdapter( leftStore, leftAdapterTypeName, leftInternalAdapterStore, leftIndexStore); final FeatureDataAdapter newLeftAdapter = createOutputAdapter(leftStore, leftAdapterTypeName, leftIndices, outLeftAdapterTypeName); final Index[] rightIndices = getIndicesForAdapter( rightStore, rightAdapterTypeName, rightInternalAdapterStore, rightIndexStore); final FeatureDataAdapter newRightAdapter = createOutputAdapter( rightStore, rightAdapterTypeName, rightIndices, outRightAdapterTypeName); // Write each feature set to new adapter and store using original // indexing methods. RDDUtils.writeRDDToGeoWave(sc, leftIndices, outputStore, newLeftAdapter, getLeftResults()); RDDUtils.writeRDDToGeoWave(sc, rightIndices, outputStore, newRightAdapter, getRightResults()); } }
public Dataset<Row> getDataFrame(GeoWaveRDD pairRDD) { if (rowRDD == null) { SimpleFeatureMapper mapper = new SimpleFeatureMapper(schema); rowRDD = pairRDD.getRawRDD().values().map(mapper); } if (dataFrame == null) { dataFrame = sparkSession.createDataFrame(rowRDD, schema); } return dataFrame; }
public boolean isLoaded() { return (getRawRDD() != null); } }
public static JavaRDD<Point> rddFeatureCentroids(GeoWaveRDD inputRDD) { if (!inputRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return null; } JavaRDD<Point> centroids = inputRDD.getRawRDD().values().map(feature -> { Geometry geom = (Geometry) feature.getDefaultGeometry(); return geom.getCentroid(); }); return centroids; }
public static SparkSession createDefaultSession() { SparkConf defaultConfig = GeoWaveSparkConf.getDefaultConfig(); return GeoWaveSparkConf.internalCreateSession(defaultConfig, null); }
@Override public void setup() throws Exception { if (defaultSession == null) { SparkConf addonOptions = new SparkConf(); addonOptions.setMaster("local[*]"); addonOptions.setAppName("CoreGeoWaveSparkITs"); defaultSession = GeoWaveSparkConf.createDefaultSession(addonOptions); if (defaultSession == null) { LOGGER.error("Unable to create default spark session for tests"); return; } } }
public JavaPairRDD<ByteArray, Tuple2<GeoWaveInputKey, Geometry>> getIndexedGeometryRDD() { return this.getIndexedGeometryRDD(0.0, false); }
public JavaPairRDD<ByteArray, Tuple2<GeoWaveInputKey, SimpleFeature>> getIndexedFeatureRDD() { return this.getIndexedFeatureRDD(0.0); }
public void reindex(final Broadcast<? extends NumericIndexStrategy> newIndexStrategy) { // Remove original indexing strategy if (indexStrategy != null) { indexStrategy.unpersist(); } indexStrategy = (Broadcast<NumericIndexStrategy>) newIndexStrategy; reset(); }
private void initContext() { if (session == null) { String jar = ""; try { jar = SqlQueryRunner.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath(); } catch (final URISyntaxException e) { LOGGER.error("Unable to set jar location in spark configuration", e); } session = GeoWaveSparkConf.createSessionFromParams(appName, master, host, jar); } }
public static void writeRDDToGeoWave( SparkContext sc, Index[] indices, DataStorePluginOptions outputStoreOptions, FeatureDataAdapter adapter, GeoWaveRDD inputRDD) throws IOException { if (!inputRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return; } for (int iStrategy = 0; iStrategy < indices.length; iStrategy += 1) { writeToGeoWave( sc, indices[iStrategy], outputStoreOptions, adapter, inputRDD.getRawRDD().values()); } }
public static SparkSession createDefaultSession(final SparkConf addonOptions) { SparkConf defaultConfig = GeoWaveSparkConf.getDefaultConfig(); return GeoWaveSparkConf.internalCreateSession(defaultConfig, addonOptions); }
String jars) { SparkConf defaultConfig = GeoWaveSparkConf.getDefaultConfig(); return GeoWaveSparkConf.internalCreateSession(defaultConfig, null);