public static GeoWaveIndexedRDD loadIndexedRDD( final SparkContext sc, final GeoWaveRDD inputRDD, final NumericIndexStrategy indexStrategy) throws IOException { if ((inputRDD == null) || !inputRDD.isLoaded()) { return null; } // Index strategy can be expensive so we will broadcast it and store it Broadcast<NumericIndexStrategy> broadcastStrategy = null; if (indexStrategy != null) { broadcastStrategy = (Broadcast<NumericIndexStrategy>) RDDUtils.broadcastIndexStrategy(sc, indexStrategy); } final GeoWaveIndexedRDD returnRDD = new GeoWaveIndexedRDD(inputRDD, broadcastStrategy); return returnRDD; }
private void writeResultsToNewAdapter() throws IOException { if (outputStore != null) { final Index[] leftIndices = getIndicesForAdapter( leftStore, leftAdapterTypeName, leftInternalAdapterStore, leftIndexStore); final FeatureDataAdapter newLeftAdapter = createOutputAdapter(leftStore, leftAdapterTypeName, leftIndices, outLeftAdapterTypeName); final Index[] rightIndices = getIndicesForAdapter( rightStore, rightAdapterTypeName, rightInternalAdapterStore, rightIndexStore); final FeatureDataAdapter newRightAdapter = createOutputAdapter( rightStore, rightAdapterTypeName, rightIndices, outRightAdapterTypeName); // Write each feature set to new adapter and store using original // indexing methods. RDDUtils.writeRDDToGeoWave(sc, leftIndices, outputStore, newLeftAdapter, getLeftResults()); RDDUtils.writeRDDToGeoWave(sc, rightIndices, outputStore, newRightAdapter, getRightResults()); } }
/** * Translate a set of objects in a JavaRDD to SimpleFeatures and push to GeoWave * * @throws IOException */ public static void writeRDDToGeoWave( SparkContext sc, Index index, DataStorePluginOptions outputStoreOptions, FeatureDataAdapter adapter, GeoWaveRDD inputRDD) throws IOException { if (!inputRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return; } writeToGeoWave(sc, index, outputStoreOptions, adapter, inputRDD.getRawRDD().values()); }
insertIds = RDDUtils.trimIndexIds(insertIds, geom, index);
public static JavaRDD<Vector> rddFeatureVectors( final GeoWaveRDD inputRDD, final String timeField, final ScaledTemporalRange scaledRange) { if (!inputRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return null; } JavaRDD<Vector> vectorRDD = inputRDD.getRawRDD().values().map(feature -> { Point centroid = ((Geometry) feature.getDefaultGeometry()).getCentroid(); int numValues = 2; Date time = null; if (timeField != null) { // if this is a ranged schema, we have to take the // midpoint if (timeField.contains("|")) { int pipeIndex = timeField.indexOf("|"); String startField = timeField.substring(0, pipeIndex); String endField = timeField.substring(pipeIndex + 1); Date start = (Date) feature.getAttribute(startField); Date end = (Date) feature.getAttribute(endField); long halfDur = (end.getTime() - start.getTime()) / 2; time = new Date(start.getTime() + halfDur); } else { time = (Date) feature.getAttribute(timeField);
insertIds = RDDUtils.trimIndexIds(insertIds, geom, index);
centroidVectors = RDDUtils.rddFeatureVectors(kmeansRDD, timeField, scaledTimeRange); centroidVectors.cache();
public static GeoWaveIndexedRDD loadIndexedRDD( final SparkContext sc, final DataStorePluginOptions storeOptions, final RDDOptions rddOpts, final NumericIndexStrategy indexStrategy) throws IOException { final GeoWaveRDD wrappedRDD = GeoWaveRDDLoader.loadRDD(sc, storeOptions, rddOpts); if (wrappedRDD == null) { return null; } // Index strategy can be expensive so we will broadcast it and store it Broadcast<NumericIndexStrategy> broadcastStrategy = null; if (indexStrategy != null) { broadcastStrategy = (Broadcast<NumericIndexStrategy>) RDDUtils.broadcastIndexStrategy(sc, indexStrategy); } final GeoWaveIndexedRDD returnRDD = new GeoWaveIndexedRDD(wrappedRDD, broadcastStrategy); return returnRDD; }
insertIds = RDDUtils.trimIndexIds(insertIds, geom, index);
public static void writeRDDToGeoWave( SparkContext sc, Index[] indices, DataStorePluginOptions outputStoreOptions, FeatureDataAdapter adapter, GeoWaveRDD inputRDD) throws IOException { if (!inputRDD.isLoaded()) { LOGGER.error("Must provide a loaded RDD."); return; } for (int iStrategy = 0; iStrategy < indices.length; iStrategy += 1) { writeToGeoWave( sc, indices[iStrategy], outputStoreOptions, adapter, inputRDD.getRawRDD().values()); } }
(Broadcast<TieredSFCIndexStrategy>) RDDUtils.broadcastIndexStrategy(sc, tieredStrategy);