final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().addTypeName(adapterTypeName).build()); rddOpts.setMinSplits(partCount); rddOpts.setMaxSplits(partCount);
private void loadStoresAndViews() throws IOException { final Collection<InputStoreInfo> addStores = inputStores.values(); for (final InputStoreInfo storeInfo : addStores) { final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().addTypeName(storeInfo.typeName).build()); storeInfo.rdd = GeoWaveRDDLoader.loadRDD(session.sparkContext(), storeInfo.storeOptions, rddOpts); // Create a DataFrame from the Left RDD final SimpleFeatureDataFrame dataFrame = new SimpleFeatureDataFrame(session); if (!dataFrame.init(storeInfo.storeOptions, storeInfo.typeName)) { LOGGER.error("Failed to initialize dataframe"); return; } LOGGER.debug(dataFrame.getSchema().json()); final Dataset<Row> dfTemp = dataFrame.getDataFrame(storeInfo.rdd); dfTemp.createOrReplaceTempView(storeInfo.viewName); } }
public static GeoWaveRDD loadRDD(final SparkContext sc, final DataStorePluginOptions storeOptions) throws IOException { final RDDOptions defaultOptions = new RDDOptions(); return GeoWaveRDDLoader.loadRDD(sc, storeOptions, defaultOptions); }
if (rddOpts.getQuery() != null) { GeoWaveInputFormat.setQuery( conf, rddOpts.getQuery(), storeOptions.createAdapterStore(), storeOptions.createInternalAdapterStore(), if ((rddOpts.getMinSplits() > -1) || (rddOpts.getMaxSplits() > -1)) { GeoWaveInputFormat.setMinimumSplitCount(conf, rddOpts.getMinSplits()); GeoWaveInputFormat.setMaximumSplitCount(conf, rddOpts.getMaxSplits()); } else { final int defaultSplitsSpark = sc.getConf().getInt("spark.default.parallelism", -1);
bldr.indexName(indexName); RDDOptions rddOptions = new RDDOptions(); rddOptions.setQuery(bldr.constraints(bldr.constraintsFactory().cqlConstraints(cqlStr)).build()); try { count =
sparkConf.setMaster("local"); final JavaSparkContext context = new JavaSparkContext(sparkConf); final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().constraints(query).build()); rddOpts.setMinSplits(minSplits); rddOpts.setMaxSplits(maxSplits); final JavaPairRDD<GeoWaveInputKey, SimpleFeature> javaRdd = GeoWaveRDDLoader.loadRDD(context.sc(), inputStoreOptions, rddOpts).getRawRDD();
final RDDOptions queryOpts = new RDDOptions(); queryOpts.setQuery(QueryBuilder.newBuilder().constraints(query).build()); queryOpts.setMinSplits(DEFAULT_SPLITS_FOR_COUNT); queryOpts.setMaxSplits(DEFAULT_SPLITS_FOR_COUNT); final GeoWaveRDD newRDD = GeoWaveRDDLoader.loadRDD(context, dataStore, queryOpts); final JavaPairRDD<GeoWaveInputKey, SimpleFeature> javaRdd = newRDD.getRawRDD();
final RDDOptions kmeansOpts = new RDDOptions(); kmeansOpts.setMinSplits(minSplits); kmeansOpts.setMaxSplits(maxSplits); kmeansOpts.setQuery(bldr.build()); final GeoWaveRDD kmeansRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kmeansOpts);