public Dataset<Row> resetDataFrame(GeoWaveRDD pairRDD) { rowRDD = null; dataFrame = null; return getDataFrame(pairRDD); } }
private String[] getTableRelations(final String[] predicateArgs) { final String[] outputRelations = { getTableNameFromArg(predicateArgs[0].trim()), getTableNameFromArg(predicateArgs[1].trim())}; return outputRelations; }
private Matcher getFirstPositiveMatcher(final Pattern compiledPattern, final String sql) { final Matcher returnMatch = compiledPattern.matcher(sql); return getNextPositiveMatcher(returnMatch); }
final File configFile = getGeoWaveConfigFile(params); final String cleanSql = initStores(configFile, sql, sparkSqlOptions.getOutputStoreName()); sqlRunner.setSql(cleanSql); sqlRunner.setAppName(sparkSqlOptions.getAppName()); sqlRunner.setHost(sparkSqlOptions.getHost()); sqlRunner.setMaster(sparkSqlOptions.getMaster()); final Dataset<Row> results = sqlRunner.run(); if (sparkSqlOptions.getShowResults() > 0) { results.show(sparkSqlOptions.getShowResults(), false); final SqlResultsWriter sqlResultsWriter = new SqlResultsWriter(results, outputDataStore); String typeName = sparkSqlOptions.getOutputTypeName(); if (typeName == null) { typeName = "sqlresults"; sqlResultsWriter.writeResults(typeName); JCommander.getConsole().println("Datastore write complete."); if (sparkSqlOptions.getCsvOutputFile() != null) { results.repartition(1).write().format("com.databricks.spark.csv").option( "header", "true").mode(SaveMode.Overwrite).save(sparkSqlOptions.getCsvOutputFile()); sqlRunner.close();
public Dataset<Row> run() throws IOException, InterruptedException, ExecutionException, ParseException { initContext(); loadStoresAndViews(); condObj.get("name").getAsJsonObject().get("funcName").getAsString(); final UDFNameAndConstructor geomUDF = UDFRegistrySPI.findFunctionByName(udfName); if (geomUDF != null) { final ExtractedGeomPredicate relevantPredicate = new ExtractedGeomPredicate(); relevantPredicate.predicate = geomUDF.getPredicateConstructor().get(); relevantPredicate.predicateName = udfName; extractedPredicates.add(relevantPredicate); final Pattern orderGroupDetect = Pattern.compile("(?i)(\"[^\"]*\"|'[^']*')|(\\bORDER BY|GROUP BY\\b)"); final Matcher filterStart = getFirstPositiveMatcher(whereDetect, sql); if (filterStart == null) { LOGGER.error("There should be a where clause matching the pattern. Running default SQL"); return runDefaultSQL(); getFirstPositiveMatcher(orderGroupDetect, sql.substring(whereStart)); if (filterEnd != null) { whereEnd = filterEnd.start(2); final Matcher compoundFilter = getFirstPositiveMatcher(andOrDetect, filterClause); if (compoundFilter != null) { LOGGER.warn(
private void loadStoresAndViews() throws IOException { final Collection<InputStoreInfo> addStores = inputStores.values(); for (final InputStoreInfo storeInfo : addStores) { final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().addTypeName(storeInfo.typeName).build()); storeInfo.rdd = GeoWaveRDDLoader.loadRDD(session.sparkContext(), storeInfo.storeOptions, rddOpts); // Create a DataFrame from the Left RDD final SimpleFeatureDataFrame dataFrame = new SimpleFeatureDataFrame(session); if (!dataFrame.init(storeInfo.storeOptions, storeInfo.typeName)) { LOGGER.error("Failed to initialize dataframe"); return; } LOGGER.debug(dataFrame.getSchema().json()); final Dataset<Row> dfTemp = dataFrame.getDataFrame(storeInfo.rdd); dfTemp.createOrReplaceTempView(storeInfo.viewName); } }
public static StructType schemaFromFeatureType(SimpleFeatureType featureType) { List<StructField> fields = new ArrayList<>(); for (AttributeDescriptor attrDesc : featureType.getAttributeDescriptors()) { SimpleFeatureDataType sfDataType = attrDescToDataType(attrDesc); String fieldName = (sfDataType.isGeom() ? "geom" : attrDesc.getName().getLocalPart()); StructField field = DataTypes.createStructField(fieldName, sfDataType.getDataType(), true); fields.add(field); } if (fields.isEmpty()) { LOGGER.error("Feature type produced empty dataframe schema!"); return null; } return DataTypes.createStructType(fields); }
final String escapedDelimRegex = java.util.regex.Pattern.quote(STORE_ADAPTER_DELIM); Matcher matchedStore = getFirstPositiveMatcher(storeDetect, sql); String replacedSQL = sql; viewName = sqlRunner.addInputStore(storeOptions, adapterName, viewName); if (viewName != null) { replacedSQL = StringUtils.replace(replacedSQL, originalStoreText, viewName, -1); matchedStore = getNextPositiveMatcher(matchedStore);
final InputStoreInfo inputInfo = new InputStoreInfo(storeOptions, addTypeName, addView); inputStores.put(addView, inputInfo); return addView;
private static SparkSession internalCreateSession(SparkConf conf, SparkConf addonOptions) { // Create initial SessionBuilder from default Configuration. Builder builder = SparkSession.builder().config(conf); // Ensure SpatialEncoders and UDTs are registered at each session // creation. GeoWaveSpatialEncoders.registerUDTs(); if (addonOptions != null) { builder = builder.config(addonOptions); } return builder.getOrCreate(); } }
public Dataset<Row> getDataFrame(GeoWaveRDD pairRDD) { if (rowRDD == null) { SimpleFeatureMapper mapper = new SimpleFeatureMapper(schema); rowRDD = pairRDD.getRawRDD().values().map(mapper); } if (dataFrame == null) { dataFrame = sparkSession.createDataFrame(rowRDD, schema); } return dataFrame; }
private static SimpleFeatureDataType attrDescToDataType(AttributeDescriptor attrDesc) { boolean isGeom = false; DataType dataTypeOut = DataTypes.NullType; if (attrDesc.getType().getBinding().equals(String.class)) { dataTypeOut = DataTypes.StringType; } else if (attrDesc.getType().getBinding().equals(Double.class)) { dataTypeOut = DataTypes.DoubleType; } else if (attrDesc.getType().getBinding().equals(Float.class)) { dataTypeOut = DataTypes.FloatType; } else if (attrDesc.getType().getBinding().equals(Long.class)) { dataTypeOut = DataTypes.LongType; } else if (attrDesc.getType().getBinding().equals(Integer.class)) { dataTypeOut = DataTypes.IntegerType; } else if (attrDesc.getType().getBinding().equals(Boolean.class)) { dataTypeOut = DataTypes.BooleanType; } else if (attrDesc.getType().getBinding().equals(Date.class)) { dataTypeOut = DataTypes.TimestampType; } // Custom geometry types get WKB encoding else if (Geometry.class.isAssignableFrom(attrDesc.getType().getBinding())) { dataTypeOut = GeoWaveSpatialEncoders.geometryUDT; isGeom = true; } return new SimpleFeatureDataType(dataTypeOut, isGeom); } }