/** * Uses existing {@link SparkSession} if present, else creates a new one */ public synchronized SparkSession getSparkSession( @NonNull final SparkArgs sparkArgs, final boolean enableHiveSupport) { if (this.sparkSession.isPresent()) { return sparkSession.get(); } final Builder sparkSessionBuilder = SparkSession.builder(); if (enableHiveSupport) { sparkSessionBuilder.enableHiveSupport(); } this.sparkSession = Optional.of(sparkSessionBuilder .config(createSparkConf(sparkArgs)).getOrCreate()); log.info("Created new SparkSession using {}", sparkArgs); updateSparkContext(sparkArgs, this.sparkSession.get().sparkContext()); return this.sparkSession.get(); }
SparkSession sc = SparkSession.builder() .appName(appName) .enableHiveSupport() .getOrCreate();
private SparkSession session() { if (System.getenv("SPARK_HOME") != null) { SparkSession sc; if (builder.getEnableHive()) { sc = SparkSession.builder() .appName(builder.getAppName()) .master(builder.getMaster()) .enableHiveSupport() .getOrCreate(); } else { sc = SparkSession.builder() .appName(builder.getAppName()) .master(builder.getMaster()) .getOrCreate(); } LOGGER .debug("Initialize SparkContext successfully, App name: {}", builder.getAppName()); return sc; } else { LOGGER.error( "Initialize SparkContext failed, the reason for which is not find spark env"); throw new RuntimeException("No available Spark to execute. Please deploy Spark and put Spark_HOME in env"); } }
private static JavaRDD<String[]> getOtherFormatHiveInput(JavaSparkContext sc, String hiveTable) { SparkSession sparkSession = SparkSession.builder().config(sc.getConf()).enableHiveSupport().getOrCreate(); final Dataset intermediateTable = sparkSession.table(hiveTable); return intermediateTable.javaRDD().map(new Function<Row, String[]>() { @Override public String[] call(Row row) throws Exception { String[] result = new String[row.size()]; for (int i = 0; i < row.size(); i++) { final Object o = row.get(i); if (o != null) { result[i] = o.toString(); } else { result[i] = null; } } return result; } }); }
private static void initializeBatchJob() { SparkConf sparkConf = getSparkConfiguration(INSTANCE.config, INSTANCE.mode); if (!sparkConf.contains("spark.master")) { LOG.warn("Spark master not provided, instead using local mode"); sparkConf.setMaster("local[*]"); } if (!sparkConf.contains("spark.app.name")) { LOG.warn("Spark application name not provided, instead using empty string"); sparkConf.setAppName(""); } SparkSession.Builder sparkSessionBuilder = SparkSession.builder(); if (enablesHiveSupport()) { sparkSessionBuilder.enableHiveSupport(); } INSTANCE.ss = sparkSessionBuilder.config(sparkConf).getOrCreate(); }
/** * Constructs the all important HiveContext, then evaluates the wrapped Statement. * Currently, the HiveContext is made as a singleton. * * @throws Throwable as required by the Statement class */ @Override public void evaluate() throws Throwable { if (sparkSqlContextSingleton == null) { SparkConf sparkConf = new SparkConf().setAppName("HiveQLUnit").setMaster(serverAddress); SparkSession sparkSession = SparkSession.builder().config(sparkConf).enableHiveSupport().getOrCreate(); sparkSqlContextSingleton = sparkSession.sqlContext(); } wrappedStatement.evaluate(); }
@Override public TryResult configure(KernelFunctionality kernel, SparkUIApi sparkUI, Message parentMessage) { SparkConf sparkConf = createSparkConf(sparkUI.getAdvancedOptions(), getSparkConfBasedOn(this.sparkSessionBuilder)); sparkConf = configureSparkConf(sparkConf, sparkUI); this.sparkSessionBuilder = SparkSession.builder().config(sparkConf); if (sparkUI.getHiveSupport()) { this.sparkSessionBuilder.enableHiveSupport(); } TryResult sparkSessionTry = createSparkSession(sparkUI, parentMessage); if (sparkSessionTry.isError()) { return sparkSessionTry; } addListener(getOrCreate().sparkContext(), sparkUI); SparkVariable.putSparkSession(getOrCreate()); TryResult tryResultSparkContext = initSparkContextInShell(kernel, parentMessage); if (!tryResultSparkContext.isError()) { kernel.registerCancelHook(SparkVariable::cancelAllJobs); } return tryResultSparkContext; }
private static JavaRDD<String[]> getOtherFormatHiveInput(JavaSparkContext sc, String hiveTable) { SparkSession sparkSession = SparkSession.builder().config(sc.getConf()).enableHiveSupport().getOrCreate(); final Dataset intermediateTable = sparkSession.table(hiveTable); return intermediateTable.javaRDD().map(new Function<Row, String[]>() { @Override public String[] call(Row row) throws Exception { String[] result = new String[row.size()]; for (int i = 0; i < row.size(); i++) { final Object o = row.get(i); if (o != null) { result[i] = o.toString(); } else { result[i] = null; } } return result; } }); }