protected final JavaStreamingContext buildStreamingContext() { log.info("Starting SparkContext with interval {} seconds", generationIntervalSec); SparkConf sparkConf = new SparkConf(); // Only for tests, really if (sparkConf.getOption("spark.master").isEmpty()) { log.info("Overriding master to {} for tests", streamingMaster); sparkConf.setMaster(streamingMaster); } // Only for tests, really if (sparkConf.getOption("spark.app.name").isEmpty()) { String appName = "Oryx" + getLayerName(); if (id != null) { appName = appName + "-" + id; } log.info("Overriding app name to {} for tests", appName); sparkConf.setAppName(appName); } extraSparkConfig.forEach((key, value) -> sparkConf.setIfMissing(key, value.toString())); // Turn this down to prevent long blocking at shutdown sparkConf.setIfMissing( "spark.streaming.gracefulStopTimeout", Long.toString(TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS))); sparkConf.setIfMissing("spark.cleaner.ttl", Integer.toString(20 * generationIntervalSec)); long generationIntervalMS = TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS); JavaSparkContext jsc = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate(sparkConf)); return new JavaStreamingContext(jsc, new Duration(generationIntervalMS)); }
conf.setIfMissing(AnalyticsConstants.SPARK_APP_NAME, DEFAULT_SPARK_APP_NAME); conf.setIfMissing(AnalyticsConstants.SPARK_DRIVER_CORES, "1"); conf.setIfMissing(AnalyticsConstants.SPARK_DRIVER_MEMORY, "512m"); conf.setIfMissing(AnalyticsConstants.SPARK_EXECUTOR_MEMORY, "512m"); conf.setIfMissing(AnalyticsConstants.SPARK_UI_PORT, "4040"); conf.setIfMissing(AnalyticsConstants.SPARK_HISTORY_OPTS, "18080"); conf.setIfMissing(AnalyticsConstants.SPARK_SERIALIZER, KryoSerializer.class.getName()); conf.setIfMissing(AnalyticsConstants.SPARK_KRYOSERIALIZER_BUFFER, "256k"); conf.setIfMissing(AnalyticsConstants.SPARK_KRYOSERIALIZER_BUFFER_MAX, "256m"); conf.setIfMissing("spark.blockManager.port", "12000"); conf.setIfMissing("spark.broadcast.port", "12500"); conf.setIfMissing("spark.driver.port", "13000"); conf.setIfMissing("spark.executor.port", "13500"); conf.setIfMissing("spark.fileserver.port", "14000"); conf.setIfMissing("spark.replClassServer.port", "14500"); conf.setIfMissing(AnalyticsConstants.SPARK_MASTER_PORT, "7077"); conf.setIfMissing("spark.master.rest.port", "6066"); conf.setIfMissing(AnalyticsConstants.SPARK_MASTER_WEBUI_PORT, "8081"); conf.setIfMissing(AnalyticsConstants.SPARK_WORKER_CORES, "1"); conf.setIfMissing(AnalyticsConstants.SPARK_WORKER_MEMORY, "1g"); conf.setIfMissing(AnalyticsConstants.SPARK_WORKER_DIR, "work"); conf.setIfMissing(AnalyticsConstants.SPARK_WORKER_PORT, "11000"); conf.setIfMissing(AnalyticsConstants.SPARK_WORKER_WEBUI_PORT, "11500");
protected final JavaStreamingContext buildStreamingContext() { log.info("Starting SparkContext with interval {} seconds", generationIntervalSec); SparkConf sparkConf = new SparkConf(); // Only for tests, really if (sparkConf.getOption("spark.master").isEmpty()) { log.info("Overriding master to {} for tests", streamingMaster); sparkConf.setMaster(streamingMaster); } // Only for tests, really if (sparkConf.getOption("spark.app.name").isEmpty()) { String appName = "Oryx" + getLayerName(); if (id != null) { appName = appName + "-" + id; } log.info("Overriding app name to {} for tests", appName); sparkConf.setAppName(appName); } extraSparkConfig.forEach((key, value) -> sparkConf.setIfMissing(key, value.toString())); // Turn this down to prevent long blocking at shutdown sparkConf.setIfMissing( "spark.streaming.gracefulStopTimeout", Long.toString(TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS))); sparkConf.setIfMissing("spark.cleaner.ttl", Integer.toString(20 * generationIntervalSec)); long generationIntervalMS = TimeUnit.MILLISECONDS.convert(generationIntervalSec, TimeUnit.SECONDS); JavaSparkContext jsc = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate(sparkConf)); return new JavaStreamingContext(jsc, new Duration(generationIntervalMS)); }