/** * Creates JavaSparkContext if its hasn't been created yet, or returns the instance. {@link #addSchema(Schema)} and * {@link #addSchemas(Collection)} must not be called once the JavaSparkContext has been created * @return the JavaSparkContext that will be used to execute the JobDags */ public JavaSparkContext getOrCreateSparkContext() { if (!this.sparkContext.isPresent()) { this.sparkContext = Optional.of(new JavaSparkContext( SparkUtil.getSparkConf( this.appName, Optional.of(this.schemas), this.serializationClasses, this.conf))); this.sparkContext.get().sc().addSparkListener(new SparkEventListener()); // Adding hadoop configuration to default this.sparkContext.get().sc().hadoopConfiguration().addResource( new HadoopConfiguration(conf).getHadoopConf()); this.appId = this.sparkContext.get().sc().applicationId(); } return this.sparkContext.get(); }
@Test public void testSparkConfOverrideDoesNotFailWithoutAnySparkConfDefinitions() { final Configuration conf = new Configuration(new File(TestConfiguration.CONFIG_YAML), Optional.absent()); SparkUtil .getSparkConf("fooApp", Optional.absent(), Arrays.asList(), conf); } }
@Test public void testSparkConfOverride() { final Configuration conf = new Configuration( TestSparkUtil.class.getResourceAsStream("/configWithScopes.yaml"), Optional.of("incremental")); final SparkConf sparkConf = SparkUtil .getSparkConf("fooApp", Optional.absent(), Arrays.asList(), conf); Assert.assertEquals("4g", sparkConf.get("spark.executor.memory")); Assert.assertEquals("4g", sparkConf.get("spark.driver.memory")); Assert.assertEquals("100s", sparkConf.get("spark.network.timeout")); }