/** * Launch application using the application DAG. * Notice that we launch the DAG one at a time, as the result of a DAG has to be immediately returned to the * Java variable before the application can be resumed. * * @param dag the application DAG. */ // When modifying the signature of this method, see CompilerTestUtil#compileDAG and make corresponding changes public static void launchDAG(final DAG dag) { launchDAG(dag, Collections.emptyMap(), ""); }
builtJobConf = getJobConf(args); .run(); final Configuration driverConf = getDriverConf(builtJobConf); final Configuration driverNcsConf = getDriverNcsConf(); final Configuration driverMessageConfig = getDriverMessageConf(); final String defaultExecutorResourceConfig = "[{\"type\":\"Transient\",\"memory_mb\":512,\"capacity\":5}," + "{\"type\":\"Reserved\",\"memory_mb\":512,\"capacity\":5}]"; final Configuration executorResourceConfig = getJSONConf(builtJobConf, JobConf.ExecutorJSONPath.class, JobConf.ExecutorJSONContents.class, defaultExecutorResourceConfig); final Configuration bandwidthConfig = getJSONConf(builtJobConf, JobConf.BandwidthJSONPath.class, JobConf.BandwidthJSONContents.class, ""); final Configuration clientConf = getClientConf(); final Configuration schedulerConf = getSchedulerConf(builtJobConf); deployModeConf = Configurations.merge(getDeployModeConf(builtJobConf), clientConf);
/** * Collect data by running the DAG. * * @param dag the DAG to execute. * @param loopVertexStack loop vertex stack. * @param lastVertex last vertex added to the dag. * @param serializer serializer for the edges. * @param <T> type of the return data. * @return the data collected. */ public static <T> List<T> collect(final DAG<IRVertex, IREdge> dag, final Stack<LoopVertex> loopVertexStack, final IRVertex lastVertex, final Serializer serializer) { final DAGBuilder<IRVertex, IREdge> builder = new DAGBuilder<>(dag); final IRVertex collectVertex = new OperatorVertex(new CollectTransform<>()); builder.addVertex(collectVertex, loopVertexStack); final IREdge newEdge = new IREdge(getEdgeCommunicationPattern(lastVertex, collectVertex), lastVertex, collectVertex); newEdge.setProperty(EncoderProperty.of(new SparkEncoderFactory(serializer))); newEdge.setProperty(DecoderProperty.of(new SparkDecoderFactory(serializer))); newEdge.setProperty(SPARK_KEY_EXTRACTOR_PROP); builder.connectVertices(newEdge); // launch DAG JobLauncher.launchDAG(builder.build(), SparkBroadcastVariables.getAll(), ""); return (List<T>) JobLauncher.getCollectedData(); }
private static DAG<IRVertex, IREdge> compileDAG(final String[] args) throws Exception { final String userMainClassName; final String[] userMainMethodArgs; try { final Configuration configuration = JobLauncher.getJobConf(args); final Injector injector = Tang.Factory.getTang().newInjector(configuration); userMainClassName = injector.getNamedInstance(JobConf.UserMainClass.class); userMainMethodArgs = injector.getNamedInstance(JobConf.UserMainArguments.class).split(" "); } catch (final Exception e) { throw new RuntimeException("An exception occurred while processing configuration for invoking user main. " + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking " + "on JobLauncher and because of this Tang may misbehave afterwards.", e); } final Class userMainClass = Class.forName(userMainClassName); final Method userMainMethod = userMainClass.getMethod("main", String[].class); final ArgumentCaptor<DAG> captor = ArgumentCaptor.forClass(DAG.class); final ArgumentCaptor<String> stringArg = ArgumentCaptor.forClass(String.class); PowerMockito.mockStatic(JobLauncher.class); PowerMockito.doNothing().when(JobLauncher.class, "launchDAG", captor.capture(), stringArg.capture()); userMainMethod.invoke(null, (Object) userMainMethodArgs); return captor.getValue(); }
@Override public SparkSession getOrCreate() { if (!options.containsKey("spark.master")) { // default spark_master option. return this.master("local[*]").getOrCreate(); } if (!options.containsKey("spark.driver.allowMultipleContexts")) { return this.config("spark.driver.allowMultipleContexts", "true").getOrCreate(); } UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser("ubuntu")); // Set up spark context with given options. final SparkConf sparkConf = new SparkConf(); if (!options.containsKey("spark.app.name")) { try { // get and override configurations from JobLauncher. final Configuration configurations = JobLauncher.getBuiltJobConf(); final Injector injector = Tang.Factory.getTang().newInjector(configurations); options.put("spark.app.name", injector.getNamedInstance(JobConf.JobId.class)); } catch (final InjectionException e) { throw new RuntimeException(e); } } options.forEach(sparkConf::set); final SparkContext sparkContext = new org.apache.nemo.compiler.frontend.spark.core.SparkContext(sparkConf); super.sparkContext(sparkContext); return SparkSession.from(super.getOrCreate(), this.options); } }
/** * @param dag the application DAG. * @param jobId job ID. */ public static void launchDAG(final DAG dag, final String jobId) { launchDAG(dag, Collections.emptyMap(), jobId); }
/** * Method to run the Pipeline. * @param pipeline the Pipeline to run. * @return The result of the pipeline. */ public NemoPipelineResult run(final Pipeline pipeline) { final PipelineVisitor pipelineVisitor = new PipelineVisitor(pipeline, nemoPipelineOptions); pipeline.traverseTopologically(pipelineVisitor); final NemoPipelineResult nemoPipelineResult = new NemoPipelineResult(); JobLauncher.launchDAG(pipelineVisitor.getConvertedPipeline(), nemoPipelineOptions.getJobName()); return nemoPipelineResult; } }