@Test public void testCustomSplitJobWithCustomClassLoaderPath() throws IOException, ProgramInvocationException { URL classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL(); PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.<Path>emptyList(), Collections.singleton(classpath)); inputSplitTestProg2.invokeInteractiveModeForExecution(); }
@Test public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException { PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); inputSplitTestProg.invokeInteractiveModeForExecution(); }
@Test public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingInputSplitTestProg.invokeInteractiveModeForExecution(); }
@Test public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(USERCODETYPE_JAR_PATH)), Collections.<URL>emptyList()); userCodeTypeProg.invokeInteractiveModeForExecution(); }
@Test public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { // regular streaming job PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingProg.invokeInteractiveModeForExecution(); }
@Test public void testKMeansJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram kMeansProg = new PackagedProgram( new File(KMEANS_JAR_PATH), new String[] { KMeansData.DATAPOINTS, KMeansData.INITIAL_CENTERS, "25" }); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(KMEANS_JAR_PATH)), Collections.<URL>emptyList()); kMeansProg.invokeInteractiveModeForExecution(); }
@Test public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException { File checkpointDir = FOLDER.newFolder(); File outputDir = FOLDER.newFolder(); final PackagedProgram program = new PackagedProgram( new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH), new String[] { checkpointDir.toURI().toString(), outputDir.toURI().toString() }); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)), Collections.<URL>emptyList()); expectedException.expectCause( Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class))); program.invokeInteractiveModeForExecution(); }
miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_CHECKPOINTED_PROG_JAR_FILE)),
ClusterClient<?> clusterClient = new MiniClusterClient(new Configuration(), miniClusterResource.getMiniCluster()); miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CUSTOM_KV_STATE_JAR_PATH)),
/** * Ensure that the program parallelism can be set even if the configuration is supplied. */ @Test public void testUserSpecificParallelism() throws Exception { Configuration config = new Configuration(); config.setString(AkkaOptions.STARTUP_TIMEOUT, VALID_STARTUP_TIMEOUT); final URI restAddress = MINI_CLUSTER_RESOURCE.getMiniCluster().getRestAddress(); final String hostname = restAddress.getHost(); final int port = restAddress.getPort(); final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment( hostname, port, config ); env.setParallelism(USER_DOP); env.getConfig().disableSysoutLogging(); DataSet<Integer> result = env.createInput(new ParallelismDependentInputFormat()) .rebalance() .mapPartition(new RichMapPartitionFunction<Integer, Integer>() { @Override public void mapPartition(Iterable<Integer> values, Collector<Integer> out) throws Exception { out.collect(getRuntimeContext().getIndexOfThisSubtask()); } }); List<Integer> resultCollection = result.collect(); assertEquals(USER_DOP, resultCollection.size()); }