@AfterClass public static void tearDownClass() { if (miniClusterResource != null) { miniClusterResource.after(); } }
@Override public void before() throws Exception { super.before(); clusterClient = createMiniClusterClient(); executionEnvironment = new TestEnvironment(getMiniCluster(), getNumberSlots(), false); executionEnvironment.setAsContext(); TestStreamEnvironment.setAsContext(getMiniCluster(), getNumberSlots()); }
@Test public void testCustomSplitJobWithCustomClassLoaderPath() throws IOException, ProgramInvocationException { URL classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL(); PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.<Path>emptyList(), Collections.singleton(classpath)); inputSplitTestProg2.invokeInteractiveModeForExecution(); }
config.setString(WebOptions.UPLOAD_DIR, uploadDir.toString()); MiniClusterResource clusterResource = new MiniClusterResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberSlotsPerTaskManager(1) .build()); clusterResource.before(); Configuration clientConfig = clusterResource.getClientConfiguration(); RestClient client = new RestClient(RestClientConfiguration.fromConfiguration(clientConfig), TestingUtils.defaultExecutor()); clusterResource.after();
@BeforeClass public static void setUp() throws Exception { Configuration config = new Configuration(); // we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again. config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem"); config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, FOLDER.newFolder().getAbsoluteFile().toURI().toString()); // Savepoint path config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, FOLDER.newFolder().getAbsoluteFile().toURI().toString()); // required as we otherwise run out of memory config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m"); miniClusterResource = new MiniClusterResource( new MiniClusterResourceConfiguration.Builder() .setNumberTaskManagers(2) .setNumberSlotsPerTaskManager(2) .setConfiguration(config) .build()); miniClusterResource.before(); }
@Before public void setUp() throws Exception { final Configuration clientConfig = new Configuration(); clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0); clientConfig.setLong(RestOptions.RETRY_DELAY, 0); clientConfig.addAll(CLUSTER.getClientConfiguration()); client = new RestClusterClient<>( clientConfig, StandaloneClusterId.getInstance() ); }
MINI_CLUSTER_RESOURCE.getClientConfiguration(), StandaloneClusterId.getInstance());
@Override public void after() { TestStreamEnvironment.unsetAsContext(); TestEnvironment.unsetAsContext(); Exception exception = null; if (clusterClient != null) { try { clusterClient.shutdown(); } catch (Exception e) { exception = e; } } clusterClient = null; super.after(); if (exception != null) { log.warn("Could not properly shut down the MiniClusterWithClientResource.", exception); } }
@Override public void before() throws Exception { super.before(); clusterClient = createMiniClusterClient(); executionEnvironment = new TestEnvironment(getMiniCluster(), getNumberSlots(), false); executionEnvironment.setAsContext(); TestStreamEnvironment.setAsContext(getMiniCluster(), getNumberSlots()); }
@Test public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException { PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); inputSplitTestProg.invokeInteractiveModeForExecution(); }
@Override public void after() { TestStreamEnvironment.unsetAsContext(); TestEnvironment.unsetAsContext(); Exception exception = null; if (clusterClient != null) { try { clusterClient.shutdown(); } catch (Exception e) { exception = e; } } clusterClient = null; super.after(); if (exception != null) { log.warn("Could not properly shut down the MiniClusterWithClientResource.", exception); } }
@Override public void before() throws Exception { super.before(); clusterClient = createMiniClusterClient(); executionEnvironment = new TestEnvironment(getMiniCluster(), getNumberSlots(), false); executionEnvironment.setAsContext(); TestStreamEnvironment.setAsContext(getMiniCluster(), getNumberSlots()); }
@Test public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingInputSplitTestProg.invokeInteractiveModeForExecution(); }
@Override public void after() { TestStreamEnvironment.unsetAsContext(); TestEnvironment.unsetAsContext(); Exception exception = null; if (clusterClient != null) { try { clusterClient.shutdown(); } catch (Exception e) { exception = e; } } clusterClient = null; super.after(); if (exception != null) { log.warn("Could not properly shut down the MiniClusterWithClientResource.", exception); } }
@Test public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(USERCODETYPE_JAR_PATH)), Collections.<URL>emptyList()); userCodeTypeProg.invokeInteractiveModeForExecution(); }
@Test public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { // regular streaming job PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingProg.invokeInteractiveModeForExecution(); }
@Test public void testKMeansJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram kMeansProg = new PackagedProgram( new File(KMEANS_JAR_PATH), new String[] { KMeansData.DATAPOINTS, KMeansData.INITIAL_CENTERS, "25" }); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(KMEANS_JAR_PATH)), Collections.<URL>emptyList()); kMeansProg.invokeInteractiveModeForExecution(); }
@Test public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException { File checkpointDir = FOLDER.newFolder(); File outputDir = FOLDER.newFolder(); final PackagedProgram program = new PackagedProgram( new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH), new String[] { checkpointDir.toURI().toString(), outputDir.toURI().toString() }); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)), Collections.<URL>emptyList()); expectedException.expectCause( Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class))); program.invokeInteractiveModeForExecution(); }
miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_CHECKPOINTED_PROG_JAR_FILE)),
ClusterClient<?> clusterClient = new MiniClusterClient(new Configuration(), miniClusterResource.getMiniCluster()); miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CUSTOM_KV_STATE_JAR_PATH)),