MiniClusterWithClientResource get() { return new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlotsPerTaskManager) .build()); } }
@AfterClass public static void tearDownClass() { if (miniClusterResource != null) { miniClusterResource.after(); } }
@BeforeClass public static void setUp() throws Exception { Configuration config = new Configuration(); // we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again. config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem"); config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, FOLDER.newFolder().getAbsoluteFile().toURI().toString()); // Savepoint path config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, FOLDER.newFolder().getAbsoluteFile().toURI().toString()); // required as we otherwise run out of memory config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m"); miniClusterResource = new MiniClusterResource( new MiniClusterResourceConfiguration.Builder() .setNumberTaskManagers(2) .setNumberSlotsPerTaskManager(2) .setConfiguration(config) .build()); miniClusterResource.before(); }
@Override public void before() throws Exception { super.before(); clusterClient = createMiniClusterClient(); executionEnvironment = new TestEnvironment(getMiniCluster(), getNumberSlots(), false); executionEnvironment.setAsContext(); TestStreamEnvironment.setAsContext(getMiniCluster(), getNumberSlots()); }
@Test public void testCustomSplitJobWithCustomClassLoaderPath() throws IOException, ProgramInvocationException { URL classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL(); PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.<Path>emptyList(), Collections.singleton(classpath)); inputSplitTestProg2.invokeInteractiveModeForExecution(); }
@Before public void setUp() throws Exception { final Configuration clientConfig = new Configuration(); clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0); clientConfig.setLong(RestOptions.RETRY_DELAY, 0); clientConfig.addAll(CLUSTER.getClientConfiguration()); client = new RestClusterClient<>( clientConfig, StandaloneClusterId.getInstance() ); }
MockResourceManagerRuntimeServices() throws Exception { scheduledExecutor = mock(ScheduledExecutor.class); highAvailabilityServices = new TestingHighAvailabilityServices(); rmLeaderElectionService = new TestingLeaderElectionService(); highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService); heartbeatServices = new TestingHeartbeatServices(5L, 5L, scheduledExecutor); metricRegistry = NoOpMetricRegistry.INSTANCE; slotManager = new SlotManager( new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()), Time.seconds(10), Time.seconds(10), Time.minutes(1)); jobLeaderIdService = new JobLeaderIdService( highAvailabilityServices, rpcService.getScheduledExecutor(), Time.minutes(5L)); }
protected SavepointMigrationTestBase() throws Exception { miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfiguration()) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(DEFAULT_PARALLELISM) .build()); }
@Override public void after() { TestStreamEnvironment.unsetAsContext(); TestEnvironment.unsetAsContext(); Exception exception = null; if (clusterClient != null) { try { clusterClient.shutdown(); } catch (Exception e) { exception = e; } } clusterClient = null; super.after(); if (exception != null) { log.warn("Could not properly shut down the MiniClusterWithClientResource.", exception); } }
@Test public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException { PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); inputSplitTestProg.invokeInteractiveModeForExecution(); }
protected final MiniClusterWithClientResource getMiniClusterResource() { return new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfigurationSafe()) .setNumberTaskManagers(2) .setNumberSlotsPerTaskManager(PARALLELISM / 2) .build()); }
@Test public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { // regular streaming job PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingProg.invokeInteractiveModeForExecution(); }
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
@Test public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingInputSplitTestProg.invokeInteractiveModeForExecution(); }
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); // we have to manage this manually because we have to create the ZooKeeper server // ahead of this miniClusterResource = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(getConfig()) .setNumberTaskManagers(NUM_TMS) .setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM) .build()); miniClusterResource.before(); client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); clusterClient = miniClusterResource.getClusterClient(); }
@Test public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(USERCODETYPE_JAR_PATH)), Collections.<URL>emptyList()); userCodeTypeProg.invokeInteractiveModeForExecution(); }
@Before public void setUp() throws Exception { jmDirectory = TMP.newFolder("jm"); hsDirectory = TMP.newFolder("hs"); Configuration clusterConfig = new Configuration(); clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(clusterConfig) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); }
@Test public void testKMeansJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram kMeansProg = new PackagedProgram( new File(KMEANS_JAR_PATH), new String[] { KMeansData.DATAPOINTS, KMeansData.INITIAL_CENTERS, "25" }); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(KMEANS_JAR_PATH)), Collections.<URL>emptyList()); kMeansProg.invokeInteractiveModeForExecution(); }
private MiniClusterWithClientResource trySetUpCluster() throws Exception { try { Configuration config = new Configuration(); config.setString(TRANSPORT_TYPE, "epoll"); MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(NUM_TASK_MANAGERS) .setNumberSlotsPerTaskManager(1) .build()); cluster.before(); return cluster; } catch (UnsatisfiedLinkError ex) { // If we failed to init netty because we are not on Linux platform, abort the test. if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) { throw new AssumptionViolatedException("This test is only supported on linux"); } throw ex; } } }
@Before public void setup() throws Exception { // detect parameter change if (currentBackend != backend) { shutDownExistingCluster(); currentBackend = backend; Configuration config = new Configuration(); final File checkpointDir = temporaryFolder.newFolder(); final File savepointDir = temporaryFolder.newFolder(); config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend); config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(numTaskManagers) .setNumberSlotsPerTaskManager(numSlots) .build()); cluster.before(); } }