private void activateHighAvailabilitySupport(ApplicationSubmissionContext appContext) throws InvocationTargetException, IllegalAccessException { ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); reflector.setKeepContainersAcrossApplicationAttempts(appContext, true); reflector.setAttemptFailuresValidityInterval(appContext, AkkaUtils.getTimeout(flinkConfiguration).toMillis()); }
@Override public String getWebInterfaceURL() { final InetSocketAddress inetSocketAddressFromAkkaURL; try { inetSocketAddressFromAkkaURL = AkkaUtils.getInetSocketAddressFromAkkaURL(getClusterConnectionInfo().getAddress()); } catch (Exception e) { throw new RuntimeException("Could not retrieve leader retrieval information.", e); } String host = inetSocketAddressFromAkkaURL.getHostName(); int port = getFlinkConfiguration().getInteger(WebOptions.PORT); return "http://" + host + ":" + port; }
@BeforeClass public static void setup() { system = AkkaUtils.createLocalActorSystem(new Configuration()); }
public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices, ActorSystemLoader actorSystemLoader) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
final String akkaHostname = AkkaUtils.getAddress(actorSystem).host().get(); final int akkaPort = (Integer) AkkaUtils.getAddress(actorSystem).port().get(); AkkaUtils.getTimeout(config).toMillis(), TimeUnit.MILLISECONDS, futureExecutor,
ActorSystem as = AkkaUtils.createActorSystem( new Configuration(), new Some<scala.Tuple2<String, Object>>(new scala.Tuple2<String, Object>(addr.getHostAddress(), port)));
testActorSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0))); taskManagerSystem = AkkaUtils.createActorSystem( config, Option.apply(new Tuple2<String, Object>("localhost", 0))); TaskManager.startTaskManagerComponentsAndActor( ActorRef leaderRef = AkkaUtils.getActorRef( leaderAddress, testActorSystem, testDeadline.timeLeft()); ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
final Configuration configuration = new Configuration(); final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); final ActorSystem actorSystem = AkkaUtils.createDefaultActorSystem(); ActorRef actorRef = null; final UUID leaderId = UUID.randomUUID(); leaderId)); final String expectedAddress = AkkaUtils.getAkkaURL(actorSystem, actorRef);
scala.Option<Object> portOption = AkkaUtils.getAddress(getContext().system()).port(); int actorSystemPort = portOption.isDefined() ? (int) portOption.get() : -1;
/** * Submits a JobGraph detached. * @param jobGraph The JobGraph * @param classLoader User code class loader to deserialize the results and errors (may contain custom classes). * @return JobSubmissionResult * @throws ProgramInvocationException */ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { waitForClusterToBeReady(); final ActorGateway jobManagerGateway; try { jobManagerGateway = getJobManagerGateway(); } catch (Exception e) { throw new ProgramInvocationException("Failed to retrieve the JobManager gateway.", jobGraph.getJobID(), e); } try { logAndSysout("Submitting Job with JobID: " + jobGraph.getJobID() + ". Returning after job submission."); JobClient.submitJobDetached( new AkkaJobManagerGateway(jobManagerGateway), flinkConfig, jobGraph, Time.milliseconds(timeout.toMillis()), classLoader); return new JobSubmissionResult(jobGraph.getJobID()); } catch (JobExecutionException e) { throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), jobGraph.getJobID(), e); } }
configuration.setString(AkkaOptions.ASK_TIMEOUT, AkkaUtils.INF_TIMEOUT().toString());
public CliFrontend( Configuration configuration, List<CustomCommandLine<?>> customCommandLines) throws Exception { this.configuration = Preconditions.checkNotNull(configuration); this.customCommandLines = Preconditions.checkNotNull(customCommandLines); try { FileSystem.initialize(this.configuration); } catch (IOException e) { throw new Exception("Error while setting the default " + "filesystem scheme from configuration.", e); } this.customCommandLineOptions = new Options(); for (CustomCommandLine<?> customCommandLine : customCommandLines) { customCommandLine.addGeneralOptions(customCommandLineOptions); customCommandLine.addRunOptions(customCommandLineOptions); } this.clientTimeout = AkkaUtils.getClientTimeout(this.configuration); this.defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM); }
TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new RestartAllStrategy.Factory(), ExecutionGraph.class.getClassLoader(), VoidBlobWriter.getInstance(), AkkaUtils.getDefaultTimeout()); try { eg.attachJobGraph(jobVertices);
public MiniClusterResourceConfiguration( Configuration configuration, int numberTaskManagers, int numberSlotsPerTaskManager) { this( configuration, numberTaskManagers, numberSlotsPerTaskManager, AkkaUtils.getTimeoutAsTime(configuration)); }
@Override public Config getAkkaConfig() { return AkkaUtils.getThreadPoolExecutorConfig(this); } }
final long expectedSum = Parallelism * sequenceEnd * (sequenceEnd + 1) / 2; final ActorSystem system = RobustActorSystem.create("Test", AkkaUtils.getDefaultAkkaConfig()); final TestingServer testingServer = new TestingServer(); final TemporaryFolder temporaryFolder = new TemporaryFolder();
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down */ public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = new LazyActorSystemLoader( highAvailabilityServices, Time.milliseconds(lookupTimeout.toMillis()), flinkConfig, log); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
this.akkaDuration = AkkaUtils.getTimeout(flinkConfig); this.clusterDescriptor = clusterDescriptor; this.numberTaskManagers = numberTaskManagers;
@Before public void setUp() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(); env.generateSequence(1, 1000).output(new DiscardingOutputFormat<Long>()); Plan plan = env.createProgramPlan(); JobWithJars jobWithJars = new JobWithJars(plan, Collections.<URL>emptyList(), Collections.<URL>emptyList()); program = mock(PackagedProgram.class); when(program.getPlanWithJars()).thenReturn(jobWithJars); final int freePort = NetUtils.getAvailablePort(); config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, "localhost"); config.setInteger(JobManagerOptions.PORT, freePort); config.setString(AkkaOptions.ASK_TIMEOUT, AkkaOptions.ASK_TIMEOUT.defaultValue()); try { scala.Tuple2<String, Object> address = new scala.Tuple2<String, Object>("localhost", freePort); jobManagerSystem = AkkaUtils.createActorSystem(config, new scala.Some<scala.Tuple2<String, Object>>(address)); } catch (Exception e) { e.printStackTrace(); fail("Setup of test actor system failed."); } }
/** * Creates a {@link MesosServices} instance depending on the high availability settings. * * @param configuration containing the high availability settings * @param hostname the hostname to advertise to remote clients * @return a mesos services instance * @throws Exception if the mesos services instance could not be created */ public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception { ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration); MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname); HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration); switch (highAvailabilityMode) { case NONE: return new StandaloneMesosServices(localActorSystem, artifactServer); case ZOOKEEPER: final String zkMesosRootPath = configuration.getString( HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH); ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory( configuration, zkMesosRootPath); return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory); default: throw new Exception("High availability mode " + highAvailabilityMode + " is not supported."); } }