@RetryOnFailure(times = 1) public void testCheckpointRecoveryFailure() throws Exception { final Deadline testDeadline = TestTimeOut.fromNow(); final String zooKeeperQuorum = ZooKeeper.getConnectString(); final String fileStateBackendPath = temporaryFolder.newFolder().toString(); fileStateBackendPath); config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 2); testActorSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0))); taskManagerSystem = AkkaUtils.createActorSystem( config, Option.apply(new Tuple2<String, Object>("localhost", 0))); TaskManager.startTaskManagerComponentsAndActor( config, leaderListener.waitForNewLeader(testDeadline.timeLeft().toMillis()); ActorRef leaderRef = AkkaUtils.getActorRef( leaderAddress, testActorSystem, testDeadline.timeLeft()); ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId); taskManagerSystem.shutdown(); testActorSystem.shutdown();
LOG.debug("keytabPath: {}", keytabPath); config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath); config.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal); final String amPortRange = config.getString( YarnConfigOptions.APPLICATION_MASTER_PORT); final String akkaHostname = AkkaUtils.getAddress(actorSystem).host().get(); final int akkaPort = (Integer) AkkaUtils.getAddress(actorSystem).port().get(); LOG); ActorRef resourceMaster = actorSystem.actorOf(resourceMasterProps); actorSystem.actorOf( Props.create(ProcessReaper.class, resourceMaster, LOG, ACTOR_DIED_EXIT_CODE), "YARN_Resource_Master_Process_Reaper"); actorSystem.actorOf( Props.create(ProcessReaper.class, jobManager, LOG, ACTOR_DIED_EXIT_CODE), "JobManager_Process_Reaper"); AkkaUtils.getTimeout(config).toMillis(), TimeUnit.MILLISECONDS, futureExecutor,
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down */ public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = new LazyActorSystemLoader( highAvailabilityServices, Time.milliseconds(lookupTimeout.toMillis()), flinkConfig, log); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices, ActorSystemLoader actorSystemLoader) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
@Override public <C extends RpcGateway, S extends RpcEndpoint<C>> C startServer(S rpcEndpoint) { checkNotNull(rpcEndpoint, "rpc endpoint"); checkState(!stopped, "RpcService is stopped"); actorRef = actorSystem.actorOf(akkaRpcActorProps, rpcEndpoint.getEndpointId()); actors.add(actorRef); final String address = AkkaUtils.getAkkaURL(actorSystem, actorRef); final String hostname; Option<String> host = actorRef.path().address().host(); if (host.isEmpty()) { hostname = "localhost"; } else { hostname = host.get();
public AkkaRpcService(final ActorSystem actorSystem, final Time timeout) { this.actorSystem = checkNotNull(actorSystem, "actor system"); this.timeout = checkNotNull(timeout, "timeout"); if (actorSystem.settings().config().hasPath(MAXIMUM_FRAME_SIZE_PATH)) { maximumFramesize = actorSystem.settings().config().getBytes(MAXIMUM_FRAME_SIZE_PATH); } else { // only local communication maximumFramesize = Long.MAX_VALUE; } Address actorSystemAddress = AkkaUtils.getAddress(actorSystem); if (actorSystemAddress.host().isDefined()) { address = actorSystemAddress.host().get(); } else { address = ""; } internalScheduledExecutor = new InternalScheduledExecutorImpl(actorSystem); }
boolean sysoutLogUpdates) { checkNotNull(jobID, "The jobID must not be null."); checkNotNull(jobManagerGateWay, "The jobManagerGateWay must not be null."); checkNotNull(configuration, "The configuration must not be null."); checkNotNull(actorSystem, "The actorSystem must not be null."); checkNotNull(highAvailabilityServices, "The high availability services must not be null."); sysoutLogUpdates); ActorRef jobClientActor = actorSystem.actorOf(jobClientActorProps); new Timeout(AkkaUtils.INF_TIMEOUT()));
private Tuple2<String, String> extractAddressHostname(ActorRef actorRef) { final String actorAddress = AkkaUtils.getAkkaURL(actorSystem, actorRef); final String hostname; Option<String> host = actorRef.path().address().host(); if (host.isEmpty()) { hostname = "localhost"; } else { hostname = host.get(); } return Tuple2.of(actorAddress, hostname); }
scala.Option<Object> portOption = AkkaUtils.getAddress(getContext().system()).port(); int actorSystemPort = portOption.isDefined() ? (int) portOption.get() : -1;
public StandaloneMiniCluster(Configuration configuration) throws Exception { this.configuration = Preconditions.checkNotNull(configuration); timeout = AkkaUtils.getTimeout(configuration); 0); port = configuration.getInteger(JobManagerOptions.PORT); highAvailabilityServices, LOCAL_HOSTNAME, Option.<String>empty(), true, TaskManager.class); taskManager, TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(), timeout.toMillis());
final HighAvailabilityServices haServices) throws Exception { checkNotNull(configuration); checkNotNull(haServices); String taskManagerHostname = configuration.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, null); Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis()); final int rpcPort = configuration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 0); Preconditions.checkState(rpcPort >= 0 && rpcPort <= 65535, "Invalid value for " + "'%s' (port for the TaskManager actor system) : %d - Leave config parameter empty or " + "use 0 to let the system choose port automatically.",
public static ActorSystem startJobClientActorSystem(Configuration config) throws IOException { LOG.info("Starting JobClient actor system"); Option<Tuple2<String, Object>> remoting = new Some<>(new Tuple2<String, Object>("", 0)); // start a remote actor system to listen on an arbitrary port ActorSystem system = AkkaUtils.createActorSystem(config, remoting); Address address = system.provider().getDefaultAddress(); String hostAddress = address.host().isDefined() ? NetUtils.ipAddressToUrlString(InetAddress.getByName(address.host().get())) : "(unknown)"; int port = address.port().isDefined() ? ((Integer) address.port().get()) : -1; LOG.info("Started JobClient actor system at " + hostAddress + ':' + port); return system; }
final HighAvailabilityServices haServices) throws Exception { checkNotNull(configuration); checkNotNull(haServices); String taskManagerHostname = configuration.getString(TaskManagerOptions.HOST); Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis()); final String portRangeDefinition = configuration.getString(TaskManagerOptions.RPC_PORT); return AkkaRpcServiceUtils.createRpcService(taskManagerHostname, portRangeDefinition, configuration);
final Address address = AkkaUtils.getAddress(actorSystem); config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, address.host().get()); config.setString(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, address.port().get().toString()); if (config.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0) >= 0) { logger.info("Starting JobManager Web Frontend"); String jobManagerAkkaURL = AkkaUtils.getAkkaURL(actorSystem, jobManager); monitor.start(jobManagerAkkaURL);
ActorSystem actorSystem) throws IOException, InterruptedException { this.leaderRetrievalService = checkNotNull(leaderRetrievalService); this.timeout = AkkaUtils.getTimeout(config); this.retriever = new JobManagerRetriever(this, actorSystem, AkkaUtils.getTimeout(config), timeout); this.cfg = new WebMonitorConfig(config); stackTraceSamples = new StackTraceSampleCoordinator(actorSystem.dispatcher(), 60000); int cleanUpInterval = config.getInteger( ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_CLEAN_UP_INTERVAL, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_CLEAN_UP_INTERVAL); int refreshInterval = config.getInteger( ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_REFRESH_INTERVAL, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_REFRESH_INTERVAL); int numSamples = config.getInteger( ConfigConstants.JOB_MANAGER_WEB_BACK_PRESSURE_NUM_SAMPLES, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_BACK_PRESSURE_NUM_SAMPLES);
Configuration config, HighAvailabilityServices highAvailabilityServices) throws Exception { Preconditions.checkNotNull(config, "Configuration"); String askTimeoutString = config.getString( ConfigConstants.AKKA_ASK_TIMEOUT, ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT); Duration timeout = FiniteDuration.apply(askTimeoutString); if (!timeout.isFinite()) { throw new IllegalConfigurationException(ConfigConstants.AKKA_ASK_TIMEOUT int lookupRetries = config.getInteger(QueryableStateOptions.CLIENT_LOOKUP_RETRIES); int lookupRetryDelayMillis = config.getInteger(QueryableStateOptions.CLIENT_LOOKUP_RETRY_DELAY); FiniteDuration.apply(lookupRetryDelayMillis, "ms")); this.actorSystem = AkkaUtils.createActorSystem(config, remoting); this.executionContext = actorSystem.dispatcher();
BlobServer blobServer) throws Exception { checkNotNull(config); checkNotNull(blobServer); config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER); timeout = AkkaUtils.getTimeout(config); } catch (NumberFormatException e) { throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage()); new StackTraceSampleCoordinator(futureExecutor, timeout.toMillis()); final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL); final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl( stackTraceSampleCoordinator, cleanUpInterval, config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES), config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL), Time.milliseconds(config.getInteger(WebOptions.BACKPRESSURE_DELAY)));
final Configuration configuration = new Configuration(); final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); final ActorSystem actorSystem = AkkaUtils.createDefaultActorSystem(); ActorRef actorRef = null; final UUID leaderId = UUID.randomUUID(); actorRef = actorSystem.actorOf( Props.create( JobClientActorTest.PlainActor.class, leaderId)); final String expectedAddress = AkkaUtils.getAkkaURL(actorSystem, actorRef); actorSystem.shutdown();
public CliFrontend( Configuration configuration, List<CustomCommandLine<?>> customCommandLines) throws Exception { this.configuration = Preconditions.checkNotNull(configuration); this.customCommandLines = Preconditions.checkNotNull(customCommandLines); try { FileSystem.initialize(this.configuration); } catch (IOException e) { throw new Exception("Error while setting the default " + "filesystem scheme from configuration.", e); } this.customCommandLineOptions = new Options(); for (CustomCommandLine<?> customCommandLine : customCommandLines) { customCommandLine.addGeneralOptions(customCommandLineOptions); customCommandLine.addRunOptions(customCommandLineOptions); } this.clientTimeout = AkkaUtils.getClientTimeout(this.configuration); this.defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM); }
@Before public void setUp() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(); env.generateSequence(1, 1000).output(new DiscardingOutputFormat<Long>()); Plan plan = env.createProgramPlan(); JobWithJars jobWithJars = new JobWithJars(plan, Collections.<URL>emptyList(), Collections.<URL>emptyList()); program = mock(PackagedProgram.class); when(program.getPlanWithJars()).thenReturn(jobWithJars); final int freePort = NetUtils.getAvailablePort(); config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, "localhost"); config.setInteger(JobManagerOptions.PORT, freePort); config.setString(AkkaOptions.ASK_TIMEOUT, AkkaOptions.ASK_TIMEOUT.defaultValue()); try { scala.Tuple2<String, Object> address = new scala.Tuple2<String, Object>("localhost", freePort); jobManagerSystem = AkkaUtils.createActorSystem(config, new scala.Some<scala.Tuple2<String, Object>>(address)); } catch (Exception e) { e.printStackTrace(); fail("Setup of test actor system failed."); } }