/** * Returns the next available HTTP response . A call to this method blocks until a response * becomes available or throws an Exception if the timeout fires. * * @param timeout Timeout in milliseconds for the next response to become available * @return The next available {@link SimpleHttpResponse} */ public SimpleHttpResponse getNextResponse(FiniteDuration timeout) throws InterruptedException, TimeoutException { SimpleHttpResponse response = responses.poll(timeout.toMillis(), TimeUnit.MILLISECONDS); if (response == null) { throw new TimeoutException("No response within timeout of " + timeout + " ms"); } else { return response; } }
@Override public void cancel(JobID jobID) throws Exception { JobTerminationMessageParameters params = new JobTerminationMessageParameters(); params.jobPathParameter.resolve(jobID); params.terminationModeQueryParameter.resolve(Collections.singletonList(TerminationModeQueryParameter.TerminationMode.CANCEL)); CompletableFuture<EmptyResponseBody> responseFuture = sendRequest( JobTerminationHeaders.getInstance(), params); responseFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS); }
@Override public void stop(JobID jobID) throws Exception { JobTerminationMessageParameters params = new JobTerminationMessageParameters(); params.jobPathParameter.resolve(jobID); params.terminationModeQueryParameter.resolve(Collections.singletonList(TerminationModeQueryParameter.TerminationMode.STOP)); CompletableFuture<EmptyResponseBody> responseFuture = sendRequest( JobTerminationHeaders.getInstance(), params); responseFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS); }
private void activateHighAvailabilitySupport(ApplicationSubmissionContext appContext) throws InvocationTargetException, IllegalAccessException { ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); reflector.setKeepContainersAcrossApplicationAttempts(appContext, true); reflector.setAttemptFailuresValidityInterval(appContext, AkkaUtils.getTimeout(flinkConfiguration).toMillis()); }
/** * Sends a {@link JobManagerMessages.DisposeSavepoint} message to the job manager. */ private void disposeSavepoint(ClusterClient<?> clusterClient, String savepointPath) throws FlinkException { Preconditions.checkNotNull(savepointPath, "Missing required argument: savepoint path. " + "Usage: bin/flink savepoint -d <savepoint-path>"); logAndSysout("Disposing savepoint '" + savepointPath + "'."); final CompletableFuture<Acknowledge> disposeFuture = clusterClient.disposeSavepoint(savepointPath); logAndSysout("Waiting for response..."); try { disposeFuture.get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS); } catch (Exception e) { throw new FlinkException("Disposing the savepoint '" + savepointPath + "' failed.", e); } logAndSysout("Savepoint '" + savepointPath + "' disposed."); }
private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, FiniteDuration timeLeft) throws ExecutionException, InterruptedException { FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())), Time.milliseconds(50L), org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())), clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= numberOfTaskManagers, new ScheduledExecutorServiceAdapter(Executors.newSingleThreadScheduledExecutor())) .get(); }
@Override public Map<String, OptionalFailure<Object>> getAccumulators(final JobID jobID, ClassLoader loader) throws Exception { final JobAccumulatorsHeaders accumulatorsHeaders = JobAccumulatorsHeaders.getInstance(); final JobAccumulatorsMessageParameters accMsgParams = accumulatorsHeaders.getUnresolvedMessageParameters(); accMsgParams.jobPathParameter.resolve(jobID); accMsgParams.includeSerializedAccumulatorsParameter.resolve(Collections.singletonList(true)); CompletableFuture<JobAccumulatorsInfo> responseFuture = sendRequest( accumulatorsHeaders, accMsgParams); Map<String, OptionalFailure<Object>> result = Collections.emptyMap(); try { result = responseFuture.thenApply((JobAccumulatorsInfo accumulatorsInfo) -> { try { return AccumulatorHelper.deserializeAccumulators( accumulatorsInfo.getSerializedUserAccumulators(), loader); } catch (Exception e) { throw new CompletionException( new FlinkException( String.format("Deserialization of accumulators for job %s failed.", jobID), e)); } }).get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (ExecutionException ee) { ExceptionUtils.rethrowException(ExceptionUtils.stripExecutionException(ee)); } return result; }
/** * Sends a request to to the server. * * <pre> * HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/overview"); * request.headers().set(HttpHeaders.Names.HOST, host); * request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE); * * sendRequest(request); * </pre> * * @param request The {@link HttpRequest} to send to the server */ public void sendRequest(HttpRequest request, FiniteDuration timeout) throws InterruptedException, TimeoutException { LOG.debug("Writing {}.", request); // Make the connection attempt. ChannelFuture connect = bootstrap.connect(host, port); Channel channel; if (connect.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) { channel = connect.channel(); } else { throw new TimeoutException("Connection failed"); } channel.writeAndFlush(request); }
String jobmanagerSavepointPath = savepointPathFuture.get(DEADLINE.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
Collection<JobStatusMessage> jobs = clusterClient.listJobs().get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); for (JobStatusMessage job : jobs) { if (job.getJobState() == JobStatus.RUNNING) { try { savepointPath = clusterClient.triggerSavepoint(jobId, null) .get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); } catch (Exception cause) { LOG.info("Failed to trigger savepoint. Retrying...", cause); invokeThread.join(deadline.timeLeft().toMillis()); assertFalse("Program invoke thread still running", invokeThread.isAlive());
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down */ public ClusterClient( Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); this.timeout = AkkaUtils.getClientTimeout(flinkConfig); this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig); this.actorSystemLoader = new LazyActorSystemLoader( highAvailabilityServices, Time.milliseconds(lookupTimeout.toMillis()), flinkConfig, log); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); this.sharedHaServices = sharedHaServices; }
futures.add(client.sendRequest(serverAddress, request)); ByteBuf buf = received.poll(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); assertNotNull("Receive timed out", buf); buf.release(); buf = received.poll(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); assertNotNull("Receive timed out", buf); buf.release(); futures.remove(0).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); fail("Did not throw expected server failure"); } catch (ExecutionException e) { futures.remove(0).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); fail("Did not throw expected server failure"); } catch (ExecutionException e) {
List<CompletableFuture<KvStateResponse>> results = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); for (CompletableFuture<KvStateResponse> result : results) { KvStateResponse actual = result.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); assertArrayEquals(serializedResult, actual.getContent());
/** * Submits a JobGraph detached. * @param jobGraph The JobGraph * @param classLoader User code class loader to deserialize the results and errors (may contain custom classes). * @return JobSubmissionResult * @throws ProgramInvocationException */ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { waitForClusterToBeReady(); final ActorGateway jobManagerGateway; try { jobManagerGateway = getJobManagerGateway(); } catch (Exception e) { throw new ProgramInvocationException("Failed to retrieve the JobManager gateway.", jobGraph.getJobID(), e); } try { logAndSysout("Submitting Job with JobID: " + jobGraph.getJobID() + ". Returning after job submission."); JobClient.submitJobDetached( new AkkaJobManagerGateway(jobManagerGateway), flinkConfig, jobGraph, Time.milliseconds(timeout.toMillis()), classLoader); return new JobSubmissionResult(jobGraph.getJobID()); } catch (JobExecutionException e) { throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), jobGraph.getJobID(), e); } }
future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); fail("Did not throw expected ConnectException"); } catch (ExecutionException e) {
ByteBuf buf = received.poll(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); assertNotNull("Receive timed out", buf); KvStateResponse serializedResult = futures.get((int) i).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); assertArrayEquals(expected, serializedResult.getContent()); } else { try { futures.get((int) i).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); fail("Did not throw expected Exception"); } catch (ExecutionException e) {
thread.join(deadline.timeLeft().toMillis());