@Nonnull private RestClient createRestClient() throws ConfigurationException { return new RestClient(RestClientConfiguration.fromConfiguration(restConfig), executor) { @Override public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRequest( final String targetAddress, final int targetPort, final M messageHeaders, final U messageParameters, final R request) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request); } } }; }
default List<MessageHeaders> getSpecs() { Comparator<String> comparator = new RestServerEndpoint.RestHandlerUrlComparator.CaseInsensitiveOrderComparator(); return initializeHandlers(CompletableFuture.completedFuture(null)).stream() .map(tuple -> tuple.f0) .filter(spec -> spec instanceof MessageHeaders) .map(spec -> (MessageHeaders) spec) .sorted((spec1, spec2) -> comparator.compare(spec1.getTargetRestEndpointURL(), spec2.getTargetRestEndpointURL())) .collect(Collectors.toList()); } }
private <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRetriableRequest(M messageHeaders, U messageParameters, R request, Collection<FileUpload> filesToUpload, Predicate<Throwable> retryPredicate) { return retry(() -> getWebMonitorBaseUrl().thenCompose(webMonitorBaseUrl -> { try { return restClient.sendRequest(webMonitorBaseUrl.getHost(), webMonitorBaseUrl.getPort(), messageHeaders, messageParameters, request, filesToUpload); } catch (IOException e) { throw new CompletionException(e); } }), retryPredicate); }
final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor()); taskManagersInfoCompletableFuture = restClient.sendRequest( webURI.getHost(), webURI.getPort(), TaskManagersHeaders.getInstance(), EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()); taskManagerInfos = taskManagersInfo.getTaskManagerInfos(); assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager)); (double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize, is(closeTo(1.0, 0.15))); } finally { restClient.shutdown(TIMEOUT); clusterClient.shutdown();
private static String createHtmlEntry(MessageHeaders<?, ?, ?> spec) { String requestEntry = createMessageHtmlEntry( spec.getRequestClass(), EmptyRequestBody.class); String responseEntry = createMessageHtmlEntry( spec.getResponseClass(), EmptyResponseBody.class); String pathParameterList = createPathParameterHtmlList(spec.getUnresolvedMessageParameters().getPathParameters()); String queryParameterList = createQueryParameterHtmlList(spec.getUnresolvedMessageParameters().getQueryParameters()); sb.append(" <tbody>\n"); sb.append(" <tr>\n"); sb.append(" <td class=\"text-left\" colspan=\"2\"><h5><strong>" + spec.getTargetRestEndpointURL() + "</strong></h5></td>\n"); sb.append(" </tr>\n"); sb.append(" <tr>\n"); sb.append(" <td class=\"text-left\" style=\"width: 20%\">Verb: <code>" + spec.getHttpMethod() + "</code></td>\n"); sb.append(" <td class=\"text-left\">Response code: <code>" + spec.getResponseStatusCode() + "</code></td>\n"); sb.append(" </tr>\n"); sb.append(" <tr>\n"); sb.append(" <td colspan=\"2\">" + escapeCharacters(spec.getDescription()) + "</td>\n"); sb.append(" </tr>\n"); sb.append(" </tr>\n"); int reqHash = spec.getTargetRestEndpointURL().hashCode() + spec.getHttpMethod().name().hashCode() + spec.getRequestClass().getCanonicalName().hashCode(); int resHash = spec.getTargetRestEndpointURL().hashCode() + spec.getHttpMethod().name().hashCode() + spec.getResponseClass().getCanonicalName().hashCode();
public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws IOException { Preconditions.checkNotNull(configuration); this.restAddress = configuration.getRestAddress(); this.restBindAddress = configuration.getRestBindAddress(); this.restBindPort = configuration.getRestBindPort(); this.sslHandlerFactory = configuration.getSslHandlerFactory(); this.uploadDir = configuration.getUploadDir(); createUploadDir(uploadDir, log); this.maxContentLength = configuration.getMaxContentLength(); this.responseHeaders = configuration.getResponseHeaders(); terminationFuture = new CompletableFuture<>(); }
Collection<FileUpload> filesToUpload = new ArrayList<>(8); filesToUpload.add(new FileUpload(jobGraphFile, RestConstants.CONTENT_TYPE_BINARY)); filesToUpload.add(new FileUpload(Paths.get(jar.toUri()), RestConstants.CONTENT_TYPE_JAR)); artifactFileNames.add(new JobSubmitRequestBody.DistributedCacheFile(artifacts.getKey(), new Path(artifacts.getValue().filePath).getName())); filesToUpload.add(new FileUpload(Paths.get(artifacts.getValue().filePath), RestConstants.CONTENT_TYPE_BINARY)); final JobSubmitRequestBody requestBody = new JobSubmitRequestBody( jobGraphFile.getFileName().toString(), jarFileNames, JobSubmitHeaders.getInstance(), EmptyMessageParameters.getInstance(), requestAndFileUploads.f0, requestAndFileUploads.f1,
RestClient client = new RestClient(RestClientConfiguration.fromConfiguration(clientConfig), TestingUtils.defaultExecutor()); client.sendRequest(host, port, headers, parameters, new JarRunRequestBody()) .get(); } catch (Exception e) { if (expected.isPresent()) { assertTrue(expected.get().getMessage().contains("ProgramInvocationException")); assertTrue(expected.get().getMessage().contains("empty.jar'. zip file is empty")); } else { throw e; client.shutdown(Time.milliseconds(10));
public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws IOException { Preconditions.checkNotNull(configuration); this.restAddress = configuration.getRestAddress(); this.restBindAddress = configuration.getRestBindAddress(); this.restBindPort = configuration.getRestBindPort(); this.sslEngineFactory = configuration.getSslEngineFactory(); this.uploadDir = configuration.getUploadDir(); createUploadDir(uploadDir, log); this.maxContentLength = configuration.getMaxContentLength(); this.responseHeaders = configuration.getResponseHeaders(); terminationFuture = new CompletableFuture<>(); }
private static Map<String, String> getFlinkConfig(final String host, final int port) throws Exception { final ClusterConfigurationInfo clusterConfigurationInfoEntries = restClient.sendRequest( host, port, ClusterConfigurationInfoHeaders.getInstance()).get(); return clusterConfigurationInfoEntries.stream().collect(Collectors.toMap( ClusterConfigurationInfoEntry::getKey, ClusterConfigurationInfoEntry::getValue)); }
private static int getNumberOfSlotsPerTaskManager(final String host, final int port) throws Exception { final TaskManagersInfo taskManagersInfo = restClient.sendRequest( host, port, TaskManagersHeaders.getInstance()).get(); return taskManagersInfo.getTaskManagerInfos() .stream() .map(TaskManagerInfo::getNumberSlots) .findFirst() .orElse(0); }
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
@AfterClass public static void tearDown() throws Exception { try { YarnTestBase.teardown(); } finally { if (restClient != null) { restClient.shutdown(Time.seconds(5)); } if (restClientExecutor != null) { restClientExecutor.shutdownNow(); } } }
@Before public void setUp() throws Exception { restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(restConfig); mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway); executor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory(RestClusterClientTest.class.getSimpleName())); jobGraph = new JobGraph("testjob"); jobId = jobGraph.getJobID(); }
@VisibleForTesting RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy, @Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception { super(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); if (restClient != null) { this.restClient = restClient; } else { this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService); } this.waitStrategy = Preconditions.checkNotNull(waitStrategy); this.clusterId = Preconditions.checkNotNull(clusterId); if (webMonitorRetrievalService == null) { this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); } else { this.webMonitorRetrievalService = webMonitorRetrievalService; } this.dispatcherRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever(); this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); }
@Override public CompletableFuture<Void> closeAsync() { return shutdownInternally(Time.seconds(10L)); }
public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws IOException { Preconditions.checkNotNull(configuration); this.restAddress = configuration.getRestAddress(); this.restBindAddress = configuration.getRestBindAddress(); this.restBindPort = configuration.getRestBindPort(); this.sslHandlerFactory = configuration.getSslHandlerFactory(); this.uploadDir = configuration.getUploadDir(); createUploadDir(uploadDir, log); this.maxContentLength = configuration.getMaxContentLength(); this.responseHeaders = configuration.getResponseHeaders(); terminationFuture = new CompletableFuture<>(); }
@Override public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRequest( final String targetAddress, final int targetPort, final M messageHeaders, final U messageParameters, final R request) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request); } } };
public static RestClusterClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException { RestClientConfiguration restClientConfiguration = RestClientConfiguration.fromConfiguration(config); final long awaitLeaderTimeout = config.getLong(RestOptions.AWAIT_LEADER_TIMEOUT); final int retryMaxAttempts = config.getInteger(RestOptions.RETRY_MAX_ATTEMPTS); final long retryDelay = config.getLong(RestOptions.RETRY_DELAY); return new RestClusterClientConfiguration(restClientConfiguration, awaitLeaderTimeout, retryMaxAttempts, retryDelay); } }
@Override public void shutdown() { ExecutorUtils.gracefulShutdown(restClusterClientConfiguration.getRetryDelay(), TimeUnit.MILLISECONDS, retryExecutorService); this.restClient.shutdown(Time.seconds(5)); ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, this.executorService); try { webMonitorRetrievalService.stop(); } catch (Exception e) { log.error("An error occurred during stopping the webMonitorRetrievalService", e); } try { dispatcherRetrievalService.stop(); } catch (Exception e) { log.error("An error occurred during stopping the dispatcherLeaderRetriever", e); } try { // we only call this for legacy reasons to shutdown components that are started in the ClusterClient constructor super.shutdown(); } catch (Exception e) { log.error("An error occurred during the client shutdown.", e); } }