appContext.setAMContainerSpec(amContainer); appContext.setResource(capability);
localResources, appMasterEnv, appMasterLaunchcommands, null, null, null); applicationContext.setAMContainerSpec(amContainer);
appContext.setAMContainerSpec(amContainer);
appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue
appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
.getApplicationSubmissionContext(); appContext.setAMContainerSpec(amContainer); appContext.setResource(getCapability());
appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setApplicationTags(Collections.singleton(job.metadata().serialize()));
appContext.setAMContainerSpec(amContainer);
/** * Clear Unused fields to free memory. */ private void clearUnusedFields() { this.submissionContext.setAMContainerSpec(null); this.submissionContext.setLogAggregationContext(null); }
UnmanagedAmYarnSubmissionHelper( final YarnConfiguration yarnConfiguration, final YarnProxyUser yarnProxyUser, final SecurityTokenProvider tokenProvider) throws IOException, YarnException { this.tokenProvider = tokenProvider; this.yarnProxyUser = yarnProxyUser; LOG.log(Level.FINE, "Initializing YARN Client"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(yarnConfiguration); this.yarnClient.start(); LOG.log(Level.FINE, "Initialized YARN Client"); LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN."); final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext( Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(), tokenProvider.getTokens()); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); this.applicationSubmissionContext.setAMContainerSpec(launchContext); this.applicationSubmissionContext.setUnmanagedAM(true); this.applicationId = this.applicationSubmissionContext.getApplicationId(); LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId); }
@Override public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext) { YarnClient yarnClient = createYarnClient(); try { ContainerLaunchContext context = launchContext.getLaunchContext(); addRMToken(context, yarnClient); context.setUser(appSubmissionContext.getUser()); context.setResource(adjustMemory(response, capability)); appSubmissionContext.setAMContainerSpec(context); yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(yarnClient, appId); } catch (YarnRemoteException e) { throw new RuntimeException("Failed to submit application " + appId, e); } finally { yarnClient.stop(); } } };
@Override public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context) { ContainerLaunchContext launchContext = context.getLaunchContext(); YarnClient yarnClient = createYarnClient(); try { addRMToken(launchContext, yarnClient, appId); appSubmissionContext.setAMContainerSpec(launchContext); appSubmissionContext.setResource(capability); configureAppSubmissionContext(appSubmissionContext); yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(appId); } catch (YarnException | IOException e) { throw new RuntimeException("Failed to submit application " + appId, e); } finally { yarnClient.stop(); } } };
@Override public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context) { ContainerLaunchContext launchContext = context.getLaunchContext(); YarnClient yarnClient = createYarnClient(); try { addRMToken(launchContext, yarnClient, appId); appSubmissionContext.setAMContainerSpec(launchContext); appSubmissionContext.setResource(capability); configureAppSubmissionContext(appSubmissionContext); yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(appId); } catch (YarnException | IOException e) { throw new RuntimeException("Failed to submit application " + appId, e); } finally { yarnClient.stop(); } } };
@Test (timeout = 30000) public void testAppRecoverPath() throws IOException { LOG.info("--- START: testAppRecoverPath ---"); ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class); sub.setAMContainerSpec(prepareContainerLaunchContext()); testCreateAppSubmittedRecovery(sub); }
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); context.setAMContainerSpec(new ContainerLaunchContextPBImpl()); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); when(mockApp.getCallerContext()) .thenReturn(new CallerContext.Builder("context").build()); store.storeNewApplication(mockApp); return mockApp; }
@Public @Stable public static ApplicationSubmissionContext newInstance( ApplicationId applicationId, String applicationName, String queue, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, String applicationType, boolean keepContainers, String appLabelExpression, ResourceRequest resourceRequest) { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setApplicationType(applicationType); context.setKeepContainersAcrossApplicationAttempts(keepContainers); context.setNodeLabelExpression(appLabelExpression); context.setAMContainerResourceRequests( Collections.singletonList(resourceRequest)); return context; }
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
@Test (timeout = 30000) public void testAppRecoverPath() throws IOException { LOG.info("--- START: testAppRecoverPath ---"); ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class); ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class); Credentials credentials = new Credentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); clc.setTokens(securityTokens); sub.setAMContainerSpec(clc); testCreateAppSubmittedRecovery(sub); }
@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }