ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext(); Set<File> systemShipFiles = new HashSet<>(shipFiles.size()); for (File file : shipFiles) {
ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId();
ApplicationSubmissionContext applicationContext = newAPP.getApplicationSubmissionContext(); applicationContext.setApplicationId(applicationId); applicationContext.setApplicationName(clientArguments.appName);
ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("ignition"); // application name appContext.setAMContainerSpec(amContainer);
ApplicationId setupAndSubmitApplication() throws IOException, YarnException { YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); ApplicationId applicationId = appSubmissionContext.getApplicationId();
ApplicationId createApplication() throws IOException, YarnException { YarnClientApplication app = yarnClient.createApplication(); return app.getApplicationSubmissionContext().getApplicationId(); }
ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId();
public JobID getNewJobID() throws IOException, InterruptedException { try { this.application = client.createApplication().getApplicationSubmissionContext(); this.applicationId = this.application.getApplicationId(); return TypeConverter.fromYarn(applicationId); } catch (YarnException e) { throw new IOException(e); } }
public JobID getNewJobID() throws IOException, InterruptedException { try { this.application = client.createApplication().getApplicationSubmissionContext(); this.applicationId = this.application.getApplicationId(); return TypeConverter.fromYarn(applicationId); } catch (YarnException e) { throw new IOException(e); } }
UnmanagedAmYarnSubmissionHelper( final YarnConfiguration yarnConfiguration, final YarnProxyUser yarnProxyUser, final SecurityTokenProvider tokenProvider) throws IOException, YarnException { this.tokenProvider = tokenProvider; this.yarnProxyUser = yarnProxyUser; LOG.log(Level.FINE, "Initializing YARN Client"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(yarnConfiguration); this.yarnClient.start(); LOG.log(Level.FINE, "Initialized YARN Client"); LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN."); final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext( Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(), tokenProvider.getTokens()); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); this.applicationSubmissionContext.setAMContainerSpec(launchContext); this.applicationSubmissionContext.setUnmanagedAM(true); this.applicationId = this.applicationSubmissionContext.getApplicationId(); LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId); }
@Test public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue() throws Exception { Map<String, String> options = new HashMap<String, String>(); EasyMock.replay(mockYarnClient, yarnClientApp); appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, tags, null); ApplicationSubmissionContext ctx = appMasterLauncher.application .getApplicationSubmissionContext(); long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval(); Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS, retryIntervalWindow); }
@Test public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws Exception { Map<String, String> options = new HashMap<String, String>(); final String expectedInterval = Integer.toString (120000); options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, expectedInterval); EasyMock.replay(mockYarnClient, yarnClientApp); appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, tags, null); ApplicationSubmissionContext ctx = appMasterLauncher.application .getApplicationSubmissionContext(); String retryIntervalWindow = Long.toString(ctx .getAttemptFailuresValidityInterval()); Assert.assertEquals(expectedInterval, retryIntervalWindow); }
public YarnSubmissionHelper(final YarnConfiguration yarnConfiguration, final REEFFileNames fileNames, final ClasspathProvider classpath, final YarnProxyUser yarnProxyUser, final SecurityTokenProvider tokenProvider, final boolean isUnmanaged, final List<String> commandPrefixList) throws IOException, YarnException { this.classpath = classpath; this.yarnProxyUser = yarnProxyUser; this.isUnmanaged = isUnmanaged; this.driverStdoutFilePath = ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStdoutFileName(); this.driverStderrFilePath = ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStderrFileName(); LOG.log(Level.FINE, "Initializing YARN Client"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(yarnConfiguration); this.yarnClient.start(); LOG.log(Level.FINE, "Initialized YARN Client"); LOG.log(Level.FINE, "Requesting Application ID from YARN."); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); this.applicationResponse = yarnClientApplication.getNewApplicationResponse(); this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); this.applicationSubmissionContext.setUnmanagedAM(isUnmanaged); this.applicationId = this.applicationSubmissionContext.getApplicationId(); this.tokenProvider = tokenProvider; this.commandPrefixList = commandPrefixList; this.configurationFilePaths = Collections.singletonList(fileNames.getDriverConfigurationPath()); LOG.log(Level.INFO, "YARN Application ID: {0}", this.applicationId); }
@Test public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws Exception { Map<String, String> options = new HashMap<String, String>(); final String expectedInterval = Integer.toString (120000); options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, expectedInterval); EasyMock.replay(mockYarnClient, yarnClientApp); appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, tags, null); ApplicationSubmissionContext ctx = appMasterLauncher.application .getApplicationSubmissionContext(); String retryIntervalWindow = Long.toString(ctx .getAttemptFailuresValidityInterval()); Assert.assertEquals(expectedInterval, retryIntervalWindow); }
@Test public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue() throws Exception { Map<String, String> options = new HashMap<String, String>(); EasyMock.replay(mockYarnClient, yarnClientApp); appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, tags, null); ApplicationSubmissionContext ctx = appMasterLauncher.application .getApplicationSubmissionContext(); long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval(); Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS, retryIntervalWindow); }
@Before public void initialize() throws Exception { mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); appSubmissionContext = EasyMock .createNiceMock(ApplicationSubmissionContext.class); EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) .andReturn(appSubmissionContext).once(); EasyMock.expect(mockYarnClient.createApplication()) .andReturn(yarnClientApp).once(); try { LogAggregationContext.class.getMethod("newInstance", String.class, String.class, String.class, String.class); isOldApi = false; rolledLogsIncludeMethod = LogAggregationContext.class .getMethod("getRolledLogsIncludePattern"); rolledLogsExcludeMethod = LogAggregationContext.class .getMethod("getRolledLogsExcludePattern"); } catch (Exception e) { isOldApi = true; } }
@Before public void initialize() throws Exception { mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); appSubmissionContext = EasyMock .createNiceMock(ApplicationSubmissionContext.class); EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) .andReturn(appSubmissionContext).once(); EasyMock.expect(mockYarnClient.createApplication()) .andReturn(yarnClientApp).once(); try { LogAggregationContext.class.getMethod("newInstance", String.class, String.class, String.class, String.class); isOldApi = false; rolledLogsIncludeMethod = LogAggregationContext.class .getMethod("getRolledLogsIncludePattern"); rolledLogsExcludeMethod = LogAggregationContext.class .getMethod("getRolledLogsExcludePattern"); } catch (Exception e) { isOldApi = true; } }
final ApplicationSubmissionContext appSubmissionContext = application.getApplicationSubmissionContext(); appSubmissionContext.setApplicationId(appId); appSubmissionContext.setApplicationName(twillSpec.getName());
final ApplicationSubmissionContext appSubmissionContext = application.getApplicationSubmissionContext(); appSubmissionContext.setApplicationId(appId); appSubmissionContext.setApplicationName(twillSpec.getName());