/** * @return the ApplicationAttemptId */ public ApplicationAttemptId getAppAttemptId() { return ApplicationAttemptId.fromString( datum.getApplicationAttemptId().toString()); }
@Public @Deprecated public static ApplicationAttemptId toApplicationAttemptId( String applicationAttemptIdStr) { return ApplicationAttemptId.fromString(applicationAttemptIdStr); }
/** * Fails an application attempt. * * @param attemptId ID of the attempt to fail. If provided, applicationId * parameter is not used. * @throws YarnException * @throws IOException */ private void failApplicationAttempt(String attemptId) throws YarnException, IOException { ApplicationId appId; ApplicationAttemptId attId; attId = ApplicationAttemptId.fromString(attemptId); appId = attId.getApplicationId(); sysout.println("Failing attempt " + attId + " of application " + appId); client.failApplicationAttempt(attId); }
/** * Fails an application attempt. * * @param attemptId ID of the attempt to fail. If provided, applicationId * parameter is not used. * @throws YarnException * @throws IOException */ private void failApplicationAttempt(String attemptId) throws YarnException, IOException { ApplicationId appId; ApplicationAttemptId attId; attId = ApplicationAttemptId.fromString(attemptId); appId = attId.getApplicationId(); sysout.println("Failing attempt " + attId + " of application " + appId); client.failApplicationAttempt(attId); }
private Entry<ApplicationAttemptId, MasterKey> getNextMasterKeyEntry( LeveldbIterator it) throws IOException { Entry<ApplicationAttemptId, MasterKey> masterKeyentry = null; try { while (it.hasNext()) { Entry<byte[], byte[]> entry = it.next(); String fullKey = asString(entry.getKey()); if (!fullKey.startsWith(NM_TOKENS_KEY_PREFIX)) { break; } String key = fullKey.substring(NM_TOKENS_KEY_PREFIX.length()); if (key.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) { ApplicationAttemptId attempt; try { attempt = ApplicationAttemptId.fromString(key); } catch (IllegalArgumentException e) { throw new IOException("Bad application master key state for " + fullKey, e); } masterKeyentry = new AbstractMap.SimpleEntry<>(attempt, parseMasterKey(entry.getValue())); break; } } } catch (DBException e) { throw new IOException(e); } return masterKeyentry; }
protected static ApplicationAttemptId parseApplicationAttemptId( String appAttemptId) { if (appAttemptId == null || appAttemptId.isEmpty()) { throw new NotFoundException("appAttemptId, " + appAttemptId + ", is empty or null"); } ApplicationAttemptId aaid = null; try { aaid = ApplicationAttemptId.fromString(appAttemptId); } catch (Exception e) { throw new BadRequestException(e); } if (aaid == null) { throw new NotFoundException("appAttemptId is null"); } return aaid; }
/** * Lists the containers matching the given application attempts * * @param appAttemptId * @throws YarnException * @throws IOException */ private void listContainers(String appAttemptId) throws YarnException, IOException { PrintWriter writer = new PrintWriter( new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); List<ContainerReport> appsReport = client.getContainers( ApplicationAttemptId.fromString(appAttemptId)); writer.println("Total number of containers " + ":" + appsReport.size()); writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time", "Finish Time", "State", "Host", "Node Http Address", "LOG-URL"); for (ContainerReport containerReport : appsReport) { writer.printf( CONTAINER_PATTERN, containerReport.getContainerId(), Times.format(containerReport.getCreationTime()), Times.format(containerReport.getFinishTime()), containerReport.getContainerState(), containerReport .getAssignedNode(), containerReport.getNodeHttpAddress() == null ? "N/A" : containerReport.getNodeHttpAddress(), containerReport.getLogUrl()); } writer.flush(); }
/** * Lists the containers matching the given application attempts * * @param appAttemptId * @throws YarnException * @throws IOException */ private void listContainers(String appAttemptId) throws YarnException, IOException { PrintWriter writer = new PrintWriter( new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); List<ContainerReport> appsReport = client.getContainers( ApplicationAttemptId.fromString(appAttemptId)); writer.println("Total number of containers " + ":" + appsReport.size()); writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time", "Finish Time", "State", "Host", "Node Http Address", "LOG-URL"); for (ContainerReport containerReport : appsReport) { writer.printf( CONTAINER_PATTERN, containerReport.getContainerId(), Times.format(containerReport.getCreationTime()), Times.format(containerReport.getFinishTime()), containerReport.getContainerState(), containerReport .getAssignedNode(), containerReport.getNodeHttpAddress() == null ? "N/A" : containerReport.getNodeHttpAddress(), containerReport.getLogUrl()); } writer.flush(); }
@Test public void testMRAppMasterForDifferentUser() throws IOException, InterruptedException { String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001"; String containerIdStr = "container_1317529182569_0004_000001_1"; String userName = "TestAppMasterUser"; ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); ContainerId containerId = ContainerId.fromString(containerIdStr); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis()); JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); Path userPath = new Path(stagingDir, userName); Path userStagingPath = new Path(userPath, ".staging"); assertEquals(userStagingPath.toString(), appMaster.stagingDirPath.toString()); }
private ApplicationAttemptStateData createAttemptState(String itemName, byte[] data) throws IOException { ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(itemName); ApplicationAttemptStateDataPBImpl attemptState = new ApplicationAttemptStateDataPBImpl( ApplicationAttemptStateDataProto.parseFrom(data)); if (!attemptId.equals(attemptState.getAttemptId())) { throw new YarnRuntimeException("The database entry for " + attemptId + " contains data for " + attemptState.getAttemptId()); } return attemptState; }
@Override protected void modifyAppState() throws Exception { // imitate appAttemptFile1 is still .new, but old one is deleted String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001"; ApplicationAttemptId attemptId1 = ApplicationAttemptId.fromString(appAttemptIdStr1); Path appDir = fsTester.store.getAppDir(attemptId1.getApplicationId().toString()); Path appAttemptFile1 = new Path(appDir, attemptId1.toString() + ".new"); FileSystemRMStateStore fileSystemRMStateStore = (FileSystemRMStateStore) fsTester.getRMStateStore(); fileSystemRMStateStore.renameFile(appAttemptFile1, new Path(appAttemptFile1.getParent(), appAttemptFile1.getName() + ".new")); }
private static CommitterEventHandler createCommitterEventHandler( Dispatcher dispatcher, OutputCommitter committer) { final SystemClock clock = SystemClock.getInstance(); AppContext appContext = mock(AppContext.class); when(appContext.getEventHandler()).thenReturn( dispatcher.getEventHandler()); when(appContext.getClock()).thenReturn(clock); RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() { @Override public long getLastHeartbeatTime() { return clock.getTime(); } @Override public void runOnNextHeartbeat(Runnable callback) { callback.run(); } }; ApplicationAttemptId id = ApplicationAttemptId.fromString( "appattempt_1234567890000_0001_0"); when(appContext.getApplicationID()).thenReturn(id.getApplicationId()); when(appContext.getApplicationAttemptId()).thenReturn(id); CommitterEventHandler handler = new CommitterEventHandler(appContext, committer, heartbeatHandler); dispatcher.register(CommitterEventType.class, handler); return handler; }
@Test (timeout = 30000) public void testMRAppMasterMaxAppAttempts() throws IOException, InterruptedException { // No matter what's the maxAppAttempt or attempt id, the isLastRetry always // equals to false Boolean[] expectedBools = new Boolean[]{ false, false, false }; String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; String containerIdStr = "container_1317529182569_0004_000002_1"; String userName = "TestAppMasterUser"; ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); ContainerId containerId = ContainerId.fromString(containerIdStr); JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); File stagingDir = new File(MRApps.getStagingAreaDir(conf, userName).toString()); stagingDir.mkdirs(); for (int i = 0; i < expectedBools.length; ++i) { MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis(), false, true); MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); assertEquals("isLastAMRetry is correctly computed.", expectedBools[i], appMaster.isLastAMRetry()); } }
JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr);
conf.setInt(MRJobConfig.NUM_REDUCES, 0); conf.set(JHAdminConfig.MR_HS_JHIST_FORMAT, "json"); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); JobId jobId = TypeConverter.toYarn(
JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); JobId jobId = TypeConverter.toYarn(
String containerIdStr = "container_1317529182569_0004_000002_1"; String userName = "TestAppMasterUser"; ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); ContainerId containerId = ContainerId.fromString(containerIdStr);
conf.setInt(org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter. FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, 1); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); JobId jobId = TypeConverter.toYarn(
private ApplicationAttemptId storeAttempt(RMStateStore store, TestDispatcher dispatcher, String appAttemptIdStr, AMRMTokenSecretManager appTokenMgr, ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, boolean createContainer) throws Exception { ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(appAttemptIdStr); Token<AMRMTokenIdentifier> appAttemptToken = null; if (appTokenMgr != null) { appAttemptToken = generateAMRMToken(attemptId, appTokenMgr); } SecretKey clientTokenKey = null; if (clientToAMTokenMgr != null) { clientTokenKey = clientToAMTokenMgr.createMasterKey(attemptId); Credentials attemptCred = new Credentials(); attemptCred.addSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME, clientTokenKey.getEncoded()); } ContainerId containerId = null; if (createContainer) { containerId = ContainerId.newContainerId(attemptId, 1); } storeAttempt(store, attemptId, containerId.toString(), appAttemptToken, clientTokenKey, dispatcher); return attemptId; }
@Test public void testDuplicateRMAppDeletion() throws Exception { TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); long submitTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis() + 1234; RMStateStore store = zkTester.getRMStateStore(); TestDispatcher dispatcher = new TestDispatcher(); store.setRMDispatcher(dispatcher); ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString( "appattempt_1352994193343_0002_000001"); ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId(); storeApp(store, appIdRemoved, submitTime, startTime); storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher); ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appIdRemoved); ApplicationStateData appStateRemoved = ApplicationStateData.newInstance( submitTime, startTime, context, "user1"); appStateRemoved.attempts.put(attemptIdRemoved, null); store.removeApplicationStateInternal(appStateRemoved); try { store.removeApplicationStateInternal(appStateRemoved); } catch (KeeperException.NoNodeException nne) { fail("NoNodeException should not happen."); } store.close(); }