public static String toString(JobId jid) { return jid.toString(); }
@Override public String toString() { StringBuilder builder = new StringBuilder(TASK); JobId jobId = getJobId(); builder.append("_").append(jobId.getAppId().getClusterTimestamp()); builder.append("_").append( JobId.jobIdFormat.get().format(jobId.getAppId().getId())); builder.append("_"); builder.append(getTaskType() == TaskType.MAP ? "m" : "r").append("_"); builder.append(taskIdFormat.get().format(getId())); return builder.toString(); }
/** * Searches the job history file FileStatus list for the specified JobId. * * @param fileStatusList * fileStatus list of Job History Files. * @param jobId * The JobId to find. * @return A FileInfo object for the jobId, null if not found. * @throws IOException */ private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId) throws IOException { for (FileStatus fs : fileStatusList) { JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath() .getName()); if (jobIndexInfo.getJobId().equals(jobId)) { String confFileName = JobHistoryUtils .getIntermediateConfFileName(jobIndexInfo.getJobId()); String summaryFileName = JobHistoryUtils .getIntermediateSummaryFileName(jobIndexInfo.getJobId()); HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path( fs.getPath().getParent(), confFileName), new Path(fs.getPath() .getParent(), summaryFileName), jobIndexInfo, true); return fileInfo; } } return null; }
@Test (timeout = 30000) public void testDeletionofStagingOnKill() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); fs = mock(FileSystem.class); when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true); //Staging Dir exists String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); appMaster.init(conf); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook(appMaster); hook.run(); verify(fs, times(0)).delete(stagingJobPath, true); }
/** * Computes a serial number used as part of directory naming for the given jobId. * @param id the jobId. * @return the serial number used as part of directory naming for the given jobid */ public static int jobSerialNumber(JobId id) { return id.getId(); }
@Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + getId(); result = prime * result + getJobId().hashCode(); result = prime * result + getTaskType().hashCode(); return result; }
@Override public int compareTo(TaskId other) { int jobIdComp = this.getJobId().compareTo(other.getJobId()); if (jobIdComp == 0) { if (this.getTaskType() == other.getTaskType()) { return this.getId() - other.getId(); } else { return this.getTaskType().compareTo(other.getTaskType()); } } else { return jobIdComp; } } }
@Override public String toString() { StringBuilder builder = new StringBuilder(TASK); JobId jobId = getJobId(); builder.append("_").append(jobId.getAppId().getClusterTimestamp()); builder.append("_").append( JobId.jobIdFormat.get().format(jobId.getAppId().getId())); builder.append("_"); builder.append(getTaskType() == TaskType.MAP ? "m" : "r").append("_"); builder.append(taskIdFormat.get().format(getId())); return builder.toString(); }
/** * Searches the job history file FileStatus list for the specified JobId. * * @param fileStatusList * fileStatus list of Job History Files. * @param jobId * The JobId to find. * @return A FileInfo object for the jobId, null if not found. * @throws IOException */ private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId) throws IOException { for (FileStatus fs : fileStatusList) { JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath() .getName()); if (jobIndexInfo.getJobId().equals(jobId)) { String confFileName = JobHistoryUtils .getIntermediateConfFileName(jobIndexInfo.getJobId()); String summaryFileName = JobHistoryUtils .getIntermediateSummaryFileName(jobIndexInfo.getJobId()); HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path( fs.getPath().getParent(), confFileName), new Path(fs.getPath() .getParent(), summaryFileName), jobIndexInfo, true); return fileInfo; } } return null; }
@Test public void testPreservePatternMatchedAndFailedStaging() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); // When RESERVE_FILES_PATTERN and PRESERVE_FAILED_TASK_FILES are set, // files in staging dir are always kept. conf.set(MRJobConfig.PRESERVE_FILES_PATTERN, "JobDir"); conf.setBoolean(MRJobConfig.PRESERVE_FAILED_TASK_FILES, true); fs = mock(FileSystem.class); when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true); //Staging Dir exists String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); appMaster.init(conf); appMaster.start(); appMaster.shutDownJob(); //test whether notifyIsLastAMRetry called Assert.assertEquals(true, ((TestMRApp) appMaster).getTestIsLastAMRetry()); verify(fs, times(0)).delete(stagingJobPath, true); }
/** * Computes a serial number used as part of directory naming for the given jobId. * @param id the jobId. * @return the serial number used as part of directory naming for the given jobid */ public static int jobSerialNumber(JobId id) { return id.getId(); }
@Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + getId(); result = prime * result + getJobId().hashCode(); result = prime * result + getTaskType().hashCode(); return result; }
@Override public int compareTo(TaskId other) { int jobIdComp = this.getJobId().compareTo(other.getJobId()); if (jobIdComp == 0) { if (this.getTaskType() == other.getTaskType()) { return this.getId() - other.getId(); } else { return this.getTaskType().compareTo(other.getTaskType()); } } else { return jobIdComp; } } }
@Override public String toString() { StringBuilder builder = new StringBuilder(TASK); JobId jobId = getJobId(); builder.append("_").append(jobId.getAppId().getClusterTimestamp()); builder.append("_").append( JobId.jobIdFormat.get().format(jobId.getAppId().getId())); builder.append("_"); builder.append(getTaskType() == TaskType.MAP ? "m" : "r").append("_"); builder.append(taskIdFormat.get().format(getId())); return builder.toString(); }
public static String toString(JobId jid) { return jid.toString(); }
/** * Searches the job history file FileStatus list for the specified JobId. * * @param fileStatusList * fileStatus list of Job History Files. * @param jobId * The JobId to find. * @return A FileInfo object for the jobId, null if not found. * @throws IOException */ private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId) throws IOException { for (FileStatus fs : fileStatusList) { JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath() .getName()); if (jobIndexInfo.getJobId().equals(jobId)) { String confFileName = JobHistoryUtils .getIntermediateConfFileName(jobIndexInfo.getJobId()); String summaryFileName = JobHistoryUtils .getIntermediateSummaryFileName(jobIndexInfo.getJobId()); HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path( fs.getPath().getParent(), confFileName), new Path(fs.getPath() .getParent(), summaryFileName), jobIndexInfo, true); return fileInfo; } } return null; }