/** * clean up staging directories for the job. * @throws IOException */ public void cleanupStagingDir() throws IOException { /* make sure we clean the staging files */ String jobTempDir = getConfig().get(MRJobConfig.MAPREDUCE_JOB_DIR); FileSystem fs = getFileSystem(getConfig()); try { if (!keepJobFiles(new JobConf(getConfig()), jobTempDir)) { jobTempDir = getConfig().get(MRJobConfig.MAPREDUCE_JOB_DIR); if (jobTempDir == null) { LOG.warn("Job Staging directory is null"); return; } Path jobTempDirPath = new Path(jobTempDir); LOG.info("Deleting staging directory " + FileSystem.getDefaultUri(getConfig()) + " " + jobTempDir); fs.delete(jobTempDirPath, true); } } catch(IOException io) { LOG.error("Failed to cleanup staging dir " + jobTempDir, io); } }
/** * clean up staging directories for the job. * @throws IOException */ public void cleanupStagingDir() throws IOException { /* make sure we clean the staging files */ String jobTempDir = getConfig().get(MRJobConfig.MAPREDUCE_JOB_DIR); FileSystem fs = getFileSystem(getConfig()); try { if (!keepJobFiles(new JobConf(getConfig()), jobTempDir)) { if (jobTempDir == null) { LOG.warn("Job Staging directory is null"); return; } Path jobTempDirPath = new Path(jobTempDir); LOG.info("Deleting staging directory " + FileSystem.getDefaultUri(getConfig()) + " " + jobTempDir); fs.delete(jobTempDirPath, true); } } catch(IOException io) { LOG.error("Failed to cleanup staging dir " + jobTempDir, io); } }
/** * clean up staging directories for the job. * @throws IOException */ public void cleanupStagingDir() throws IOException { /* make sure we clean the staging files */ String jobTempDir = null; FileSystem fs = getFileSystem(getConfig()); try { if (!keepJobFiles(new JobConf(getConfig()))) { jobTempDir = getConfig().get(MRJobConfig.MAPREDUCE_JOB_DIR); if (jobTempDir == null) { LOG.warn("Job Staging directory is null"); return; } Path jobTempDirPath = new Path(jobTempDir); LOG.info("Deleting staging directory " + FileSystem.getDefaultUri(getConfig()) + " " + jobTempDir); fs.delete(jobTempDirPath, true); } } catch(IOException io) { LOG.error("Failed to cleanup staging dir " + jobTempDir, io); } }
private boolean isRecoverySupported() throws IOException { boolean isSupported = false; Configuration conf = getConfig(); if (committer != null) { final JobContext _jobContext = getJobContextFromConf(conf); isSupported = callWithJobClassLoader(conf, new ExceptionAction<Boolean>() { public Boolean call(Configuration conf) throws IOException { return committer.isRecoverySupported(_jobContext); } }); } return isSupported; }
private boolean isCommitJobRepeatable() throws IOException { boolean isRepeatable = false; Configuration conf = getConfig(); if (committer != null) { final JobContext jobContext = getJobContextFromConf(conf); isRepeatable = callWithJobClassLoader(conf, new ExceptionAction<Boolean>() { public Boolean call(Configuration conf) throws IOException { return committer.isCommitJobRepeatable(jobContext); } }); } return isRepeatable; }
private boolean isCommitJobRepeatable() throws IOException { boolean isRepeatable = false; Configuration conf = getConfig(); if (committer != null) { final JobContext jobContext = getJobContextFromConf(conf); isRepeatable = callWithJobClassLoader(conf, new ExceptionAction<Boolean>() { public Boolean call(Configuration conf) throws IOException { return committer.isCommitJobRepeatable(jobContext); } }); } return isRepeatable; }
private boolean isRecoverySupported() throws IOException { boolean isSupported = false; Configuration conf = getConfig(); if (committer != null) { final JobContext _jobContext = getJobContextFromConf(conf); isSupported = callWithJobClassLoader(conf, new ExceptionAction<Boolean>() { public Boolean call(Configuration conf) throws IOException { return committer.isRecoverySupported(_jobContext); } }); } return isSupported; }
private boolean isCommitJobRepeatable() throws IOException { boolean isRepeatable = false; Configuration conf = getConfig(); if (committer != null) { final JobContext jobContext = getJobContextFromConf(conf); isRepeatable = callWithJobClassLoader(conf, new ExceptionAction<Boolean>() { public Boolean call(Configuration conf) throws IOException { return committer.isCommitJobRepeatable(jobContext); } }); } return isRepeatable; }
private boolean isRecoverySupported() throws IOException { boolean isSupported = false; Configuration conf = getConfig(); if (committer != null) { final JobContext _jobContext = getJobContextFromConf(conf); isSupported = callWithJobClassLoader(conf, new ExceptionAction<Boolean>() { public Boolean call(Configuration conf) throws IOException { return committer.isRecoverySupported(_jobContext); } }); } return isSupported; }
boolean recoveryEnabled = getConfig().getBoolean( MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE_DEFAULT); int reducerCount = getConfig().getInt(MRJobConfig.NUM_REDUCES, 0); boolean spillEncrypted = CryptoUtils.isEncryptedSpillEnabled(getConfig()); if (reducerCount > 0 && spillEncrypted) { LOG.info("Not attempting to recover. Intermediate spill encryption" +
private void cleanUpPreviousJobOutput() { // recovered application masters should not remove data from previous job if (!isFirstAttempt() && !recovered()) { JobContext jobContext = getJobContextFromConf(getConfig()); try { LOG.info("Starting to clean up previous job's temporary files"); this.committer.abortJob(jobContext, State.FAILED); LOG.info("Finished cleaning up previous job temporary files"); } catch (FileNotFoundException e) { LOG.info("Previous job temporary files do not exist, " + "no clean up was necessary."); } catch (Exception e) { // the clean up of a previous attempt is not critical to the success // of this job - only logging the error LOG.error("Error while trying to clean up previous job's temporary " + "files", e); } } }
private void parsePreviousJobHistory() throws IOException { FSDataInputStream in = getPreviousJobHistoryStream(getConfig(), appAttemptID); JobHistoryParser parser = new JobHistoryParser(in);
private void parsePreviousJobHistory() throws IOException { FSDataInputStream in = getPreviousJobHistoryStream(getConfig(), appAttemptID); JobHistoryParser parser = new JobHistoryParser(in);
FSDataInputStream inputStream = null; try { inputStream = getPreviousJobHistoryStream(getConfig(), appAttemptID); EventReader jobHistoryEventReader = new EventReader(inputStream);
FSDataInputStream inputStream = null; try { inputStream = getPreviousJobHistoryStream(getConfig(), appAttemptID); EventReader jobHistoryEventReader = new EventReader(inputStream);
FSDataInputStream inputStream = null; try { inputStream = getPreviousJobHistoryStream(getConfig(), appAttemptID); EventReader jobHistoryEventReader = new EventReader(inputStream);