@Override public JobContext createJobContext(Configuration conf, JobID jobId) { return new JobContextImpl(conf instanceof JobConf? new JobConf(conf) : conf, jobId); }
/** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getArchiveTimestamps() { return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf)); }
/** * The user-specified job name. */ public String getJobName() { if (state == JobState.DEFINE) { return super.getJobName(); } ensureState(JobState.RUNNING); return status.getJobName(); }
/** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getArchiveTimestamps() { return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf)); }
/** * The user-specified job name. */ public String getJobName() { if (state == JobState.DEFINE) { return super.getJobName(); } ensureState(JobState.RUNNING); return status.getJobName(); }
@Override public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job) { return new JobContextImpl(job.getConfiguration(), job.getJobID()); }
/** * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); }
/** * The user-specified job name. */ public String getJobName() { if (state == JobState.DEFINE) { return super.getJobName(); } ensureState(JobState.RUNNING); return status.getJobName(); }
@Override public BaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException { // only gather base statistics for FileInputFormats if (!(mapreduceInputFormat instanceof FileInputFormat)) { return null; } JobContext jobContext = new JobContextImpl(configuration, null); final FileBaseStatistics cachedFileStats = (cachedStats instanceof FileBaseStatistics) ? (FileBaseStatistics) cachedStats : null; try { final org.apache.hadoop.fs.Path[] paths = FileInputFormat.getInputPaths(jobContext); return getFileStats(cachedFileStats, paths, new ArrayList<FileStatus>(1)); } catch (IOException ioex) { if (LOG.isWarnEnabled()) { LOG.warn("Could not determine statistics due to an io error: " + ioex.getMessage()); } } catch (Throwable t) { if (LOG.isErrorEnabled()) { LOG.error("Unexpected problem while getting the file statistics: " + t.getMessage(), t); } } // no statistics available return null; }
/** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getArchiveTimestamps() { return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf)); }
/** * The user-specified job name. */ public String getJobName() { if (state == JobState.DEFINE) { return super.getJobName(); } ensureState(JobState.RUNNING); return status.getJobName(); }
@Override public HadoopInputSplit[] createInputSplits(int minNumSplits) throws IOException { configuration.setInt("mapreduce.input.fileinputformat.split.minsize", minNumSplits); JobContext jobContext = new JobContextImpl(configuration, new JobID()); List<InputSplit> splits; try { splits = this.hCatInputFormat.getSplits(jobContext); } catch (InterruptedException e) { throw new IOException("Could not get Splits.", e); } HadoopInputSplit[] hadoopInputSplits = new HadoopInputSplit[splits.size()]; for (int i = 0; i < hadoopInputSplits.length; i++){ hadoopInputSplits[i] = new HadoopInputSplit(i, splits.get(i), jobContext); } return hadoopInputSplits; }
/** * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); }
/** * The user-specified job name. */ public String getJobName() { if (state == JobState.DEFINE) { return super.getJobName(); } ensureState(JobState.RUNNING); return status.getJobName(); }
@Override public HadoopInputSplit[] createInputSplits(int minNumSplits) throws IOException { configuration.setInt("mapreduce.input.fileinputformat.split.minsize", minNumSplits); JobContext jobContext = new JobContextImpl(configuration, new JobID()); jobContext.getCredentials().addAll(this.credentials); Credentials currentUserCreds = getCredentialsFromUGI(UserGroupInformation.getCurrentUser()); if (currentUserCreds != null) { jobContext.getCredentials().addAll(currentUserCreds); } List<org.apache.hadoop.mapreduce.InputSplit> splits; try { splits = this.mapreduceInputFormat.getSplits(jobContext); } catch (InterruptedException e) { throw new IOException("Could not get Splits.", e); } HadoopInputSplit[] hadoopInputSplits = new HadoopInputSplit[splits.size()]; for (int i = 0; i < hadoopInputSplits.length; i++) { hadoopInputSplits[i] = new HadoopInputSplit(i, splits.get(i), jobContext); } return hadoopInputSplits; }
/** * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); }
this.context = new TaskAttemptContextImpl(this.configuration, taskAttemptID); this.outputCommitter = this.mapreduceOutputFormat.getOutputCommitter(this.context); this.outputCommitter.setupJob(new JobContextImpl(this.configuration, new JobID())); } catch (Exception e) { throw new RuntimeException(e);
/** * Get the timestamps of the archives. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getArchiveTimestamps() { return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf)); }
@Override public void finalizeGlobal(int parallelism) throws IOException { JobContext jobContext; TaskAttemptContext taskContext; try { TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(1).length()) + "s", " ").replace(" ", "0") + Integer.toString(1) + "_0"); jobContext = new JobContextImpl(this.configuration, new JobID()); taskContext = new TaskAttemptContextImpl(this.configuration, taskAttemptID); this.outputCommitter = this.mapreduceOutputFormat.getOutputCommitter(taskContext); } catch (Exception e) { throw new RuntimeException(e); } jobContext.getCredentials().addAll(this.credentials); Credentials currentUserCreds = getCredentialsFromUGI(UserGroupInformation.getCurrentUser()); if (currentUserCreds != null) { jobContext.getCredentials().addAll(currentUserCreds); } // finalize HDFS output format if (this.outputCommitter != null) { this.outputCommitter.commitJob(jobContext); } }
/** * Get the timestamps of the files. Used by internal * DistributedCache and MapReduce code. * @return a string array of timestamps */ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); }