/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
baseOutputCommitter.setupJob(currJobContext);
/** {@inheritDoc} */ @Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException { HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx; try { ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf()); OutputCommitter committer = ctx.jobConf().getOutputCommitter(); if (committer != null) committer.setupJob(ctx.jobContext()); } catch (IOException e) { throw new IgniteCheckedException(e); } } }
/** * This method implements the new interface by calling the old method. Note * that the input types are different between the new and old apis and this * is a bridge between the two. */ @Override public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext ) throws IOException { setupJob((JobContext) jobContext); }
/** * This method implements the new interface by calling the old method. Note * that the input types are different between the new and old apis and this * is a bridge between the two. */ @Override public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext ) throws IOException { setupJob((JobContext) jobContext); }
/** * This method implements the new interface by calling the old method. Note * that the input types are different between the new and old apis and this * is a bridge between the two. */ @Override public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext ) throws IOException { setupJob((JobContext) jobContext); }
/** * This method implements the new interface by calling the old method. Note * that the input types are different between the new and old apis and this * is a bridge between the two. */ @Override public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext ) throws IOException { setupJob((JobContext) jobContext); }
/** * This method implements the new interface by calling the old method. Note * that the input types are different between the new and old apis and this * is a bridge between the two. */ @Override public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext ) throws IOException { setupJob((JobContext) jobContext); }
/** * This method implements the new interface by calling the old method. Note * that the input types are different between the new and old apis and this * is a bridge between the two. */ @Override public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext ) throws IOException { setupJob((JobContext) jobContext); }
@Override public void setupJob(JobContext jobContext) throws IOException { baseOutputCommitter.setupJob(jobContext); }
@Override public void setupJob(org.apache.hadoop.mapreduce.JobContext context) throws IOException { baseCommitter.setupJob(hackJobContext(context)); }
@Override public void init(@Nonnull Context context) throws Exception { outputCommitter = jobConf.getOutputCommitter(); jobContext = new JobContextImpl(jobConf, new JobID()); outputCommitter.setupJob(jobContext); }
@Override public void init(@Nonnull Context context) { outputCommitter = jobConf.getOutputCommitter(); jobContext = new JobContextImpl(jobConf, new JobID()); uncheckRun(() -> outputCommitter.setupJob(jobContext)); }
protected void runJobSetupTask(TaskUmbilicalProtocol umbilical) throws IOException { // do the setup getProgress().setStatus("setup"); conf.getOutputCommitter().setupJob(jobContext); done(umbilical); }
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
baseOutputCommitter.setupJob(currJobContext);
OutputCommitter committer = new CommitterWithoutCleanup(); JobContext jContext = new JobContextImpl(job, attemptID.getJobID()); committer.setupJob(jContext);
OutputCommitter committer = new CommitterWithoutCleanup(); JobContext jContext = new JobContextImpl(job, attemptID.getJobID()); committer.setupJob(jContext);