currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible());
private void logTask(TaskAttemptContext taskContext) { if (LOG.isInfoEnabled()) { LOG.info(MessageFormat.format( "Old-style output committer is used in the task: {0} ({1})", taskContext.getJobName(), taskContext.getTaskAttemptID())); } } }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); if (fs.exists(taskOutputPath)) { throw new IOException(); } } } }
Path getWorkPath(TaskAttemptContext taskContext, Path basePath) throws IOException { // ${mapred.out.dir}/_temporary Path jobTmpDir = new Path(basePath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fs = jobTmpDir.getFileSystem(taskContext.getJobConf()); if (!fs.exists(jobTmpDir)) { throw new IOException("The temporary job-output directory " + jobTmpDir.toString() + " doesn't exist!"); } // ${mapred.out.dir}/_temporary/_${taskid} String taskid = taskContext.getTaskAttemptID().toString(); Path taskTmpDir = new Path(jobTmpDir, "_" + taskid); if (!fs.mkdirs(taskTmpDir)) { throw new IOException("Mkdirs failed to create " + taskTmpDir.toString()); } return taskTmpDir; } }
Path getWorkPath(TaskAttemptContext taskContext, Path basePath) throws IOException { // ${mapred.out.dir}/_temporary Path jobTmpDir = new Path(basePath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fs = jobTmpDir.getFileSystem(taskContext.getJobConf()); if (!fs.exists(jobTmpDir)) { throw new IOException("The temporary job-output directory " + jobTmpDir.toString() + " doesn't exist!"); } // ${mapred.out.dir}/_temporary/_${taskid} String taskid = taskContext.getTaskAttemptID().toString(); Path taskTmpDir = new Path(jobTmpDir, "_" + taskid); if (!fs.mkdirs(taskTmpDir)) { throw new IOException("Mkdirs failed to create " + taskTmpDir.toString()); } return taskTmpDir; } }
Path getWorkPath(TaskAttemptContext taskContext, Path basePath) throws IOException { // ${mapred.out.dir}/_temporary Path jobTmpDir = new Path(basePath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fs = jobTmpDir.getFileSystem(taskContext.getJobConf()); if (!fs.exists(jobTmpDir)) { throw new IOException("The temporary job-output directory " + jobTmpDir.toString() + " doesn't exist!"); } // ${mapred.out.dir}/_temporary/_${taskid} String taskid = taskContext.getTaskAttemptID().toString(); Path taskTmpDir = new Path(jobTmpDir, "_" + taskid); if (!fs.mkdirs(taskTmpDir)) { throw new IOException("Mkdirs failed to create " + taskTmpDir.toString()); } return taskTmpDir; } }
Path getTempTaskOutputPath(TaskAttemptContext taskContext) throws IOException { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } return null; }
@Override public String generateFileName(JobContext context, String topic, String brokerId, int partitionId, int count, long offset, String encodedPartition) { //adding taskId to filename to avoid name collision int taskId = ((TaskAttemptContext) context).getTaskAttemptID().getTaskID().getId(); return super.generateFileName(context, topic, String.valueOf(System.currentTimeMillis() / 1000), taskId, count, 0, encodedPartition); }
@Override public String generateFileName(JobContext context, String topic, String brokerId, int partitionId, int count, long offset, String encodedPartition) { //adding taskId to filename to avoid name collision int taskId = ((TaskAttemptContext) context).getTaskAttemptID().getTaskID().getId(); return super.generateFileName(context, topic, String.valueOf(System.currentTimeMillis() / 1000), taskId, count, 0, encodedPartition); }
Path getTempTaskOutputPath(TaskAttemptContext taskContext) throws IOException { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); try { FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } catch (IOException ie) { LOG.warn("Error getting temp task output in " + outputPath, ie); throw ie; } } return null; }
Path getTempTaskOutputPath(TaskAttemptContext taskContext) { JobConf conf = taskContext.getJobConf(); Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath != null) { Path p = new Path(outputPath, (FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskContext.getTaskAttemptID().toString())); try { FileSystem fs = p.getFileSystem(conf); return p.makeQualified(fs); } catch (IOException ie) { LOG.warn(StringUtils .stringifyException(ie)); return p; } } return null; }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
@Override public String getWorkingFileName(JobContext context, String topic, String brokerId, int partitionId, String encodedPartition) { //adding taskId to filename to avoid name collision int taskId = ((TaskAttemptContext) context).getTaskAttemptID().getTaskID().getId(); return super.getWorkingFileName(context, topic, "0", taskId, encodedPartition); }
@Override public String getWorkingFileName(JobContext context, String topic, String brokerId, int partitionId, String encodedPartition) { //adding taskId to filename to avoid name collision int taskId = ((TaskAttemptContext) context).getTaskAttemptID().getTaskID().getId(); return super.getWorkingFileName(context, topic, "0", taskId, encodedPartition); }
Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) {
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTaskAttemptPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); if (fs.exists(taskOutputPath)) { throw new IOException(); } } } }
Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) {
Path taskOutput) throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) {