static void setWorkOutputPath(TaskAttemptContext context) throws IOException { String outputPath = context.getConfiguration().get("mapred.output.dir"); //we need to do this to get the task path and set it for mapred implementation //since it can't be done automatically because of mapreduce->mapred abstraction if (outputPath != null) context.getConfiguration().set("mapred.work.output.dir", new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString()); } }
/** * Gets the target output stream where the Avro container file should be written. * * @param context The task attempt context. * @return The target output stream. */ protected OutputStream getAvroFileOutputStream(TaskAttemptContext context) throws IOException { Path path = new Path(((FileOutputCommitter)getOutputCommitter(context)).getWorkPath(), getUniqueFile(context,context.getConfiguration().get("avro.mo.config.namedOutput","part"),org.apache.avro.mapred.AvroOutputFormat.EXT)); return path.getFileSystem(context.getConfiguration()).create(path); }
Path outputPath = committer.getWorkPath();
this.configuration.set("mapreduce.task.output.dir", ((FileOutputCommitter) this.outputCommitter).getWorkPath().toString());
final Path taskAttemptOutputdir = new FileOutputCommitter(outputdir, tac).getWorkPath(); final org.apache.hadoop.mapreduce.RecordWriter< ImmutableBytesWritable, Cell> fileWriter = getFileWriter(tac);
final Path outputDir = ((FileOutputCommitter)committer).getWorkPath(); final Configuration conf = context.getConfiguration(); final boolean writeMultipleTables = conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false) ;
final Path outputdir = ((FileOutputCommitter) committer).getWorkPath(); final Configuration conf = context.getConfiguration(); LOG.debug("Task output path: " + outputdir);
"mapred.work.output.dir", new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext) .getWorkPath().toString());
@Override public void commitTask(TaskAttemptContext context) throws IOException { long recordCount = getRecordCountCounter(context); String fileName = "part-r-" + recordCount + "." + System.currentTimeMillis() + ".avro"; for (FileStatus status : fs.listStatus(super.getWorkPath())) { if (status.getPath().getName().endsWith("avro")) { LOG.info(String.format("Moving %s to %s", status.getPath(), new Path(outputPath, fileName))); fs.rename(status.getPath(), new Path(outputPath, fileName)); } } super.commitTask(context); }
FileSystem fs = FileSystem.get(context.getConfiguration()); if (EtlMultiOutputFormat.isRunMoveData(context)) { Path workPath = super.getWorkPath(); log.info("work path: " + workPath); Path baseOutDir = EtlMultiOutputFormat.getDestinationPath(context); new Path(super.getWorkPath(), EtlMultiOutputFormat.getUniqueFile(context, EtlMultiOutputFormat.OFFSET_PREFIX, "")), EtlKey.class, NullWritable.class);
@Override public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException { final FileOutputCommitter foc = (FileOutputCommitter) getOutputCommitter(context); return new Path(new Path(foc.getWorkPath(), SUB_DIR), getUniqueFile(context, getOutputName(context), extension)); } };
final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); final Configuration conf = context.getConfiguration(); final FileSystem fs = outputdir.getFileSystem(conf);
File expectedFile = new File(new Path(committer.getWorkPath(), partFile) .toString()); assertFalse("task temp dir still exists", expectedFile.exists());
new Path(outDir, FileOutputCommitter.PENDING_DIR_NAME).toString()); File taskOutputDir = new File(Path.getPathWithoutSchemeAndAuthority( committer.getWorkPath()).toString()); assertTrue("job temp dir does not exist", jobOutputDir.exists()); assertTrue("task temp dir does not exist", taskOutputDir.exists());
/** * Verify that if the committer output path is null, you get back * a FileOutputCommitter with null output & work paths. */ @Test public void testCommitterNullOutputPath() throws Throwable { // bind http to schema Configuration conf = newBondedConfiguration(); // then ask committers for a null path FileOutputCommitter committer = createCommitter( FileOutputCommitterFactory.class, FileOutputCommitter.class, null, conf); assertNull(committer.getOutputPath()); assertNull(committer.getWorkPath()); }
static void setWorkOutputPath(TaskAttemptContext context) throws IOException { String outputPath = context.getConfiguration().get("mapred.output.dir"); //we need to do this to get the task path and set it for mapred implementation //since it can't be done automatically because of mapreduce->mapred abstraction if (outputPath != null) context.getConfiguration().set("mapred.work.output.dir", new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString()); } }
static void setWorkOutputPath(TaskAttemptContext context) throws IOException { String outputPath = context.getConfiguration().get("mapred.output.dir"); //we need to do this to get the task path and set it for mapred implementation //since it can't be done automatically because of mapreduce->mapred abstraction if (outputPath != null) context.getConfiguration().set("mapred.work.output.dir", new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString()); } }
static void setWorkOutputPath(TaskAttemptContext context) throws IOException { String outputPath = context.getConfiguration().get("mapred.output.dir"); //we need to do this to get the task path and set it for mapred implementation //since it can't be done automatically because of mapreduce->mapred abstraction if (outputPath != null) context.getConfiguration().set("mapred.work.output.dir", new FileOutputCommitter(new Path(outputPath), context).getWorkPath().toString()); } }
public static Path getDefaultWorkFileOverride( org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<?, ?> outputFormat, String name, TaskAttemptContext context, String extension ) throws IOException { FileOutputCommitter committer = (FileOutputCommitter) outputFormat.getOutputCommitter(context); return new Path(committer.getWorkPath(), name + extension); } }
/** * Gets the target output stream where the Avro container file should be written. * * @param context The task attempt context. * @return The target output stream. */ protected OutputStream getAvroFileOutputStream(TaskAttemptContext context) throws IOException { Path path = new Path(((FileOutputCommitter)getOutputCommitter(context)).getWorkPath(), getUniqueFile(context,context.getConfiguration().get("avro.mo.config.namedOutput","part"),org.apache.avro.mapred.AvroOutputFormat.EXT)); return path.getFileSystem(context.getConfiguration()).create(path); }