public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) { this(mapredOutputFormat, job); super.getJobConf().setOutputCommitter(outputCommitterClass); }
/** * Hive uses side effect files exclusively for it's output. It also manages * the setup/cleanup/commit of output from the hive client. As a result it does * not need support for the same inside the MR framework * * This routine sets the appropriate options related to bypass setup/cleanup/commit * support in the MR framework, but does not set the OutputFormat class. */ public static void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(MRJobConfig.SETUP_CLEANUP_NEEDED, false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, false); } }
/** * Hive uses side effect files exclusively for it's output. It also manages * the setup/cleanup/commit of output from the hive client. As a result it does * not need support for the same inside the MR framework * * This routine sets the appropriate options related to bypass setup/cleanup/commit * support in the MR framework, but does not set the OutputFormat class. */ public static void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(MRJobConfig.SETUP_CLEANUP_NEEDED, false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, false); } }
job.setInputFormat(CompactorInputFormat.class); job.setOutputFormat(NullOutputFormat.class); job.setOutputCommitter(CompactorOutputCommitter.class);
job.setInputFormat(CompactorInputFormat.class); job.setOutputFormat(NullOutputFormat.class); job.setOutputCommitter(CompactorOutputCommitter.class);
public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) { this(mapredOutputFormat, job); super.getJobConf().setOutputCommitter(outputCommitterClass); }
public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) { this(mapredOutputFormat, job); super.getJobConf().setOutputCommitter(outputCommitterClass); }
public static void setAsOutputFormat(JobConf jobConf) { jobConf.setOutputFormat(DeprecatedParquetOutputFormat.class); jobConf.setOutputCommitter(MapredParquetOutputCommitter.class); }
private void setConfForFileOutputCommitter(JobConf job) { job.set(JobContext.TASK_ATTEMPT_ID, attempt); job.setOutputCommitter(FileOutputCommitter.class); FileOutputFormat.setOutputPath(job, outDir); }
public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) { this(mapredOutputFormat, job); super.getJobConf().setOutputCommitter(outputCommitterClass); }
public void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false); }
public void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false); }
@Override public void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false); }
private void setConfForFileOutputCommitter(JobConf job) { job.set(JobContext.TASK_ATTEMPT_ID, attempt); job.setOutputCommitter(FileOutputCommitter.class); FileOutputFormat.setOutputPath(job, outDir); }
@Override public void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false); }
public void testCommitFail() throws IOException { final Path inDir = new Path(rootDir, "./input"); final Path outDir = new Path(rootDir, "./output"); JobConf jobConf = createJobConf(); jobConf.setMaxMapAttempts(1); jobConf.setOutputCommitter(CommitterWithCommitFail.class); RunningJob rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 0); rJob.waitForCompletion(); assertEquals(JobStatus.FAILED, rJob.getJobState()); }
public void prepareJobOutput(JobConf conf) { conf.setOutputCommitter(NullOutputCommitter.class); // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false); // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206) // but can be backported. So we disable setup/cleanup in all versions >= 0.19 conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false); }
private void testFailCommitter(Class<? extends OutputCommitter> theClass, JobConf jobConf) throws IOException { jobConf.setOutputCommitter(theClass); RunningJob job = UtilsForTests.runJob(jobConf, inDir, outDir); // wait for the job to finish. job.waitForCompletion(); assertEquals(JobStatus.FAILED, job.getJobState()); }
private void configureJobCommon(DomainBuilderProperties properties, int versionNumber, int numPartitions, JobConf conf) throws IOException { // Hank specific configuration properties.setJobConfProperties(conf, versionNumber); // Output Committer conf.setOutputCommitter(DomainBuilderOutputCommitter.class); // Output path (set to tmp output path) FileOutputFormat.setOutputPath(conf, new Path(properties.getTmpOutputPath(versionNumber))); // Output format conf.setOutputFormat(properties.getOutputFormatClass()); // Num reduce tasks conf.setNumReduceTasks(numPartitions); }
public void testCommitFail() throws IOException { final Path inDir = new Path(rootDir, "./input"); final Path outDir = new Path(rootDir, "./output"); JobConf jobConf = createJobConf(); jobConf.setMaxMapAttempts(1); jobConf.setOutputCommitter(CommitterWithCommitFail.class); RunningJob rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 0); rJob.waitForCompletion(); assertEquals(JobStatus.FAILED, rJob.getJobState()); }