/** * Expert: Set the number of maximum attempts that will be made to run a * map task. * * @param n the number of attempts per map task. */ public void setMaxMapAttempts(int n) { ensureState(JobState.DEFINE); conf.setMaxMapAttempts(n); }
/** * Expert: Set the number of maximum attempts that will be made to run a * map task. * * @param n the number of attempts per map task. */ public void setMaxMapAttempts(int n) { ensureState(JobState.DEFINE); conf.setMaxMapAttempts(n); }
/** * Expert: Set the number of maximum attempts that will be made to run a * map task. * * @param n the number of attempts per map task. */ public void setMaxMapAttempts(int n) { ensureState(JobState.DEFINE); conf.setMaxMapAttempts(n); }
/** * Expert: Set the number of maximum attempts that will be made to run a * map task. * * @param n the number of attempts per map task. */ public void setMaxMapAttempts(int n) { ensureState(JobState.DEFINE); conf.setMaxMapAttempts(n); }
/** * Expert: Set the number of maximum attempts that will be made to run a * map task. * * @param n the number of attempts per map task. */ public void setMaxMapAttempts(int n) { ensureState(JobState.DEFINE); conf.setMaxMapAttempts(n); }
/** * Method to run a failing mapper on a given Cluster. * * @param conf * the JobConf for the job * @param inputPath * input path for the job. * @param outputDir * output directory for job. * @throws IOException */ static JobID runFailingMapJob(JobConf conf, Path inputPath, Path outputDir) throws IOException { conf.setMapDebugScript(SCRIPT_FILE); conf.setMaxMapAttempts(0); conf.set("mapred.committer.job.setup.cleanup.needed", "false"); RunningJob rJob = UtilsForTests.runJobFail(conf, inputPath, outputDir); return rJob.getID(); } }
public void testCommitFail() throws IOException { final Path inDir = new Path(rootDir, "./input"); final Path outDir = new Path(rootDir, "./output"); JobConf jobConf = createJobConf(); jobConf.setMaxMapAttempts(1); jobConf.setOutputCommitter(CommitterWithCommitFail.class); RunningJob rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 0); rJob.waitForCompletion(); assertEquals(JobStatus.FAILED, rJob.getJobState()); }
public static RunningJob runJobFail(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-fail"); conf.setMapperClass(FailMapper.class); conf.setReducerClass(IdentityReducer.class); conf.setMaxMapAttempts(1); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); while (!job.isComplete()) { try { Thread.sleep(100); } catch (InterruptedException e) { break; } } return job; }
private void testFailedJob(JobConf job, MyListener myListener) throws IOException { LOG.info("Testing job-fail"); Path inDir = new Path(TEST_ROOT_DIR + "/jiplistenerfailjob/input"); Path outDir = new Path(TEST_ROOT_DIR + "/jiplistenerfailjob/output"); job.setNumMapTasks(1); job.setNumReduceTasks(0); job.setMaxMapAttempts(1); // submit a job that fails RunningJob rJob = UtilsForTests.runJobFail(job, inDir, outDir); JobID id = rJob.getID(); // check if the job failure was notified assertFalse("Missing event notification on failing a running job", myListener.contains(id)); // check if failed assertEquals("Job failed!", JobStatus.FAILED, rJob.getJobState()); }
/** * Runs a job that will fail and verifies if the subprocesses of failed map * task are killed properly or not. */ private static void runFailingJobAndValidate(JobTracker jt, JobConf conf) throws IOException { conf.setJobName("testfailjobsubprocesses"); conf.setMapperClass(FailingMapperWithChildren.class); // We don't want to run the failing map task 4 times. So we run it once and // check if all the subprocesses are killed properly. conf.setMaxMapAttempts(1); RunningJob job = runJobAndSetProcessHandle(jt, conf); signalTask(signalFile.toString(), conf); validateKillingSubprocesses(job, conf); // Checking the Job status assertEquals(job.getJobState(), JobStatus.FAILED); }
private void testFailedJob(String fileName, Class<? extends OutputCommitter> committer, String[] exclude) throws IOException { JobConf jc = mr.createJobConf(); Path outDir = getNewOutputDir(); configureJob(jc, "fail job with abort()", 1, 0, outDir); jc.setMaxMapAttempts(1); // set the job to fail jc.setMapperClass(UtilsForTests.FailMapper.class); jc.setOutputCommitter(committer); JobClient jobClient = new JobClient(jc); RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); job.waitForCompletion(); if (fileName != null) { Path testFile = new Path(outDir, fileName); assertTrue("File " + testFile + " missing for failed job " + id, fileSys.exists(testFile)); } // check if the files from the missing set exists for (String ex : exclude) { Path file = new Path(outDir, ex); assertFalse("File " + file + " should not be present for failed job " + id, fileSys.exists(file)); } }
/** * Run the test * * @throws IOException on error */ public static void runTests() throws IOException { config.setLong("io.bytes.per.checksum", bytesPerChecksum); JobConf job = new JobConf(config, NNBench.class); job.setJobName("NNBench-" + operation); FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); // Explicitly set number of max map attempts to 1. job.setMaxMapAttempts(1); // Explicitly turn off speculative execution job.setSpeculativeExecution(false); job.setMapperClass(NNBenchMapper.class); job.setReducerClass(NNBenchReducer.class); FileOutputFormat.setOutputPath(job, new Path(baseDir, OUTPUT_DIR_NAME)); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks((int) numberOfReduces); JobClient.runJob(job); }
public void testCommitFail() throws IOException { final Path inDir = new Path(rootDir, "./input"); final Path outDir = new Path(rootDir, "./output"); JobConf jobConf = createJobConf(); jobConf.setMaxMapAttempts(1); jobConf.setOutputCommitter(CommitterWithCommitFail.class); RunningJob rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 0); rJob.waitForCompletion(); assertEquals(JobStatus.FAILED, rJob.getJobState()); }
public static RunningJob runJobFail(JobConf conf, Path inDir, Path outDir) throws IOException { conf.setJobName("test-job-fail"); conf.setMapperClass(FailMapper.class); conf.setReducerClass(IdentityReducer.class); conf.setMaxMapAttempts(1); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); long sleepCount = 0; while (!job.isComplete()) { try { if (sleepCount > 300) { // 30 seconds throw new IOException("Job didn't finish in 30 seconds"); } Thread.sleep(100); sleepCount++; } catch (InterruptedException e) { break; } } return job; }
/** * Test {@link Reporter}'s progress for map-reduce job. */ @Test public void testReporterProgressForMRJob() throws IOException { Path test = new Path(testRootTempDir, "testReporterProgressForMRJob"); JobConf conf = new JobConf(); conf.setMapperClass(ProgressTesterMapper.class); conf.setReducerClass(ProgressTestingReducer.class); conf.setMapOutputKeyClass(Text.class); // fail early conf.setMaxMapAttempts(1); conf.setMaxReduceAttempts(1); RunningJob job = UtilsForTests.runJob(conf, new Path(test, "in"), new Path(test, "out"), 1, 1, INPUT); job.waitForCompletion(); assertTrue("Job failed", job.isSuccessful()); }
/** * Test {@link Reporter}'s progress for a map-only job. * This will make sure that only the map phase decides the attempt's progress. */ @SuppressWarnings("deprecation") @Test public void testReporterProgressForMapOnlyJob() throws IOException { Path test = new Path(testRootTempDir, "testReporterProgressForMapOnlyJob"); JobConf conf = new JobConf(); conf.setMapperClass(ProgressTesterMapper.class); conf.setMapOutputKeyClass(Text.class); // fail early conf.setMaxMapAttempts(1); conf.setMaxReduceAttempts(0); RunningJob job = UtilsForTests.runJob(conf, new Path(test, "in"), new Path(test, "out"), 1, 0, INPUT); job.waitForCompletion(); assertTrue("Job failed", job.isSuccessful()); }
public static void runJobFail(JobConf conf, Path inDir, Path outDir) throws IOException, InterruptedException { conf.setJobName("test-job-fail"); conf.setMapperClass(FailMapper.class); conf.setJarByClass(FailMapper.class); conf.setReducerClass(IdentityReducer.class); conf.setMaxMapAttempts(1); boolean success = runJob(conf, inDir, outDir, 1, 0); Assert.assertFalse("Job expected to fail succeeded", success); }
/** * Run the test * * @throws IOException on error */ public static void runTests() throws IOException { config.setLong("io.bytes.per.checksum", bytesPerChecksum); JobConf job = new JobConf(config, NNBench.class); job.setJobName("NNBench-" + operation); FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME)); job.setInputFormat(SequenceFileInputFormat.class); // Explicitly set number of max map attempts to 1. job.setMaxMapAttempts(1); // Explicitly turn off speculative execution job.setSpeculativeExecution(false); job.setMapperClass(NNBenchMapper.class); job.setReducerClass(NNBenchReducer.class); FileOutputFormat.setOutputPath(job, new Path(baseDir, OUTPUT_DIR_NAME)); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks((int) numberOfReduces); JobClient.runJob(job); }
private void testFailedJob(String fileName, Class<? extends OutputCommitter> committer, String[] exclude) throws IOException { JobConf jc = mr.createJobConf(); Path outDir = getNewOutputDir(); configureJob(jc, "fail job with abort()", 1, 0, outDir); jc.setMaxMapAttempts(1); // set the job to fail jc.setMapperClass(UtilsForTests.FailMapper.class); jc.setOutputCommitter(committer); JobClient jobClient = new JobClient(jc); RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); job.waitForCompletion(); assertEquals("Job did not fail", JobStatus.FAILED, job.getJobState()); if (fileName != null) { Path testFile = new Path(outDir, fileName); assertTrue("File " + testFile + " missing for failed job " + id, fileSys.exists(testFile)); } // check if the files from the missing set exists for (String ex : exclude) { Path file = new Path(outDir, ex); assertFalse("File " + file + " should not be present for failed job " + id, fileSys.exists(file)); } }
conf.setMaxMapAttempts(SkipBadRecords.getAttemptsToStartSkipping(conf)+1+ mapperBadRecords.size()); conf.setMaxReduceAttempts(SkipBadRecords.getAttemptsToStartSkipping(conf)+