Refine search
@Override protected void runJob(String jobName, Configuration c, List<Scan> scans) throws IOException, InterruptedException, ClassNotFoundException { JobConf job = new JobConf(TEST_UTIL.getConfiguration()); job.setJobName(jobName); job.setMapperClass(Mapper.class); job.setReducerClass(Reducer.class); TableMapReduceUtil.initMultiTableSnapshotMapperJob(getSnapshotScanMapping(scans), Mapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job, true, restoreDir); TableMapReduceUtil.addDependencyJars(job); job.setReducerClass(Reducer.class); job.setNumReduceTasks(1); // one to get final "first" and "last" key FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); LOG.info("Started " + job.getJobName()); RunningJob runningJob = JobClient.runJob(job); runningJob.waitForCompletion(); assertTrue(runningJob.isSuccessful()); LOG.info("After map/reduce completion - job " + jobName); }
JobConf conf = prepareJobConf(baseJobConf); FileSystem fs = outputDir.getFileSystem(conf); if(fs.exists(outputDir)) { info("Deleting previous output in " + outputDir + " for building store " + this.storeDef.getName()); fs.delete(outputDir, true); FileSystem outputFs = outputDir.getFileSystem(conf); if(outputFs.exists(outputDir)) { throw new IOException("Final output directory already exists."); JobClient jc = new JobClient(conf); RunningJob runningJob = jc.submitJob(conf); Counters counters; if (!jc.monitorAndPrintJob(conf, runningJob)) { counters = runningJob.getCounters(); "=" + suggestedTargetChunkSize); } else { logger.error("Job Failed: " + runningJob.getFailureInfo()); counters = runningJob.getCounters(); long numberOfRecords = counters.getCounter(Task.Counter.REDUCE_INPUT_GROUPS);
public double getProgress() throws IOException { if(_runningJob == null) return 0.0; else return (double) (_runningJob.mapProgress() + _runningJob.reduceProgress()) / 2.0d; }
private void updateMapRedStatsJson(MapRedStats stats, RunningJob rj) throws IOException, JSONException { if (statsJSON == null) { statsJSON = new JSONObject(); } if (stats != null) { if (stats.getNumMap() >= 0) { statsJSON.put(NUMBER_OF_MAPPERS, stats.getNumMap()); } if (stats.getNumReduce() >= 0) { statsJSON.put(NUMBER_OF_REDUCERS, stats.getNumReduce()); } if (stats.getCounters() != null) { statsJSON.put(COUNTERS, getCountersJson(stats.getCounters())); } } if (rj != null) { statsJSON.put(JOB_ID, rj.getID().toString()); statsJSON.put(JOB_FILE, rj.getJobFile()); statsJSON.put(TRACKING_URL, rj.getTrackingURL()); statsJSON.put(MAP_PROGRESS, Math.round(rj.mapProgress() * 100)); statsJSON.put(REDUCE_PROGRESS, Math.round(rj.reduceProgress() * 100)); statsJSON.put(CLEANUP_PROGRESS, Math.round(rj.cleanupProgress() * 100)); statsJSON.put(SETUP_PROGRESS, Math.round(rj.setupProgress() * 100)); statsJSON.put(COMPLETE, rj.isComplete()); statsJSON.put(SUCCESSFUL, rj.isSuccessful()); } }
success = true; HiveFileFormatUtils.prepareJobOutput(job); job.setOutputFormat(HiveOutputFormatImpl.class); job.setMapperClass(work.getMapperClass()); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(NullWritable.class); if(work.getNumMapTasks() != null) { Path tempOutPath = Utilities.toTempPath(outputPath); try { FileSystem fs = tempOutPath.getFileSystem(job); if (!fs.exists(tempOutPath)) { fs.mkdirs(tempOutPath); HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE"); JobClient jc = new JobClient(job); rj = jc.submitJob(job); this.jobID = rj.getJobID(); returnVal = jobExecHelper.progress(rj, jc, ctx); success = (returnVal == 0); String mesg = rj != null ? ("Ended Job = " + rj.getJobID()) : "Job Submission failed"; rj.killJob();
job.setOutputFormat(HiveOutputFormatImpl.class); job.setMapRunnerClass(ExecMapRunner.class); job.setMapperClass(ExecMapper.class); Path hdfsPath = mWork.getTmpHDFSPath(); hdfs.copyFromLocalFile(archivePath, hdfsFilePath); jc = new JobClient(job); rj = jc.submitJob(job); this.jobID = rj.getJobID(); updateStatusInQueryDisplay(); returnVal = jobExecHelper.progress(rj, jc, ctx); String mesg = " with exception '" + Utilities.getNameMessage(e) + "'"; if (rj != null) { mesg = "Ended Job = " + rj.getJobID() + mesg; } else { mesg = "Job Submission failed" + mesg; killJob(); jobID = rj.getID().toString(); jc.close();
edge_path = new Path(args[0]+"/pr_edge_block"); vector_path = new Path(args[0]+"/pr_iv_block"); tempmv_path = new Path(args[0]+"/pr_tempmv_block"); output_path = new Path(args[0]+"/pr_output_block"); vector_unfold_path = new Path(args[0]+"/pr_vector"); System.out.println("[PEGASUS] Computing PageRank using block method. Max iteration = " +niteration + ", threshold = " + converge_threshold + "\n"); fs = FileSystem.get(getConf()); JobClient.runJob(configStage1()); RunningJob job = JobClient.runJob(configStage2()); Counters c = job.getCounters(); long changed = c.getCounter(PrCounters.CONVERGE_CHECK); System.out.println("Iteration = " + i + ", changed reducer = " + changed); fs.delete(vector_path); fs.delete(tempmv_path); fs.rename(output_path, vector_path); break; JobClient.runJob(configStage25());
private void mrRun() throws Exception { FileSystem fs = FileSystem.get(getJobConf()); Path inputDir = new Path("input"); fs.mkdirs(inputDir); Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt"))); writer.write("hello"); writer.close(); Path outputDir = new Path("output", "output"); JobConf jobConf = new JobConf(getJobConf()); jobConf.setInt("mapred.map.tasks", 1); jobConf.setInt("mapred.map.max.attempts", 1); jobConf.setInt("mapred.reduce.max.attempts", 1); jobConf.set("mapred.input.dir", inputDir.toString()); jobConf.set("mapred.output.dir", outputDir.toString()); JobClient jobClient = new JobClient(jobConf); RunningJob runJob = jobClient.submitJob(jobConf); runJob.waitForCompletion(); assertTrue(runJob.isComplete()); assertTrue(runJob.isSuccessful()); }
private void testSuccessfulJob(String filename, Class<? extends OutputCommitter> committer, String[] exclude) throws IOException { JobConf jc = mr.createJobConf(); Path outDir = getNewOutputDir(); configureJob(jc, "job with cleanup()", 1, 0, outDir); jc.setOutputCommitter(committer); JobClient jobClient = new JobClient(jc); RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); job.waitForCompletion(); Path testFile = new Path(outDir, filename); assertTrue("Done file missing for job " + id, fileSys.exists(testFile)); // check if the files from the missing set exists for (String ex : exclude) { Path file = new Path(outDir, ex); assertFalse("File " + file + " should not be present for successful job " + id, fileSys.exists(file)); } }
/** * Test if the completed job status is persisted to localfs. */ public void testLocalPersistency() throws Exception { FileSystem fs = FileSystem.getLocal(new JobConf()); fs.delete(TEST_DIR, true); Properties config = new Properties(); config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS, "true"); config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS_DIR, fs.makeQualified(TEST_DIR).toString()); startCluster(true, config); JobID jobId = runJob(); JobClient jc = new JobClient(createJobConf()); RunningJob rj = jc.getJob(jobId); assertNotNull(rj); // check if the local fs has the data Path jobInfo = new Path(TEST_DIR, rj.getID() + ".info"); assertTrue("Missing job info from the local fs", fs.exists(jobInfo)); fs.delete(TEST_DIR, true); }
private void logJob(String logDir, String jobID, PrintWriter listWriter) throws IOException { RunningJob rj = jobClient.getJob(JobID.forName(jobID)); String jobURLString = rj.getTrackingURL(); Path jobDir = new Path(logDir, jobID); fs.mkdirs(jobDir); logJobConf(jobID, jobURLString, jobDir.toString()); } catch (IOException e) { System.err.println("Cannot retrieve job.xml.html for " + jobID); listWriter.println("job: " + jobID + "(" + "name=" + rj.getJobName() + "," + "status=" + JobStatus.getJobRunState(rj.getJobState()) + ")"); logAttempt(jobID, attempt, jobDir.toString()); listWriter.println(" attempt:" + attempt.id + "(" + "type=" + attempt.type + "," + "status=" + attempt.status + ","
JobConf conf = mrCluster.createJobConf(); conf.setJobName("name * abc + Evalue]"); conf.setInputFormat(TextInputFormat.class); conf.setLong(JobContext.MAP_MAX_ATTEMPTS, 2); makeInput(inDir, conf); JobClient jobClient = new JobClient(conf); RunningJob job = jobClient.submitJob(conf); while (job.getJobState() != JobStatus.RUNNING) { try { Thread.sleep(100); job.killJob(); job.waitForCompletion(); assertTrue(job.isComplete()); assertEquals(JobStatus.KILLED, job.getJobState()); myListener.contains(job.getID())); TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT, job.getID());
private JobID runJob() throws Exception { OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); Writer wr = new OutputStreamWriter(os); wr.write("hello1\n"); wr.write("hello2\n"); wr.write("hello3\n"); wr.write("hello4\n"); wr.close(); JobConf conf = createJobConf(); conf.setJobName("mr"); conf.setInputFormat(TextInputFormat.class); conf.setMapOutputKeyClass(LongWritable.class); conf.setMapOutputValueClass(Text.class); conf.setOutputFormat(TextOutputFormat.class); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(Text.class); conf.setMapperClass(org.apache.hadoop.mapred.lib.IdentityMapper.class); conf.setReducerClass(org.apache.hadoop.mapred.lib.IdentityReducer.class); FileInputFormat.setInputPaths(conf, getInputDir()); FileOutputFormat.setOutputPath(conf, getOutputDir()); return JobClient.runJob(conf).getID(); }
public static void main(String[] args) throws Exception { long startTime = System.currentTimeMillis(); Configuration conf = new Configuration(); JobConf jobconf = new JobConf(conf, SecondarySortProjectionDriver.class); jobconf.setJobName("SecondarySortProjectionDriver"); jobconf.setMapperClass(SecondarySortProjectionMapper.class); jobconf.setReducerClass(SecondarySortProjectionReducer.class); FileInputFormat.setInputPaths(jobconf, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(jobconf, new Path(otherArgs[1])); jobconf.setInputFormat(TextInputFormat.class); jobconf.setOutputValueGroupingComparator(NaturalKeyGroupingComparator.class); JobClient.runJob(jobconf).waitForCompletion();
dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); final Path inDir = new Path("./input"); final Path outDir = new Path("./output"); String input = "The quick brown fox\nhas many silly\nred fox sox\n"; JobConf jobConf = mr.createJobConf(); jobConf.set(Job.COMPLETION_POLL_INTERVAL_KEY, "50"); jobConf.setOutputCommitter(CommitterWithLogs.class); RunningJob rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt); fileSys.delete(outDir, true); jobConf.setOutputCommitter(CommitterWithFailTaskCleanup.class); rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt); fileSys.delete(outDir, true); jobConf.setOutputCommitter(CommitterWithFailTaskCleanup2.class); rJob = launchJob(jobConf, inDir, outDir, input); rJob.waitForCompletion(); validateJob(rJob, jt); } finally {
FileSystem fileSys = null; Configuration conf = new Configuration(); fileSys = FileSystem.get(conf); JobConf jtConf = new JobConf(); jtConf.setInt(JTConfig.JT_MAX_TRACKER_BLACKLISTS, 1); mr = new MiniMRCluster(3, fileSys.getUri().toString(), 1, null, hosts, jtConf); JobClient jc = new JobClient(mrConf); RunningJob running = JobClient.runJob(job); assertEquals("Job failed", JobStatus.SUCCEEDED, running.getJobState()); assertEquals("Did not blacklist the host", 1, jc.getClusterStatus().getBlacklistedTrackers()); assertEquals("Fault count should be 1", 1, mr.getFaultCount(hosts[0])); running = JobClient.runJob(job); assertEquals("Job failed", JobStatus.SUCCEEDED, running.getJobState()); assertEquals("Didn't blacklist the host", 1, jc.getClusterStatus().getBlacklistedTrackers()); assertEquals("Fault count should be 1", 1, mr.getFaultCount(hosts[0])); if (fileSys != null) { fileSys.close(); } if (mr!= null) { mr.shutdown(); }
jc.setMapperClass(UtilsForTests.KillMapper.class); jc.setOutputCommitter(committer); JobClient jobClient = new JobClient(jc); RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); Counters counters = job.getCounters(); counters = job.getCounters(); job.killJob(); // kill the job job.waitForCompletion(); // wait for the job to complete assertEquals("Job was not killed", JobStatus.KILLED, job.getJobState()); Path testFile = new Path(outDir, fileName); assertTrue("File " + testFile + " missing for job " + id, fileSys.exists(testFile)); Path file = new Path(outDir, ex); assertFalse("File " + file + " should not be present for killed job " + id, fileSys.exists(file));
private void testKilledJob(String fileName, Class<? extends OutputCommitter> committer, String[] exclude) throws IOException { JobConf jc = mr.createJobConf(); Path outDir = getNewOutputDir(); configureJob(jc, "kill job with abort()", 1, 0, outDir); jc.setMapperClass(UtilsForTests.KillMapper.class); jc.setOutputCommitter(committer); JobClient jobClient = new JobClient(jc); RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); JobInProgress jip = mr.getJobTrackerRunner().getJobTracker().getJob(job.getID()); job.killJob(); // kill the job job.waitForCompletion(); // wait for the job to complete Path testFile = new Path(outDir, fileName); assertTrue("File " + testFile + " missing for job " + id, fileSys.exists(testFile)); Path file = new Path(outDir, ex); assertFalse("File " + file + " should not be present for killed job " + id, fileSys.exists(file));
public void testCommitFail() throws IOException { final Path inDir = new Path(rootDir, "./input"); final Path outDir = new Path(rootDir, "./output"); JobConf jobConf = createJobConf(); jobConf.setMaxMapAttempts(1); jobConf.setOutputCommitter(CommitterWithCommitFail.class); RunningJob rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 0); rJob.waitForCompletion(); assertEquals(JobStatus.FAILED, rJob.getJobState()); }