return -1; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setJobId(taskId.getJobID()); mapOutputFile.setConf(conf);
public Job(JobID jobid, JobConf conf) throws IOException { this.file = new Path(getSystemDir(), jobid + "/job.xml"); this.id = jobid; this.mapoutputFile = new MapOutputFile(jobid); this.mapoutputFile.setConf(conf); this.localFile = new JobConf(conf).getLocalPath(jobDir+id+".xml"); this.localFs = FileSystem.getLocal(conf); fs.copyToLocalFile(file, localFile); this.job = new JobConf(localFile); profile = new JobProfile(job.getUser(), id, file.toString(), "http://localhost:8080/", job.getJobName()); status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING); jobs.put(id, this); this.start(); }
public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = tip.getTask(); this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(t.getJobID()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); }
MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setJobId(taskId.getJobID()); mapOutputFile.setConf(conf);
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
public Job(JobID jobid, JobConf conf) throws IOException { this.doSequential = conf.getBoolean("mapred.localrunner.sequential", true); this.id = jobid; this.mapoutputFile = new MapOutputFile(jobid); this.mapoutputFile.setConf(conf); this.localFile = new JobConf(conf).getLocalPath(jobDir+id+".xml"); this.localFs = FileSystem.getLocal(conf); persistConf(this.localFs, this.localFile, conf); this.job = new JobConf(localFile); profile = new JobProfile(job.getUser(), id, localFile.toString(), "http://localhost:8080/", job.getJobName()); status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING); jobs.put(id, this); numSlots = conf.getInt(LOCAL_RUNNER_SLOTS, DEFAULT_LOCAL_RUNNER_SLOTS); executor = Executors.newFixedThreadPool(numSlots); int handlerCount = numSlots; umbilicalServer = RPC.getServer(this, LOCALHOST, 0, handlerCount, false, conf); umbilicalServer.start(); umbilicalPort = umbilicalServer.getListenerAddress().getPort(); this.start(); }
public TaskRunner( TaskTracker.TaskInProgress tip, Task task, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = task; this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(task.getJobID(), tracker.getAsyncDiskService()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); }
TaskRunner.setupChildMapredLocalDirs(map, localConf); MapOutputFile mapOutput = new MapOutputFile(); mapOutput.setConf(localConf); mapOutputFiles.put(mapId, mapOutput);
TaskAttemptID mapId = mapIds.get(i); Path mapOut = mapOutputFiles.get(mapId).getOutputFile(); MapOutputFile localOutputFile = new MapOutputFile(); localOutputFile.setConf(localConf); Path reduceIn =
this.spilledRecordsCounter = spilledRecordsCounter; this.mergedMapOutputsCounter = mergedMapOutputsCounter; this.mapOutputFile = new MapOutputFile(); this.mapOutputFile.setConf(jobConf);