conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); MapTask mockTask = mock(MapTask.class);
@Override public void setConf(Configuration conf) { if (!(conf instanceof JobConf)) { conf = new JobConf(conf); } super.setConf(conf); }
@Override public void setConf(Configuration conf) { if (!(conf instanceof JobConf)) { conf = new JobConf(conf); } super.setConf(conf); }
@Override public void setConf(Configuration conf) { if (!(conf instanceof JobConf)) { conf = new JobConf(conf); } super.setConf(conf); }
@Override public void setConf(Configuration conf) { if (!(conf instanceof JobConf)) { conf = new JobConf(conf); } super.setConf(conf); }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile.setConf(this.conf); this.lDirAlloc = new LocalDirAllocator("mapred.local.dir"); loadStaticResolutions(conf); }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile.setConf(this.conf); this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical // node. String hostToResolved[] = conf.getStrings(TTConfig.TT_STATIC_RESOLUTIONS); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile.setConf(this.conf); this.lDirAlloc = new LocalDirAllocator("mapred.local.dir"); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical // node. String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions"); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
mapOutputFile.setConf(conf);
public Job(JobID jobid, JobConf conf) throws IOException { this.file = new Path(getSystemDir(), jobid + "/job.xml"); this.id = jobid; this.mapoutputFile = new MapOutputFile(jobid); this.mapoutputFile.setConf(conf); this.localFile = new JobConf(conf).getLocalPath(jobDir+id+".xml"); this.localFs = FileSystem.getLocal(conf); fs.copyToLocalFile(file, localFile); this.job = new JobConf(localFile); profile = new JobProfile(job.getUser(), id, file.toString(), "http://localhost:8080/", job.getJobName()); status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING); jobs.put(id, this); this.start(); }
public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = tip.getTask(); this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(t.getJobID()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); }
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
public Job(JobID jobid, JobConf conf) throws IOException { this.doSequential = conf.getBoolean("mapred.localrunner.sequential", true); this.id = jobid; this.mapoutputFile = new MapOutputFile(jobid); this.mapoutputFile.setConf(conf); this.localFile = new JobConf(conf).getLocalPath(jobDir+id+".xml"); this.localFs = FileSystem.getLocal(conf); persistConf(this.localFs, this.localFile, conf); this.job = new JobConf(localFile); profile = new JobProfile(job.getUser(), id, localFile.toString(), "http://localhost:8080/", job.getJobName()); status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING); jobs.put(id, this); numSlots = conf.getInt(LOCAL_RUNNER_SLOTS, DEFAULT_LOCAL_RUNNER_SLOTS); executor = Executors.newFixedThreadPool(numSlots); int handlerCount = numSlots; umbilicalServer = RPC.getServer(this, LOCALHOST, 0, handlerCount, false, conf); umbilicalServer.start(); umbilicalPort = umbilicalServer.getListenerAddress().getPort(); this.start(); }
public TaskRunner( TaskTracker.TaskInProgress tip, Task task, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = task; this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(task.getJobID(), tracker.getAsyncDiskService()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); }
mapOutput.setConf(localConf); mapOutputFiles.put(mapId, mapOutput);
mapOutput.setConf(localConf); mapOutputFiles.put(mapId, mapOutput);
mapOutput.setConf(localConf); mapOutputFiles.put(mapId, mapOutput);
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MROutputFiles(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }