/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath(MRConfig.LOCAL_DIR, pathString); }
/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath(MRConfig.LOCAL_DIR, pathString); }
/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath(MRConfig.LOCAL_DIR, pathString); }
/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath(MRConfig.LOCAL_DIR, pathString); }
/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath(MRConfig.LOCAL_DIR, pathString); }
/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath("mapred.local.dir", pathString); }
/** * Constructs a local file name. Files are distributed among configured * local directories. */ public Path getLocalPath(String pathString) throws IOException { return getLocalPath("mapred.local.dir", pathString); }
public static void copyJobFileLocally(Path jobDir, JobID jobid, JobConf default_conf) throws IOException { FileSystem fs = jobDir.getFileSystem(default_conf); JobConf default_job_conf = new JobConf(default_conf); Path localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR + "/" + jobid + ".xml"); Path jobFile = new Path(jobDir, "job.xml"); fs.copyToLocalFile(jobFile, localJobFile); }
@Override public void configure(JobConf conf) { // Create unique local directory String uniqueString = UUID.randomUUID().toString(); String localTmpOutputPath; try { localTmpOutputPath = conf.getLocalPath(uniqueString).toString(); } catch (IOException e) { throw new RuntimeException("Failed to determine local temporary output directory", e); } localTmpOutput = new File(localTmpOutputPath); if (localTmpOutput.exists() || !localTmpOutput.mkdirs()) { throw new RuntimeException("Failed to initialize local temporary output directory " + localTmpOutputPath); } // Load configuration items final String domainName = DomainBuilderProperties.getDomainName(conf); final int versionNumberToCompact = DomainCompactorProperties.getVersionNumberToCompact(domainName, conf); // Create Coordinator coordinator = RunWithCoordinator.createCoordinator(DomainBuilderProperties.getConfigurator(conf)); // Determine version to compact try { Domain domain = DomainBuilderProperties.getDomain(coordinator, domainName); HadoopDomainCompactorMapper.this.storageEngine = domain.getStorageEngine(); HadoopDomainCompactorMapper.this.domainVersionToCompact = DomainBuilderProperties.getDomainVersion(coordinator, domainName, versionNumberToCompact); } catch (IOException e) { throw new RuntimeException("Failed to load configuration.", e); } }
public Job(JobID jobid, JobConf conf) throws IOException { this.file = new Path(getSystemDir(), jobid + "/job.xml"); this.id = jobid; this.mapoutputFile = new MapOutputFile(jobid); this.mapoutputFile.setConf(conf); this.localFile = new JobConf(conf).getLocalPath(jobDir+id+".xml"); this.localFs = FileSystem.getLocal(conf); fs.copyToLocalFile(file, localFile); this.job = new JobConf(localFile); profile = new JobProfile(job.getUser(), id, file.toString(), "http://localhost:8080/", job.getJobName()); status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING); jobs.put(id, this); this.start(); }
public Job(JobID jobid, JobConf conf) throws IOException { this.doSequential = conf.getBoolean("mapred.localrunner.sequential", true); this.id = jobid; this.mapoutputFile = new MapOutputFile(jobid); this.mapoutputFile.setConf(conf); this.localFile = new JobConf(conf).getLocalPath(jobDir+id+".xml"); this.localFs = FileSystem.getLocal(conf); persistConf(this.localFs, this.localFile, conf); this.job = new JobConf(localFile); profile = new JobProfile(job.getUser(), id, localFile.toString(), "http://localhost:8080/", job.getJobName()); status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING); jobs.put(id, this); numSlots = conf.getInt(LOCAL_RUNNER_SLOTS, DEFAULT_LOCAL_RUNNER_SLOTS); executor = Executors.newFixedThreadPool(numSlots); int handlerCount = numSlots; umbilicalServer = RPC.getServer(this, LOCALHOST, 0, handlerCount, false, conf); umbilicalServer.start(); umbilicalPort = umbilicalServer.getListenerAddress().getPort(); this.start(); }
String user = UserGroupInformation.getCurrentUser().getShortUserName(); this.localJobDir = localFs.makeQualified(new Path( new Path(conf.getLocalPath(jobDir), user), jobid.toString())); this.localJobFile = new Path(this.localJobDir, id + ".xml");
String user = UserGroupInformation.getCurrentUser().getShortUserName(); this.localJobDir = localFs.makeQualified(new Path( new Path(conf.getLocalPath(jobDir), user), jobid.toString())); this.localJobFile = new Path(this.localJobDir, id + ".xml");
String user = UserGroupInformation.getCurrentUser().getShortUserName(); this.localJobDir = localFs.makeQualified(new Path( new Path(conf.getLocalPath(jobDir), user), jobid.toString())); this.localJobFile = new Path(this.localJobDir, id + ".xml");
String user = UserGroupInformation.getCurrentUser().getShortUserName(); this.localJobDir = localFs.makeQualified(new Path( new Path(conf.getLocalPath(jobDir), user), jobid.toString())); this.localJobFile = new Path(this.localJobDir, id + ".xml");
JobConf conf = new JobConf(systemJobFile); this.localFs = FileSystem.getLocal(conf); this.localJobDir = localFs.makeQualified(conf.getLocalPath(jobDir)); this.localJobFile = new Path(this.localJobDir, id + ".xml");
this.localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR +"/"+jobid + ".xml"); this.localJarFile = default_job_conf.getLocalPath(JobTracker.SUBDIR +"/"+ jobid + ".jar"); Path sysDir = new Path(this.jobtracker.getSystemDir());
TaskAttemptID mapId = new TaskAttemptID(new TaskID(jobId, true, i),0); mapIds.add(mapId); Path taskJobFile = job.getLocalPath(jobDir + id + "_" + mapId + ".xml"); MapTask map = new MapTask(taskJobFile.toString(), mapId, i,
this.localJobFile = default_job_conf.getLocalPath(JobTracker.SUBDIR +"/"+jobid + ".xml");
this.localJobFile = default_conf.getLocalPath(JobTracker.SUBDIR + "/" + this.jobId + ".xml");