int origFactor = factor; int passNo = 1; LocalDirAllocator lDirAlloc = new LocalDirAllocator("io.seqfile.local.dir"); do {
private static synchronized LocalDirAllocator getAllocator( Configuration conf, String key) { if (directoryAllocator != null) { String bufferDir = conf.get(key) != null ? key : Constants.HADOOP_TMP_DIR; directoryAllocator = new LocalDirAllocator(bufferDir); } return directoryAllocator; }
new TaskID(jobId, TaskType.MAP, 2), 0); LocalDirAllocator lda = new LocalDirAllocator(MRConfig.LOCAL_DIR);
private File newBackupFile() throws IOException { if (lDirAlloc == null) { lDirAlloc = new LocalDirAllocator("fs.s3.buffer.dir"); } File result = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf); result.deleteOnExit(); return result; }
synchronized File createTmpFileForWrite(String pathStr, long size) throws IOException { LOG.trace("Create temp file for write {}. size {}", pathStr, size); if (directoryAllocator == null) { String bufferTargetDir = !bufferDir.isEmpty() ? BUFFER_DIR : "hadoop.tmp.dir"; LOG.trace("Local buffer directorykey is {}", bufferTargetDir); directoryAllocator = new LocalDirAllocator(bufferTargetDir); } return directoryAllocator.createTmpFileForWrite(pathStr, size, conf); }
synchronized File createTmpFileForWrite(String pathStr, long size) throws IOException { LOG.trace("Create temp file for write {}. size {}", pathStr, size); if (directoryAllocator == null) { String bufferTargetDir = !bufferDir.isEmpty() ? BUFFER_DIR : "hadoop.tmp.dir"; LOG.trace("Local buffer directorykey is {}", bufferTargetDir); directoryAllocator = new LocalDirAllocator(bufferTargetDir); } return directoryAllocator.createTmpFileForWrite(pathStr, size, conf); }
public TrackerDistributedCacheManager(Configuration conf, TaskController taskController) throws IOException { this.localFs = FileSystem.getLocal(conf); this.trackerConf = conf; this.lDirAllocator = new LocalDirAllocator(TTConfig.LOCAL_DIR); this.taskController = taskController; // setting the cache size to a default of 10GB this.allowedCacheSize = conf.getLong(TTConfig.TT_LOCAL_CACHE_SIZE, DEFAULT_CACHE_SIZE); // setting the cache number of subdirectories limit to a default of 10000 this.allowedCacheSubdirs = conf.getLong( TTConfig.TT_LOCAL_CACHE_SUBDIRS_LIMIT, DEFAULT_CACHE_SUBDIR_LIMIT); this.cleanupThread = new CleanupThread(conf); }
/** * Demand create the directory allocator, then create a temporary file. * {@link LocalDirAllocator#createTmpFileForWrite(String, long, Configuration)}. * @param pathStr prefix for the temporary file * @param size the size of the file that is going to be written * @param conf the Configuration object * @return a unique temporary file * @throws IOException IO problems */ synchronized File createTmpFileForWrite(String pathStr, long size, Configuration conf) throws IOException { if (directoryAllocator == null) { String bufferDir = conf.get(BUFFER_DIR) != null ? BUFFER_DIR : HADOOP_TMP_DIR; directoryAllocator = new LocalDirAllocator(bufferDir); } return directoryAllocator.createTmpFileForWrite(pathStr, size, conf); }
public void cleanup(String strPath) { if (deletionService == null) return; LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); try { Iterable<Path> iter = lDirAllocator.getAllLocalPathsToRead(strPath, systemConf); FileSystem localFS = FileSystem.getLocal(systemConf); for (Path path : iter) { deletionService.delete(localFS.makeQualified(path)); } } catch (IOException e) { LOG.error(e.getMessage(), e); } }
@VisibleForTesting //TODO: Refactor following to make use of methods from TezTaskOutputFiles to be consistent. protected Path getShuffleInputFileName(String pathComponent, String suffix) throws IOException { LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); suffix = suffix != null ? suffix : ""; String outputPath = Constants.TEZ_RUNTIME_TASK_OUTPUT_DIR + Path.SEPARATOR + pathComponent + Path.SEPARATOR + Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING + suffix; String pathFromLocalDir = getPathForLocalDir(outputPath); return localDirAllocator.getLocalPathToRead(pathFromLocalDir.toString(), conf); }
public void cleanup(String strPath) { if (deletionService == null) return; LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); try { Iterable<Path> iter = lDirAllocator.getAllLocalPathsToRead(strPath, systemConf); FileSystem localFS = FileSystem.getLocal(systemConf); for (Path path : iter) { deletionService.delete(localFS.makeQualified(path)); } } catch (IOException e) { LOG.error(e.getMessage(), e); } }
public HashShuffleAppenderManager(TajoConf systemConf) throws IOException { this.systemConf = systemConf; // initialize LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); // initialize DFS and LocalFileSystems defaultFS = TajoConf.getTajoRootDir(systemConf).getFileSystem(systemConf); localFS = FileSystem.getLocal(systemConf); pageSize = systemConf.getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME) * 1024 * 1024; }
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile.setConf(this.conf); this.lDirAlloc = new LocalDirAllocator("mapred.local.dir"); loadStaticResolutions(conf); }
public HashShuffleAppenderManager(TajoConf systemConf) throws IOException { this.systemConf = systemConf; // initialize LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); // initialize DFS and LocalFileSystems defaultFS = TajoConf.getTajoRootDir(systemConf).getFileSystem(systemConf); localFS = FileSystem.getLocal(systemConf); pageSize = systemConf.getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME) * StorageUnit.MB; Iterable<Path> allLocalPath = lDirAllocator.getAllLocalPathsToRead(".", systemConf); //add async hash shuffle writer for (Path path : allLocalPath) { temporalPaths.add(localFS.makeQualified(path).toString()); executors.put(temporalPaths.size() - 1, Executors.newSingleThreadExecutor()); } }
public HashShuffleAppenderManager(TajoConf systemConf) throws IOException { this.systemConf = systemConf; // initialize LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); // initialize DFS and LocalFileSystems defaultFS = TajoConf.getTajoRootDir(systemConf).getFileSystem(systemConf); localFS = FileSystem.getLocal(systemConf); pageSize = systemConf.getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME) * StorageUnit.MB; Iterable<Path> allLocalPath = lDirAllocator.getAllLocalPathsToRead(".", systemConf); //add async hash shuffle writer for (Path path : allLocalPath) { temporalPaths.add(localFS.makeQualified(path).toString()); executors.put(temporalPaths.size() - 1, Executors.newSingleThreadExecutor()); } }
protected void initNettyMapOutputHttpServer(JobConf conf) throws IOException { int nettyHttpPort = conf.getInt(NETTY_MAPOUTPUT_HTTP_PORT, 0); NettyMapOutputAttributes attributes = new NettyMapOutputAttributes( conf, this, FileSystem.getLocal(conf), new LocalDirAllocator("mapred.local.dir"), shuffleServerMetrics); nettyMapOutputServer = new NettyMapOutputHttpServer(nettyHttpPort); nettyMapOutputServer.init(conf); shuffleServerMetrics.setNettyWorkerThreadPool( nettyMapOutputServer.getWorkerThreadPool()); HttpMapOutputPipelineFactory pipelineFactory = new HttpMapOutputPipelineFactory(attributes, nettyHttpPort); this.nettyMapOutputHttpPort = nettyMapOutputServer.start(pipelineFactory); }
private Path getAttemptJobXml(JobConf conf, JobID jobId, TaskType taskType) throws IOException, LoginException { String taskid = new TaskAttemptID(new TaskID(jobId, taskType, 0), 0).toString(); return new LocalDirAllocator(MRConfig.LOCAL_DIR).getLocalPathToRead( TaskTracker.getTaskConfFile(UserGroupInformation.getCurrentUser() .getUserName(), jobId.toString(), taskid, false), conf); }
@Test (timeout = 30000) public void testRemoveContext() throws IOException { String dir = buildBufferDir(ROOT, 0); try { String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs"; conf.set(contextCfgItemName, dir); LocalDirAllocator localDirAllocator = new LocalDirAllocator( contextCfgItemName); localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf); assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); LocalDirAllocator.removeContext(contextCfgItemName); assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); } finally { rmBufferDirs(); } }
@Test (timeout = 30000) public void testRemoveContext() throws IOException { String dir = buildBufferDir(ROOT, 0); try { String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs"; conf.set(contextCfgItemName, dir); LocalDirAllocator localDirAllocator = new LocalDirAllocator( contextCfgItemName); localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf); assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); LocalDirAllocator.removeContext(contextCfgItemName); assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); } finally { rmBufferDirs(); } }
private ExternalSortExec(final TaskAttemptContext context, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); this.plan = plan; this.defaultFanout = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT); if (defaultFanout < 2) { throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. this.sortBufferBytesNum = context.getQueryContext().getInt(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); this.localFS = new RawLocalFileSystem(); this.intermediateMeta = CatalogUtil.newTableMeta(BuiltinStorages.DRAW, context.getConf()); this.inputStats = new TableStats(); }