private File createTempFile(long size) throws IOException { File result = dirAllocator.createTmpFileForWrite(FILENAME, size, conf); result.delete(); return result; }
private File createTempFile() throws IOException { File result = dirAllocator.createTmpFileForWrite(FILENAME, -1, conf); result.delete(); return result; }
public static synchronized File getTempBufferDir(Configuration conf) throws IOException { File tmpFile = dirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf); String dir = tmpFile.toPath().getParent().toUri().getPath(); LOG.debug("choose oss buffer dir: " + dir); return new File(dir, "data/" + loginUser + "/oss"); }
synchronized File createTmpFileForWrite(String pathStr, long size) throws IOException { LOG.trace("Create temp file for write {}. size {}", pathStr, size); if (directoryAllocator == null) { String bufferTargetDir = !bufferDir.isEmpty() ? BUFFER_DIR : "hadoop.tmp.dir"; LOG.trace("Local buffer directorykey is {}", bufferTargetDir); directoryAllocator = new LocalDirAllocator(bufferTargetDir); } return directoryAllocator.createTmpFileForWrite(pathStr, size, conf); }
private File newBackupFile() throws IOException { if (lDirAlloc == null) { lDirAlloc = new LocalDirAllocator("fs.s3.buffer.dir"); } File result = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf); result.deleteOnExit(); return result; }
/** * Create a temp file. * @param conf configuration to use when creating the allocator * @param prefix filename prefix * @param size file size, or -1 if not known * @return the temp file. The file has been created. * @throws IOException IO failure */ public static File tempFile(Configuration conf, String prefix, long size) throws IOException { return getAllocator(conf, BUFFER_DIR).createTmpFileForWrite( prefix, size, conf); }
synchronized File createTmpFileForWrite(String pathStr, long size) throws IOException { LOG.trace("Create temp file for write {}. size {}", pathStr, size); if (directoryAllocator == null) { String bufferTargetDir = !bufferDir.isEmpty() ? BUFFER_DIR : "hadoop.tmp.dir"; LOG.trace("Local buffer directorykey is {}", bufferTargetDir); directoryAllocator = new LocalDirAllocator(bufferTargetDir); } return directoryAllocator.createTmpFileForWrite(pathStr, size, conf); }
/** * Demand create the directory allocator, then create a temporary file. * @param path prefix for the temporary file * @param size the size of the file that is going to be written * @param conf the Configuration object * @return a unique temporary file * @throws IOException IO problems */ public static File createTmpFileForWrite(String path, long size, Configuration conf) throws IOException { if (conf.get(BUFFER_DIR_KEY) == null) { conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss"); } if (directoryAllocator == null) { directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY); } return directoryAllocator.createTmpFileForWrite(path, size, conf); }
/** * Demand create the directory allocator, then create a temporary file. * {@link LocalDirAllocator#createTmpFileForWrite(String, long, Configuration)}. * @param pathStr prefix for the temporary file * @param size the size of the file that is going to be written * @param conf the Configuration object * @return a unique temporary file * @throws IOException IO problems */ synchronized File createTmpFileForWrite(String pathStr, long size, Configuration conf) throws IOException { if (directoryAllocator == null) { String bufferDir = conf.get(BUFFER_DIR) != null ? BUFFER_DIR : HADOOP_TMP_DIR; directoryAllocator = new LocalDirAllocator(bufferDir); } return directoryAllocator.createTmpFileForWrite(pathStr, size, conf); }
private static File createTempFile(long size) throws IOException { File result = dirAllocator.createTmpFileForWrite(FILENAME, size, conf); result.delete(); return result; }
private static File createTempFile(long size) throws IOException { File result = dirAllocator.createTmpFileForWrite(FILENAME, size, conf); result.delete(); return result; }
public S3AOutputStream(Configuration conf, AmazonS3Client client, S3AFileSystem fs, String bucket, String key, Progressable progress, CannedAccessControlList cannedACL, FileSystem.Statistics statistics, String serverSideEncryptionAlgorithm) throws IOException { this.bucket = bucket; this.key = key; this.client = client; this.progress = progress; this.fs = fs; this.cannedACL = cannedACL; this.statistics = statistics; this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; partSize = conf.getLong(NEW_MULTIPART_SIZE, conf.getLong(OLD_MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE)); partSizeThreshold = conf.getInt(NEW_MIN_MULTIPART_THRESHOLD, conf.getInt(OLD_MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD)); if (conf.get(BUFFER_DIR, null) != null) { lDirAlloc = new LocalDirAllocator(BUFFER_DIR); } else { lDirAlloc = new LocalDirAllocator("${hadoop.tmp.dir}/s3a"); } backupFile = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf); closed = false; LOG.info("OutputStream for key '" + key + "' writing to tempfile: " + this.backupFile); this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile)); }
public S3AOutputStream(Configuration conf, TransferManager transfers, S3AFileSystem fs, String bucket, String key, Progressable progress, CannedAccessControlList cannedACL, FileSystem.Statistics statistics, String serverSideEncryptionAlgorithm) throws IOException { this.bucket = bucket; this.key = key; this.transfers = transfers; this.progress = progress; this.fs = fs; this.cannedACL = cannedACL; this.statistics = statistics; this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); partSizeThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD); if (conf.get(BUFFER_DIR, null) != null) { lDirAlloc = new LocalDirAllocator(BUFFER_DIR); } else { lDirAlloc = new LocalDirAllocator("${hadoop.tmp.dir}/s3a"); } backupFile = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf); closed = false; if (LOG.isDebugEnabled()) { LOG.debug("OutputStream for key '" + key + "' writing to tempfile: " + this.backupFile); } this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile)); }
/** Test no side effect files are left over. After creating a temp * temp file, remove both the temp file and its parent. Verify that * no files or directories are left over as can happen when File objects * are mistakenly created from fully qualified path strings. * @throws IOException */ @Test (timeout = 30000) public void testNoSideEffects() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); File result = dirAllocator.createTmpFileForWrite(FILENAME, -1, conf); assertTrue(result.delete()); assertTrue(result.getParentFile().delete()); assertFalse(new File(dir).exists()); } finally { Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, BUFFER_DIR_ROOT)); rmBufferDirs(); } }
/** Test no side effect files are left over. After creating a temp * temp file, remove both the temp file and its parent. Verify that * no files or directories are left over as can happen when File objects * are mistakenly created from fully qualified path strings. * @throws IOException */ @Test (timeout = 30000) public void testNoSideEffects() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); File result = dirAllocator.createTmpFileForWrite(FILENAME, -1, conf); assertTrue(result.delete()); assertTrue(result.getParentFile().delete()); assertFalse(new File(dir).exists()); } finally { Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, BUFFER_DIR_ROOT)); rmBufferDirs(); } }
/** * Test getLocalPathToRead() returns correct filename and "file" schema. * * @throws IOException */ @Test (timeout = 30000) public void testGetLocalPathToRead() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); assertTrue(localFs.mkdirs(new Path(dir))); File f1 = dirAllocator.createTmpFileForWrite(FILENAME, SMALL_FILE_SIZE, conf); Path p1 = dirAllocator.getLocalPathToRead(f1.getName(), conf); assertEquals(f1.getName(), p1.getName()); assertEquals("file", p1.getFileSystem(conf).getUri().getScheme()); } finally { Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, BUFFER_DIR_ROOT)); rmBufferDirs(); } }
/** * Test getLocalPathToRead() returns correct filename and "file" schema. * * @throws IOException */ @Test (timeout = 30000) public void testGetLocalPathToRead() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); assertTrue(localFs.mkdirs(new Path(dir))); File f1 = dirAllocator.createTmpFileForWrite(FILENAME, SMALL_FILE_SIZE, conf); Path p1 = dirAllocator.getLocalPathToRead(f1.getName(), conf); assertEquals(f1.getName(), p1.getName()); assertEquals("file", p1.getFileSystem(conf).getUri().getScheme()); } finally { Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, BUFFER_DIR_ROOT)); rmBufferDirs(); } }