@Override public long getFileLength() throws IOException { return _fs.getFileStatus(_path).getLen(); }
@Override public long getModTime() throws IOException { if (_modTime == null) { FileSystem fs = _path.getFileSystem(_hadoopConf); _modTime = fs.getFileStatus(_path).getModificationTime(); } return _modTime; }
/** * Get the fully-qualified path * NOTE: FileSystem.resolvePath() is not available in Hadoop 1.2.1 * @throws IOException */ public Path resolvePath(Path path) throws IOException { return fs.getFileStatus(path).getPath(); }
/** Construct given a path and a configuration. */ public HadoopInput(Path path, Configuration conf) throws IOException { this.stream = path.getFileSystem(conf).open(path); this.len = path.getFileSystem(conf).getFileStatus(path).getLen(); }
@Test public void testFromInstrumentedScheme() throws Exception { File tmpDir = Files.createTempDir(); tmpDir.deleteOnExit(); FileSystem fs = FileSystem.get(new URI(InstrumentedLocalFileSystem.SCHEME + ":///"), new Configuration()); Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("instrumented-file:///tmp")); Assert.assertEquals(fs.getUri().getScheme(), "instrumented-file"); Path basePath = new Path(tmpDir.getAbsolutePath()); Assert.assertTrue(fs.exists(basePath)); Path file = new Path(basePath, "file"); Assert.assertFalse(fs.exists(file)); fs.create(new Path(basePath, "file")); Assert.assertTrue(fs.exists(file)); Assert.assertEquals(fs.getFileStatus(file).getLen(), 0); Assert.assertEquals(fs.listStatus(basePath).length, 1); fs.delete(file, false); Assert.assertFalse(fs.exists(file)); }
@Override public long getBlockSizeByte(String path) throws IOException { Path tPath = new Path(path); FileSystem hdfs = getFs(); if (!hdfs.exists(tPath)) { throw new FileNotFoundException(path); } FileStatus fs = hdfs.getFileStatus(tPath); return fs.getBlockSize(); }
public boolean isReplayWALFinished(String wal) throws IOException { Path walPath = new Path(walRootDir, wal); return fs.getFileStatus(walPath).getLen() == 0; }
@Override public void setMode(String path, short mode) throws IOException { FileSystem hdfs = getFs(); try { FileStatus fileStatus = hdfs.getFileStatus(new Path(path)); hdfs.setPermission(fileStatus.getPath(), new FsPermission(mode)); } catch (IOException e) { LOG.warn("Fail to set permission for {} with perm {} : {}", path, mode, e.getMessage()); throw e; } }
Path rootHDFSDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR)); FsPermission writableHDFSDirPermission = new FsPermission((short)00733); FileSystem fs = rootHDFSDirPath.getFileSystem(conf); if (!fs.exists(rootHDFSDirPath)) { Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true); FsPermission currentHDFSDirPermission = fs.getFileStatus(rootHDFSDirPath).getPermission(); if (rootHDFSDirPath != null && rootHDFSDirPath.toUri() != null) { String schema = rootHDFSDirPath.toUri().getScheme();
/** * @param conf * @return path to destination directory on hdfs * @throws LoginException if we are unable to figure user information * @throws IOException when any dfs operation fails. */ @SuppressWarnings("deprecation") public Path getDefaultDestDir(Configuration conf) throws LoginException, IOException { UserGroupInformation ugi = Utils.getUGI(); String userName = ugi.getShortUserName(); String userPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_USER_INSTALL_DIR); Path userPath = new Path(userPathStr); FileSystem fs = userPath.getFileSystem(conf); String jarPathStr = userPathStr + "/" + userName; String hdfsDirPathStr = jarPathStr; Path hdfsDirPath = new Path(hdfsDirPathStr); try { FileStatus fstatus = fs.getFileStatus(hdfsDirPath); if (!fstatus.isDir()) { throw new IOException(ErrorMsg.INVALID_DIR.format(hdfsDirPath.toString())); } } catch (FileNotFoundException e) { // directory does not exist, create it fs.mkdirs(hdfsDirPath); } Path retPath = new Path(hdfsDirPath.toString() + "/.hiveJars"); fs.mkdirs(retPath); return retPath; }
/** * @param fs * @param dir Directory to create file in. * @return random filename inside passed <code>dir</code> */ static Path getUniqueFile(final FileSystem fs, final Path dir) throws IOException { if (!fs.getFileStatus(dir).isDirectory()) { throw new IOException("Expecting " + dir.toString() + " to be a directory"); } return new Path(dir, dash.matcher(UUID.randomUUID().toString()).replaceAll("")); }
/** Construct given a path and a {@code FileSystem}. */ public FsInput(Path path, FileSystem fileSystem) throws IOException { this.len = fileSystem.getFileStatus(path).getLen(); this.stream = fileSystem.open(path); }
private void createNewLogFileIfNeeded() throws IOException { if (LogCopier.this.destFs.exists(this.destLogFile) && (this.watch.elapsed(TimeUnit.MINUTES) > LogCopier.this.maxMinutesPerLogFile || LogCopier.this.destFs.getFileStatus(this.destLogFile).getLen() > LogCopier.this.maxBytesPerLogFile)) { HadoopUtils.renamePath(LogCopier.this.destFs, this.destLogFile, new Path(this.destLogFile.toString() + "." + System.currentTimeMillis())); this.watch.reset(); this.watch.start(); } }
public String decrypt(final String cipheredText, final String passphrasePath, final FileSystem fs) throws IOException { Preconditions.checkNotNull(cipheredText); Preconditions.checkNotNull(passphrasePath); final Path path = new Path(passphrasePath); Preconditions.checkArgument(fs.exists(path), "File does not exist at " + passphrasePath); Preconditions .checkArgument(fs.isFile(path), "Passphrase path is not a file. " + passphrasePath); final FileStatus fileStatus = fs.getFileStatus(path); Preconditions.checkArgument(USER_READ_PERMISSION_ONLY.equals(fileStatus.getPermission()), "Passphrase file should only have read only permission on only user. " + passphrasePath); final Crypto crypto = new Crypto(); try (BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path), Charset.defaultCharset()))) { final String passphrase = br.readLine(); final String decrypted = crypto.decrypt(cipheredText, passphrase); Preconditions.checkNotNull(decrypted, "Was not able to decrypt"); return decrypted; } } }
@Override public void close(boolean abort) throws IOException { recordWriter.close(abort); if (!abort) { length = target.getFileSystem(conf).getFileStatus(target).getLen(); } } };
private void appendTmpDir(Job job, FileSystem fs, Path tmpDir, StringBuilder jarList, StringBuilder fileList) { try { FileStatus[] fList = fs.listStatus(tmpDir); for (FileStatus file : fList) { Path p = file.getPath(); if (fs.getFileStatus(p).isDirectory()) { appendTmpDir(job, fs, p, jarList, fileList); continue; } StringBuilder list = (p.getName().endsWith(".jar")) ? jarList : fileList; if (list.length() > 0) list.append(","); list.append(fs.getFileStatus(p).getPath().toString()); } } catch (IOException e) { throw new RuntimeException(e); } }
@Override protected RawResource getResourceImpl(String resPath) throws IOException { Path p = getRealHDFSPath(resPath); if (fs.exists(p) && fs.isFile(p)) { FileStatus fileStatus = fs.getFileStatus(p); if (fileStatus.getLen() == 0) { logger.warn("Zero length file: {}. ", p); } FSDataInputStream in = fs.open(p); long ts = fileStatus.getModificationTime(); return new RawResource(resPath, ts, in); } else { return null; } }
public boolean refresh(final Path path) throws IOException { if (_fileStatus.isPresent()) { Optional<FileStatus> oldStatus = this._fileStatus; try { this._fileStatus = Optional.of(this.fs.getFileStatus(path)); this.exists = this._fileStatus.isPresent(); return (oldStatus.isPresent() != this._fileStatus.isPresent() || oldStatus.get().getModificationTime() != this._fileStatus.get().getModificationTime() || oldStatus.get().isDirectory() != this._fileStatus.get().isDirectory() || oldStatus.get().getLen() != this._fileStatus.get().getLen()); } catch (FileNotFoundException e) { _fileStatus = Optional.absent(); this.exists = false; return true; } } else { if (path.getFileSystem(new Configuration()).exists(path)) { _fileStatus = Optional.of(this.fs.getFileStatus(path)); return true; } else { return false; } } }
/** * Checks whether existing file in filesystem is the expected file (compares length and modificaiton time). */ private boolean existsAndIsExpectedFile(FileStatus status) throws IOException { if (!getFS().exists(status.getPath())) { return false; } FileStatus currentFileStatus = getFS().getFileStatus(status.getPath()); if (currentFileStatus.getLen() != status.getLen() || currentFileStatus.getModificationTime() > status.getModificationTime()) { return false; } return true; }
/** @throws Exception If failed. */ @Test public void testOpen() throws Exception { Path fsHome = new Path(primaryFsUri); Path file = new Path(fsHome, "someFile"); int cnt = 2 * 1024; try (FSDataOutputStream out = fs.create(file, true, 1024)) { for (long i = 0; i < cnt; i++) out.writeLong(i); } assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner()); try (FSDataInputStream in = fs.open(file, 1024)) { for (long i = 0; i < cnt; i++) assertEquals(i, in.readLong()); } }