@Override public long getModTime() throws IOException { if (_modTime == null) { FileSystem fs = _path.getFileSystem(_hadoopConf); _modTime = fs.getFileStatus(_path).getModificationTime(); } return _modTime; }
private static boolean sameFile(FileStatus fileInSource, FileStatus fileInTarget) { return fileInTarget.getLen() == fileInSource.getLen() && fileInSource.getModificationTime() <= fileInTarget .getModificationTime(); } }
/** * Create a Store File Info from an HFileLink * @param conf The {@link Configuration} to use * @param fs The current file system to use * @param fileStatus The {@link FileStatus} of the file * @param reference The reference instance */ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, final Reference reference) { this.fs = fs; this.conf = conf; this.initialPath = fileStatus.getPath(); this.createdTimestamp = fileStatus.getModificationTime(); this.reference = reference; this.link = null; }
private String getKey(final FileStatus fileStatus) { return fileStatus.getPath() + ":" + fileStatus.getLen() + ":" + fileStatus.getModificationTime(); }
public static Object getFileId( FileSystem fileSystem, Path path, boolean allowSynthetic) throws IOException { if (fileSystem instanceof DistributedFileSystem) { return SHIMS.getFileId(fileSystem, path.toUri().getPath()); } if (!allowSynthetic) { LOG.warn("Cannot get unique file ID from " + fileSystem.getClass().getSimpleName() + "; returning null"); return null; } FileStatus fs = fileSystem.getFileStatus(path); return new SyntheticFileId(path, fs.getLen(), fs.getModificationTime()); }
/** * Checks whether existing file in filesystem is the expected file (compares length and modificaiton time). */ private boolean existsAndIsExpectedFile(FileStatus status) throws IOException { if (!getFS().exists(status.getPath())) { return false; } FileStatus currentFileStatus = getFS().getFileStatus(status.getPath()); if (currentFileStatus.getLen() != status.getLen() || currentFileStatus.getModificationTime() > status.getModificationTime()) { return false; } return true; }
public static long getModificationTimeMsRecursive(String path) throws IOException { FileSystem fs = getFileSystem(path); Path fsPath = new Path(path); FileStatus status = fs.getFileStatus(fsPath); long modificationTime = status.getModificationTime(); FileStatus[] statuses = fs.listStatus(fsPath); if (statuses != null) { for (FileStatus fileStatus : statuses) { Path statusPath = fileStatus.getPath(); String stringPath; if (path.startsWith("s3://") || path.startsWith("s3n://") || path.startsWith("s3a://") || path.startsWith("swift://") || path.startsWith("gs://")) { stringPath = statusPath.toUri().toString(); } else { stringPath = statusPath.toUri().getPath(); } if (!stringPath.equals(path)) { modificationTime = Math.max(modificationTime, getModificationTimeMsRecursive(stringPath)); } } } return modificationTime; }
public SyntheticFileId(FileStatus file) { this(file.getPath(), file.getLen(), file.getModificationTime()); }
public boolean refresh(final Path path) throws IOException { if (_fileStatus.isPresent()) { Optional<FileStatus> oldStatus = this._fileStatus; try { this._fileStatus = Optional.of(this.fs.getFileStatus(path)); this.exists = this._fileStatus.isPresent(); return (oldStatus.isPresent() != this._fileStatus.isPresent() || oldStatus.get().getModificationTime() != this._fileStatus.get().getModificationTime() || oldStatus.get().isDirectory() != this._fileStatus.get().isDirectory() || oldStatus.get().getLen() != this._fileStatus.get().getLen()); } catch (FileNotFoundException e) { _fileStatus = Optional.absent(); this.exists = false; return true; } } else { if (path.getFileSystem(new Configuration()).exists(path)) { _fileStatus = Optional.of(this.fs.getFileStatus(path)); return true; } else { return false; } } }
/** * Create a Store File Info from an HFileLink and a Reference * @param conf The {@link Configuration} to use * @param fs The current file system to use * @param fileStatus The {@link FileStatus} of the file * @param reference The reference instance * @param link The link instance */ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, final Reference reference, final HFileLink link) { this.fs = fs; this.conf = conf; this.initialPath = fileStatus.getPath(); this.createdTimestamp = fileStatus.getModificationTime(); this.reference = reference; this.link = link; }
@Override public long getLastModified() { try { final FileSystem fs = path.getFileSystem(config); return fs.getFileStatus(path).getModificationTime(); } catch (IOException ex) { throw new HdfsIOException(ex); } }
final FileStatus zipFile = outputFS.getFileStatus(indexZipFilePath); final FileStatus finalIndexZipFile = outputFS.getFileStatus(finalIndexZipFilePath); if (zipFile.getModificationTime() >= finalIndexZipFile.getModificationTime() || zipFile.getLen() != finalIndexZipFile.getLen()) { log.info( "File[%s / %s / %sB] existed, but wasn't the same as [%s / %s / %sB]", finalIndexZipFile.getPath(), DateTimes.utc(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen(), zipFile.getPath(), DateTimes.utc(zipFile.getModificationTime()), zipFile.getLen() ); log.info( "File[%s / %s / %sB] existed and will be kept", finalIndexZipFile.getPath(), DateTimes.utc(finalIndexZipFile.getModificationTime()), finalIndexZipFile.getLen() );
dirStatus = fs.getFileStatus(snapshotDir); } catch (FileNotFoundException e) { if (this.cache.size() > 0) { if (dirStatus.getModificationTime() <= this.lastModifiedTime) return; this.lastModifiedTime = dirStatus.getModificationTime(); String name = snapshot.getPath().getName(); if (files == null || files.hasBeenModified(snapshot.getModificationTime())) { Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath()); files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
private static boolean shouldReplaceFile(FileStatus referencePath, FileStatus replacementFile) { return replacementFile.getLen() != referencePath.getLen() || referencePath.getModificationTime() < replacementFile.getModificationTime(); }
public SyntheticFileId(FileStatus file) { this(file.getPath(), file.getLen(), file.getModificationTime()); }
public static Object getFileId(FileSystem fileSystem, Path path, boolean allowSynthetic, boolean checkDefaultFs, boolean forceSyntheticIds) throws IOException { if (forceSyntheticIds == false && fileSystem instanceof DistributedFileSystem) { DistributedFileSystem dfs = (DistributedFileSystem) fileSystem; if ((!checkDefaultFs) || isDefaultFs(dfs)) { Object result = SHIMS.getFileId(dfs, path.toUri().getPath()); if (result != null) return result; } } if (!allowSynthetic) { LOG.warn("Cannot get unique file ID from " + fileSystem.getClass().getSimpleName() + "; returning null"); return null; } FileStatus fs = fileSystem.getFileStatus(path); return new SyntheticFileId(path, fs.getLen(), fs.getModificationTime()); }
@Override public boolean apply(FileStatus file) { String wal = file.getPath().getName(); boolean logInReplicationQueue = wals.contains(wal); if (logInReplicationQueue) { LOG.debug("Found up in ZooKeeper, NOT deleting={}", wal); } return !logInReplicationQueue && (file.getModificationTime() < readZKTimestamp); } });
/*** * Given a {@link FileStatus} at src FileSystem, decide if it is a {@link CopyableFile}. * * Return true if the {@link Path} of the given {@link FileStatus} does not exist on target {@link FileSystem}, or it * has a newer modification time stamp on source {@link FileSystem} than target {@link FileSystem}. */ private boolean isCopyableFile(FileStatus srcFileStatus, Path targetPath) throws IOException { if (!this.targetFs.exists(targetPath)) { return true; } else if (srcFileStatus.getModificationTime() > this.targetFs.getFileStatus(targetPath).getModificationTime()) { return true; } else { return false; } } }
/** * Assert that a given path exists in a given FileSystem. * * @param fs FileSystem to check. * @param p Path to check. * @throws IOException if the path does not exist. */ private void assertPathExists(FileSystem fs, Path p) throws IOException { FileStatus fileStatus = fs.getFileStatus(p); assertEquals(p, fileStatus.getPath()); assertNotSame(0, fileStatus.getModificationTime()); }
private static boolean isSame(File localCoprocessorFile, FileStatus fileStatus) { return fileStatus.getLen() == localCoprocessorFile.length() && fileStatus.getModificationTime() == localCoprocessorFile.lastModified(); }