@Override public long getModificationTime() { return myFs.getModificationTime(); }
/** list files sorted by modification time that have not been modified since 'olderThan'. if * 'olderThan' is <= 0 then the filtering is disabled */ public static ArrayList<Path> listFilesByModificationTime(FileSystem fs, Path directory, long olderThan) throws IOException { ArrayList<LocatedFileStatus> fstats = new ArrayList<>(); RemoteIterator<LocatedFileStatus> itr = fs.listFiles(directory, false); while (itr.hasNext()) { LocatedFileStatus fileStatus = itr.next(); if (olderThan > 0) { if (fileStatus.getModificationTime() <= olderThan) { fstats.add(fileStatus); } } else { fstats.add(fileStatus); } } Collections.sort(fstats, new ModifTimeComparator()); ArrayList<Path> result = new ArrayList<>(fstats.size()); for (LocatedFileStatus fstat : fstats) { result.add(fstat.getPath()); } return result; }
@Override public void killOlderThan(long timestamp) throws IOException { Path taskLogDir = new Path(config.getDirectory()); FileSystem fs = taskLogDir.getFileSystem(hadoopConfig); if (fs.exists(taskLogDir)) { if (!fs.isDirectory(taskLogDir)) { throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir); } RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(taskLogDir); while (iter.hasNext()) { LocatedFileStatus file = iter.next(); if (file.getModificationTime() < timestamp) { Path p = file.getPath(); log.info("Deleting hdfs task log [%s].", p.toUri().toString()); fs.delete(p, true); } if (Thread.currentThread().isInterrupted()) { throw new IOException( new InterruptedException("Thread interrupted. Couldn't delete all tasklogs.") ); } } } } }
timestampedSegments, dataSegment, locatedFileStatus.getModificationTime() ); } else {
if (filter.matches(resPath, status.getModificationTime())) { RawResource raw; if (loadContent) raw = new RawResource(resPath, status.getModificationTime(), fs.open(status.getPath())); else raw = new RawResource(resPath, status.getModificationTime());
@Override public long getModificationTime() { return myFs.getModificationTime(); }
@Override public long getModificationTime() { return myFs.getModificationTime(); }
@Override public long getModificationTime() { return myFs.getModificationTime(); }
@Override public long getModificationTime() { return myFs.getModificationTime(); }
private void sweep(String spillDir, long targetTime) { try { final Path spillDirPath = new Path(spillDir); FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); RemoteIterator<LocatedFileStatus> files = fileSystem.listLocatedStatus(spillDirPath); while (files.hasNext()) { LocatedFileStatus st = files.next(); if (st.getModificationTime() <= targetTime) { fileSystem.delete(st.getPath(), true); } } } catch (IOException e) { // exception silently ignored. Directory will be revisited at the next sweep } } }
/** list files sorted by modification time that have not been modified since 'olderThan'. if * 'olderThan' is <= 0 then the filtering is disabled */ public static ArrayList<Path> listFilesByModificationTime(FileSystem fs, Path directory, long olderThan) throws IOException { ArrayList<LocatedFileStatus> fstats = new ArrayList<>(); RemoteIterator<LocatedFileStatus> itr = fs.listFiles(directory, false); while( itr.hasNext() ) { LocatedFileStatus fileStatus = itr.next(); if(olderThan>0) { if( fileStatus.getModificationTime()<=olderThan ) fstats.add(fileStatus); } else { fstats.add(fileStatus); } } Collections.sort(fstats, new ModifTimeComparator() ); ArrayList<Path> result = new ArrayList<>(fstats.size()); for (LocatedFileStatus fstat : fstats) { result.add(fstat.getPath()); } return result; }
while (files.hasNext()) { LocatedFileStatus file = files.next(); Path filePath = file.getPath(); // Get only the files with created date = current date if (DateUtils.truncate(new Date(file.getModificationTime()), java.util.Calendar.DAY_OF_MONTH).equals(dateNow)) { if (pattern.matcher(filePath.getName()).matches()) { fileInfo.put(file.getPath().getName(), file.getLen()); } } }
if (filter.matches(resPath, status.getModificationTime())) { RawResource raw; if (loadContent) raw = new RawResource(resPath, status.getModificationTime(), fs.open(status.getPath())); else raw = new RawResource(resPath, status.getModificationTime());
ConverterUtils.getYarnUrlFromPath(lfs.getPath()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, lfs.getLen(), lfs.getModificationTime()); localResources.put(lfs.getPath().getName(), localRsrc);
/** * Recursively import every path under path. * @return number of items inserted into MetadataStore * @throws IOException on I/O errors. */ private long importDir(FileStatus status) throws IOException { Preconditions.checkArgument(status.isDirectory()); RemoteIterator<LocatedFileStatus> it = getFilesystem() .listFilesAndEmptyDirectories(status.getPath(), true); long items = 0; while (it.hasNext()) { LocatedFileStatus located = it.next(); FileStatus child; if (located.isDirectory()) { child = DynamoDBMetadataStore.makeDirStatus(located.getPath(), located.getOwner()); dirCache.add(child.getPath()); } else { child = new S3AFileStatus(located.getLen(), located.getModificationTime(), located.getPath(), located.getBlockSize(), located.getOwner()); } putParentsIfNotPresent(child); getStore().put(new PathMetadata(child)); items++; } return items; }
private static void checkEquals(RemoteIterator<LocatedFileStatus> i1, RemoteIterator<LocatedFileStatus> i2) throws IOException { while (i1.hasNext()) { assertTrue(i2.hasNext()); // Compare all the fields but the path name, which is relative // to the original path from listFiles. LocatedFileStatus l1 = i1.next(); LocatedFileStatus l2 = i2.next(); assertEquals(l1.getAccessTime(), l2.getAccessTime()); assertEquals(l1.getBlockSize(), l2.getBlockSize()); assertEquals(l1.getGroup(), l2.getGroup()); assertEquals(l1.getLen(), l2.getLen()); assertEquals(l1.getModificationTime(), l2.getModificationTime()); assertEquals(l1.getOwner(), l2.getOwner()); assertEquals(l1.getPermission(), l2.getPermission()); assertEquals(l1.getReplication(), l2.getReplication()); } assertFalse(i2.hasNext()); }