@Override public FileStatus[] globStatus(Path pathPattern) throws IOException { FileStatus[] statuses = super.globStatus(pathPattern); if (statuses == null) { acquirePermit("globStatus " + pathPattern); } else { acquirePermits(statuses.length / LISTING_FILES_PER_PERMIT + 1, "globStatus " + pathPattern); } return statuses; }
@Override public FileStatus[] listStatus(Path path) throws IOException { FileStatus[] statuses = super.listStatus(path); if (statuses == null) { acquirePermit("listStatus " + path); } else { acquirePermits(statuses.length / LISTING_FILES_PER_PERMIT + 1, "listStatus " + path); } return statuses; }
/** * Initialize the observer. * @throws IOException if an error occurs */ public void initialize() throws IOException { rootEntry.refresh(rootEntry.getPath()); final FileStatusEntry[] children = doListPathsEntry(rootEntry.getPath(), rootEntry); rootEntry.setChildren(children); }
/** * Add {@link PathAlterationObserverScheduler}s for the given * root directory and any nested subdirectories under the root directory to the given * {@link PathAlterationObserverScheduler}. * * @param monitor a {@link PathAlterationObserverScheduler} * @param listener a {@link org.apache.gobblin.util.filesystem.PathAlterationListener} * @param rootDirPath root directory */ public static void addPathAlterationObserver(PathAlterationObserverScheduler monitor, PathAlterationListener listener, Path rootDirPath) throws IOException { PathAlterationObserver observer = new PathAlterationObserver(rootDirPath); observer.addListener(listener); monitor.addObserver(observer); }
@Override public FileStatus[] listStatus(Path path, PathFilter filter) throws IOException { CountingPathFilterDecorator decoratedFilter = new CountingPathFilterDecorator(filter); FileStatus[] statuses = super.listStatus(path, decoratedFilter); if (statuses == null) { acquirePermit("listStatus " + path); } else { acquirePermits(decoratedFilter.getPathsProcessed().get() / LISTING_FILES_PER_PERMIT + 1, "listStatus " + path); } return statuses; }
@Override public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { this.acquirePermit("create " + path); return super.create(path, permission, overwrite, bufferSize, replication, blockSize, progress); }
@Override public boolean mkdirs(Path path, FsPermission permission) throws IOException { this.acquirePermit("mkdirs " + path); return super.mkdirs(path, permission); }
@Override public boolean delete(Path path, boolean recursive) throws IOException { this.acquirePermit("delete " + path); return super.delete(path, recursive); }
@Override public FileStatus getFileStatus(Path path) throws IOException { this.acquirePermit("getFileStatus " + path); return super.getFileStatus(path); }
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { this.acquirePermit("append " + path); return super.append(path, bufferSize, progress); }
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { this.acquirePermit("open " + path); return super.open(path, bufferSize); }
@Override public void close() throws IOException { getRateLimiter().stop(); super.close(); }
/** * Stop monitoring * * @throws Exception if an error occurs initializing the observer */ public synchronized void stop() throws IOException, InterruptedException { stop(interval); }
@Override public boolean exists(Path path) throws IOException { this.acquirePermit("exists " + path); return super.exists(path); }
@Override public void close() throws IOException { super.close(); // Should print out statistics here }
@Override public void run() { if (!FileSystemInstrumentation.this.closed) { onClose(); } } });
/** * Construct an observer for the specified directory and file filter. * * @param directory the directory to observe * @param pathFilter The file filter or null if none */ public PathAlterationObserver(final Path directory, final PathFilter pathFilter) throws IOException { this(new FileStatusEntry(directory), pathFilter); }
/** * Return the level * * @return the level */ public int getLevel() { return parent == null ? 0 : parent.getLevel() + 1; }
@Override public boolean delete(Path path) throws IOException { return this.delete(path, true); }