@Autowired public void setImportExportThreadPool( @Qualifier("importExportThreadPool") ExecutorService importExportThreadPool) { this.importExportThreadPool = importExportThreadPool; this.directoryScanner = new ConcurrentDirectoryScanner(this.importExportThreadPool); }
protected <T> void scanDirectory( File directory, ConcurrentMap<File, T> results, FileFilter fileFilter, Function<Resource, T> fileProcessor) { final Queue<Tuple<File, Future<T>>> futures = new ConcurrentLinkedQueue<Tuple<File, Future<T>>>(); recurseOnDirectory(futures, results, directory, fileFilter, fileProcessor); waitForFutures(futures, results); }
@Override public void scanDirectoryNoResults( File directory, FileFilter fileFilter, Function<Resource, ?> fileProcessor) { this.scanDirectory(directory, null, fileFilter, fileProcessor); }
protected <T> void recurseOnDirectory( final Queue<Tuple<File, Future<T>>> futures, final ConcurrentMap<File, T> results, final File directory, final FileFilter fileFilter, final Function<Resource, T> fileProcessor) { logger.debug("processing directory: {}", directory); final File[] children = directory.listFiles(fileFilter); for (final File child : children) { // If the child is a directory and recursion is on recurse on it if (child.isDirectory()) { if (recursive) { if (processDirectories) { submitProcessFile(futures, child, fileProcessor); } submitDirectoryRecurse(futures, results, child, fileFilter, fileProcessor); } } // Otherwise pass the file into the fileHandler via the executor service else { submitProcessFile(futures, child, fileProcessor); } } // Clean up any completed futures from the queue cleanFutures(futures, results); }
protected <T> void waitForFutures( final Queue<Tuple<File, Future<T>>> futures, final ConcurrentMap<File, T> results) { while (!futures.isEmpty()) { final Tuple<File, Future<T>> future = futures.poll(); try { processResult(results, future); } catch (InterruptedException e) { Thread.interrupted(); return; } } }
@Override public T call() throws Exception { recurseOnDirectory( futures, results, directory, fileFilter, fileProcessor); return null; } });
@Test public void testDirectoryScanner() { final AntPatternFileFilter fileFilter = new AntPatternFileFilter( true, true, Collections.<String>singleton("**"), Collections.<String>emptySet()); final Map<File, Resource> results = directoryScanner.scanDirectoryWithResults( testDirectory, fileFilter, Functions.<Resource>identity()); assertEquals(7, results.size()); } }
protected <T> void recurseOnDirectory( final Queue<Tuple<File, Future<T>>> futures, final ConcurrentMap<File, T> results, final File directory, final FileFilter fileFilter, final Function<Resource, T> fileProcessor) { logger.debug("processing directory: {}", directory); final File[] children = directory.listFiles(fileFilter); for (final File child : children) { // If the child is a directory and recursion is on recurse on it if (child.isDirectory()) { if (recursive) { if (processDirectories) { submitProcessFile(futures, child, fileProcessor); } submitDirectoryRecurse(futures, results, child, fileFilter, fileProcessor); } } // Otherwise pass the file into the fileHandler via the executor service else { submitProcessFile(futures, child, fileProcessor); } } // Clean up any completed futures from the queue cleanFutures(futures, results); }
protected <T> void waitForFutures( final Queue<Tuple<File, Future<T>>> futures, final ConcurrentMap<File, T> results) { while (!futures.isEmpty()) { final Tuple<File, Future<T>> future = futures.poll(); try { processResult(results, future); } catch (InterruptedException e) { Thread.interrupted(); return; } } }
@Override public T call() throws Exception { recurseOnDirectory( futures, results, directory, fileFilter, fileProcessor); return null; } });
protected <T> void scanDirectory( File directory, ConcurrentMap<File, T> results, FileFilter fileFilter, Function<Resource, T> fileProcessor) { final Queue<Tuple<File, Future<T>>> futures = new ConcurrentLinkedQueue<Tuple<File, Future<T>>>(); recurseOnDirectory(futures, results, directory, fileFilter, fileProcessor); waitForFutures(futures, results); }
@Autowired public void setImportExportThreadPool( @Qualifier("importExportThreadPool") ExecutorService importExportThreadPool) { this.importExportThreadPool = importExportThreadPool; this.directoryScanner = new ConcurrentDirectoryScanner(this.importExportThreadPool); }
@Override public void scanDirectoryNoResults( File directory, FileFilter fileFilter, Function<Resource, ?> fileProcessor) { this.scanDirectory(directory, null, fileFilter, fileProcessor); }
protected <T> void cleanFutures( final Queue<Tuple<File, Future<T>>> futures, final ConcurrentMap<File, T> results) { for (final Iterator<Tuple<File, Future<T>>> futureItr = futures.iterator(); futureItr.hasNext(); ) { final Tuple<File, Future<T>> future = futureItr.next(); if (future.second.isDone()) { futureItr.remove(); try { processResult(results, future); } catch (InterruptedException e) { Thread.interrupted(); return; } } } }
@Before public void setup() throws Exception { testDirectory = File.createTempFile("ConcurrentDirectoryScannerTest_", "_dir"); testDirectory.delete(); testDirectory.mkdirs(); final InputStream dataZipStream = this.getClass().getResourceAsStream("/org/apereo/portal/utils/DirScannerData.zip"); ZipUtils.extract(dataZipStream, testDirectory); IOUtils.closeQuietly(dataZipStream); executorServiceFactory = new ThreadPoolExecutorFactoryBean(); executorServiceFactory.setCorePoolSize(0); executorServiceFactory.setMaxPoolSize(1); executorServiceFactory.setQueueCapacity(500); executorServiceFactory.setKeepAliveSeconds(30); executorServiceFactory.setAllowCoreThreadTimeOut(true); executorServiceFactory.setDaemon(true); executorServiceFactory.afterPropertiesSet(); this.directoryScanner = new ConcurrentDirectoryScanner(executorServiceFactory.getObject()); }
@Override public <T> Map<File, T> scanDirectoryWithResults( File directory, FileFilter fileFilter, Function<Resource, T> fileProcessor) { final ConcurrentMap<File, T> results = new ConcurrentHashMap<File, T>(); this.scanDirectory(directory, results, fileFilter, fileProcessor); return results; }
protected <T> void cleanFutures( final Queue<Tuple<File, Future<T>>> futures, final ConcurrentMap<File, T> results) { for (final Iterator<Tuple<File, Future<T>>> futureItr = futures.iterator(); futureItr.hasNext(); ) { final Tuple<File, Future<T>> future = futureItr.next(); if (future.second.isDone()) { futureItr.remove(); try { processResult(results, future); } catch (InterruptedException e) { Thread.interrupted(); return; } } } }
@Override public <T> Map<File, T> scanDirectoryWithResults( File directory, FileFilter fileFilter, Function<Resource, T> fileProcessor) { final ConcurrentMap<File, T> results = new ConcurrentHashMap<File, T>(); this.scanDirectory(directory, results, fileFilter, fileProcessor); return results; }