protected void reallyScan(java.util.Map<String, MFile> map) throws IOException { getController(); // make sure a controller is instantiated // run through all scanners and collect MFile instances into the Map for (MCollection mc : scanList) { // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); continue; } int count = 0; while (iter.hasNext()) { MFile mfile = iter.next(); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++; } logger.debug("{} : was scanned nfiles= {} ", collectionName, count); } if (map.size() == 0) { logger.warn("MFileCollectionManager: No files found for {}", collectionName); } }
@Override public List<CrawlableDataset> listDatasets(CrawlableDatasetFilter filter) throws IOException { MCollection mc = new MCollection(mfile.getPath(), mfile.getPath(), false, (MFileFilter) null, null); Iterator<MFile> iter = controller.getInventoryTop(mc, true); List<CrawlableDataset> result = new ArrayList<CrawlableDataset>(); while (iter.hasNext()) { CrawlableMFile crf = new CrawlableMFile(controller, iter.next()); if (filter == null || filter.accept(crf)) result.add(crf); } return result; }
Iterator<MFile> iter = controller.getSubdirs(mc, true); if (iter == null) { logger.error("TimePartitionCollections Invalid collection, no subdirectories found; root = {}, collection= {} ", sp.getRootDir(), mc);
protected void reallyScan(java.util.Map<String, MFile> map) throws IOException { getController(); // make sure a controller is instantiated // run through all scanners and collect MFile instances into the Map int count = 0; for (CollectionConfig mc : scanList) { long start = System.currentTimeMillis(); // System.out.printf("MFileCollectionManager reallyScan %s %s%n", mc.getDirectoryName(), CalendarDate.present()); // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); continue; } while (iter.hasNext()) { MFile mfile = iter.next(); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++; } long took = (System.currentTimeMillis() - start) / 1000; // System.out.printf("MFileCollectionManager reallyScan %s took %d secs%n", collectionName, took); if (logger.isDebugEnabled()) { long took2 = (System.currentTimeMillis() - start) / 1000; logger.debug("{} : was scanned nfiles= {} took={} secs", collectionName, count, took2); } } if (map.size() == 0) { logger.warn("MFileCollectionManager: No files found for {}", collectionName); } }
/** * Do not use * @throws IOException */ public void scanDebug(Formatter f) throws IOException { getController(); // make sure a controller is instantiated // run through all scanners and collect MFile instances into the Map for (MCollection mc : scanList) { // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); continue; } int count = 0; while (iter.hasNext()) { MFile mfile = iter.next(); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++; } logger.debug("{} : was scanned nfiles= {} ", collectionName, count); } }
Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc);