private MFile nextFilteredFile() { if (orgIter == null) return null; if (!orgIter.hasNext()) return null; MFile pdata = orgIter.next(); while ((pdata.isDirectory() != wantDirs) || !mc.accept(pdata)) { // skip directories, and filter if (!orgIter.hasNext()) return null; /// 6 pdata = orgIter.next(); } return pdata; } }
public MFileCollectionManager(String name, CollectionConfig mc, CalendarDate startPartition, org.slf4j.Logger logger) { super(name, logger); this.startCollection = startPartition; this.scanList.add(mc); this.root = mc.getDirectoryName(); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
@Override public String toString() { Formatter f = new Formatter(); f.format("DatasetCollectionManager{ collectionName='%s' recheck=%s ", collectionName, recheck); for (CollectionConfig mc : scanList) { f.format("%n dir=%s filter=%s", mc.getDirectoryName(), mc.getFileFilter()); } return f.toString(); }
CollectionConfig mc = new CollectionConfig(dirName, dirName, wantSubdirs, filter, auxInfo);
protected void reallyScan(java.util.Map<String, MFile> map) throws IOException { getController(); // make sure a controller is instantiated // run through all scanners and collect MFile instances into the Map int count = 0; for (CollectionConfig mc : scanList) { long start = System.currentTimeMillis(); // System.out.printf("MFileCollectionManager reallyScan %s %s%n", mc.getDirectoryName(), CalendarDate.present()); // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); continue; } while (iter.hasNext()) { MFile mfile = iter.next(); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++; } long took = (System.currentTimeMillis() - start) / 1000; // System.out.printf("MFileCollectionManager reallyScan %s took %d secs%n", collectionName, took); if (logger.isDebugEnabled()) { long took2 = (System.currentTimeMillis() - start) / 1000; logger.debug("{} : was scanned nfiles= {} took={} secs", collectionName, count, took2); } } if (map.size() == 0) { logger.warn("MFileCollectionManager: No files found for {}", collectionName); } }
public CollectionConfig subdir(MFile child) { return new CollectionConfig( name+"/"+child.getName(), dirName+"/"+child.getName(), wantSubdirs, ff, child.getAuxInfo()); }
Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++;
public MFileCollectionManager(String name, CollectionConfig mc, CalendarDate startPartition, org.slf4j.Logger logger) { super(name, logger); this.startCollection = startPartition; this.scanList.add(mc); this.root = mc.getDirectoryName(); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
public CollectionConfig subdir(MFile child) { return new CollectionConfig( name+"/"+child.getName(), dirName+"/"+child.getName(), wantSubdirs, ff, child.getAuxInfo()); }
@Override public String toString() { Formatter f = new Formatter(); f.format("DatasetCollectionManager{ collectionName='%s' recheck=%s ", collectionName, recheck); for (CollectionConfig mc : scanList) { f.format("%n dir=%s filter=%s", mc.getDirectoryName(), mc.getFileFilter()); } return f.toString(); }
private MFile nextFilteredFile() { if (orgIter == null) return null; if (!orgIter.hasNext()) return null; MFile pdata = orgIter.next(); while ((pdata.isDirectory() != wantDirs) || !mc.accept(pdata)) { // skip directories, and filter if (!orgIter.hasNext()) return null; /// 6 pdata = orgIter.next(); } return pdata; } }
@Override public Iterator<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); } File cd = new File(path); if (!cd.exists()) return null; if (!cd.isDirectory()) return null; return new FilteredIterator(mc, new MFileIterator(cd), false); // removes subdirs }
wantSubdirs = false; CollectionConfig mc = new CollectionConfig(dirName, dirName, wantSubdirs, filters, auxInfo);
@Override public Iterator<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); } File cd = new File(path); if (!cd.exists()) return null; if (!cd.isDirectory()) return null; return new FilteredIterator(mc, new MFileIteratorAll(cd), false); }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
@Override public Iterator<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); } File cd = new File(path); if (!cd.exists()) return null; if (!cd.isDirectory()) return null; return new FilteredIterator(mc, new MFileIteratorAll(cd), false); }
private MFileCollectionManager(String collectionName, String collectionSpec, String olderThan, Formatter errlog) { super(collectionName, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(2); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
public Iterator<MFile> getSubdirs(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); } File cd = new File(path); if (!cd.exists()) return null; if (!cd.isDirectory()) return null; return new FilteredIterator(mc, new MFileIterator(cd), true); // return only subdirs }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.name != null ? config.name : config.spec, logger); this.config = config; CollectionSpecParser sp = new CollectionSpecParser(config.spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); if (config.protoConfig != null) protoChoice = config.protoConfig.choice; if (config.updateConfig != null) { this.recheck = makeRecheck(config.updateConfig.recheckAfter); // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); } if (this.auxInfo == null) this.auxInfo = new HashMap<>(10); this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config); }
@Override public Iterator<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); } File cd = new File(path); if (!cd.exists()) return null; if (!cd.isDirectory()) return null; return new FilteredIterator(mc, new MFileIterator(cd), false); // removes subdirs }