public DateExtractor getDateExtractor() { if (dateFormatMark != null) return new DateExtractorFromName(dateFormatMark, false); else { CollectionSpecParser sp = getCollectionSpecParser(null); if (sp.getDateFormatMark() != null) return new DateExtractorFromName(sp.getDateFormatMark(), true); } return new DateExtractorNone(); }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
private MFileCollectionManager(String collectionName, String collectionSpec, String olderThan, Formatter errlog) { super(collectionName, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(2); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
private MFileCollectionManager(String collectionSpec, String olderThan, Formatter errlog) { super(collectionSpec, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(2); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.name != null ? config.name : config.spec, logger); this.config = config; CollectionSpecParser sp = new CollectionSpecParser(config.spec, errlog); this.root = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); if (config.protoConfig != null) protoChoice = config.protoConfig.choice; if (config.updateConfig != null) { this.recheck = makeRecheck(config.updateConfig.recheckAfter); // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); } if (this.auxInfo == null) this.auxInfo = new HashMap<>(10); this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config); }
public MFileCollectionManager(String name, String spec, Formatter errlog, org.slf4j.Logger logger) { super(name, logger); CollectionSpecParser sp = new CollectionSpecParser(spec, errlog); this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.olderThanInMsecs = -1; }
private MFileCollectionManager(String collectionName, String collectionSpec, String olderThan, Formatter errlog) { super(collectionName, null); CollectionSpecParser sp = new CollectionSpecParser(collectionSpec, errlog); this.recheck = null; this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(olderThan); dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.name != null ? config.name : config.spec, logger); this.config = config; CollectionSpecParser sp = new CollectionSpecParser(config.spec, errlog); this.rootDir = sp.getRootDir(); List<MFileFilter> filters = new ArrayList<MFileFilter>(3); if (null != sp.getFilter()) filters.add(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new MCollection(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); this.recheck = makeRecheck(config.updateConfig.recheckAfter); protoChoice = config.protoConfig.choice; // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); }
@Test public void testCalendarDateFromPath() throws IOException { Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser("[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America/FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm#.grib1", errlog); System.out.printf("errlog=%s%n", errlog); System.out.printf("specp=%s%n", specp); // test parsing assert specp.getRootDir().equals("[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America"); assert !specp.wantSubdirs(); assert specp.getFilter().toString().equals("FNMOC_COAMPS_Equatorial_America_..............grib1"); assert specp.getDateFormatMark().equals("FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm"); // test filter String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); // test date extractor DateExtractor extractor = new DateExtractorFromName(specp.getDateFormatMark(), true); CalendarDate cd = extractor.getCalendarDateFromPath(mfile.toString()); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.collectionName != null ? config.collectionName : config.spec, logger); this.config = config; CollectionSpecParser sp = config.getCollectionSpecParser(errlog); this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); if (config.protoConfig != null) protoChoice = config.protoConfig.choice; if (config.updateConfig != null) { this.recheck = makeRecheck(config.updateConfig.recheckAfter); // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); } if (this.auxInfo == null) this.auxInfo = new HashMap<>(10); this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config); }
@Test public void testParseCalendarDate() throws IOException { Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser("/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km/GFS_CONUS_95km_#yyyyMMdd_HHmm#.grib2", errlog); System.out.printf("errlog=%s%n", errlog); System.out.printf("specp=%s%n", specp); // test parsing assert specp.getRootDir().equals("/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km"); assert !specp.wantSubdirs(); assert specp.getFilter().toString().equals("GFS_CONUS_95km_..............grib2"); assert specp.getDateFormatMark().equals("GFS_CONUS_95km_#yyyyMMdd_HHmm"); // test filter MFileFilter mfilter = new WildcardMatchOnName(specp.getFilter()); String path = "/data/ldm/pub/native/grid/NCEP/GFS/CONUS_95km/GFS_CONUS_95km_20141203_0000.grib2"; MFile mfile = new MFileOS7(Paths.get(path), null); assert mfilter.accept(mfile); // 2014-12-03T16:43:59.433 -0700 ERROR - ucar.nc2.units.DateFromString - Must delineate Date between 2 '#' chars, dateFormatString = GFS_CONUS_95km_#yyyyMMdd_HHmm // test date extractor DateExtractor extractor = new DateExtractorFromName(specp.getDateFormatMark(), true); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-03T00:00:00Z")); }