public static void main(String[] args) throws IOException { FeatureCollectionConfig config = new FeatureCollectionConfig(); config.spec = "G:/nomads/cfsr/timeseries/**/.*grb2$"; config.dateFormatMark = "#timeseries/#yyyyMM"; config.timePartition = "true"; //doit(config); config = new FeatureCollectionConfig(); config.spec = "G:/mlode/gefs/.*grib2$"; config.dateFormatMark = "#GEFS_Global_1p0deg_Ensemble_#yyyyMMdd_HHmm"; config.timePartition = "day"; // doit(config); config = new FeatureCollectionConfig(); config.spec = "g:/mlode/SREF_CONUS_40km/.*grib2$"; config.dateFormatMark = "yyyyMMdd_HHmm#.grib2#"; config.timePartition = "day"; //doit(config); config = new FeatureCollectionConfig(); config.spec = "g:/mlode/radar/**/.*grib2$"; config.dateFormatMark = "#radar/#yyyyMMdd"; config.timePartition = "directory"; doit(config); } }
@Test public void testFeatureCollectionConfig() throws IOException { FeatureCollectionConfig config = new FeatureCollectionConfig("fnmoc", "test/fnmoc", FeatureCollectionType.GRIB1, "[DATA_DIR]/native/grid/FNMOC/COAMPS/Equatorial_America/FNMOC_COAMPS_Equatorial_America_#yyyyMMdd_HHmm#.grib1", null, null, null, "file", null); DateExtractor extractor = config.getDateExtractor(); String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }
/** * This is only used for the top level GribCollection. * * @param config use this FeatureCollectionConfig * @return index File */ static private File makeTopIndexFileFromConfig(FeatureCollectionConfig config) { Formatter errlog = new Formatter(); CollectionSpecParser specp = config.getCollectionSpecParser(errlog); String name = StringUtil2.replace(config.collectionName, '\\', "/"); // String cname = DirectoryCollection.makeCollectionName(name, Paths.get(specp.getRootDir())); return makeIndexFile(name, new File(specp.getRootDir())); }
FeatureCollectionConfig config = new FeatureCollectionConfig(name, path, fcType, spec, collectionName, dateFormatMark, olderThan, timePartition, innerNcml); config.setFilter(rootDir, regExp); config.setFilesSort( featureCollectionElement.getChild("filesSort", Catalog.defNS)); config.finish();
FeatureCollectionConfig config = new FeatureCollectionConfig(name, path, fcType, spec, collectionName, dateFormatMark, olderThan, timePartition, innerNcml); config.setFilter(rootDir, regExp); config.finish();
@Ignore("B: not visible on spock") @Test public void testNcdcNarr() throws IOException { Grib.setDebugFlags(new DebugFlagsImpl("Grib/debugGbxIndexOnly")); FeatureCollectionConfig config = new FeatureCollectionConfig("NcdcNarr", "NcdcNarr", FeatureCollectionType.GRIB1, "B:/ncdc/0402/home/tomcat/dans-tdm-content/content/tdm/cache/GribIndex/global/nomads/nexus/narr/**/.*gbx9", null, null, null, null, null); config.setFilter("B:/ncdc/0402/home/tomcat/dans-tdm-content/content/tdm/cache/GribIndex/global/nomads/nexus/narr/", "\\d{6}/\\d{8}/.*gbx9$"); boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger); System.out.printf("changed = %s%n", changed); Grib.setDebugFlags(new DebugFlagsImpl()); }
FeatureCollectionConfig config = new FeatureCollectionConfig(collName, path, fcType, spec, dateFormatMark, olderThan, timePartition, useIndexOnly, innerNcml); config.finish();
private DateExtractor getDateExtractor() { if (extractor == null) extractor = config.getDateExtractor(); return extractor; }
public CollectionPathMatcher(FeatureCollectionConfig config, CollectionSpecParser specp, Logger logger) { super(config.collectionName, logger); this.config = config; this.wantSubdirs = specp.wantSubdirs(); setRoot(specp.getRootDir()); DateExtractor extract = config.getDateExtractor(); if (extract != null && !(extract instanceof DateExtractorNone)) setDateExtractor(extract); putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); matcher = specp.getPathMatcher(); // LOOK still need to decide what you are matching on name, path, etc this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString( config.olderThan); }
private Fmrc(String collectionSpec, Formatter errlog) throws IOException { this.manager = MFileCollectionManager.open(collectionSpec, collectionSpec, null, errlog); // LOOK no name this.config = new FeatureCollectionConfig(); this.config.spec = collectionSpec; }
@Test @Category(NeedsCdmUnitTest.class) public void testFeatureCollectionReader() throws IOException, JDOMException { File configFile = new File(TestDir.cdmUnitTestDir+"tds/config/fnmoc.xml"); org.jdom2.Document doc; SAXBuilder builder = new SAXBuilder(); doc = builder.build(configFile); XMLOutputter xmlOut = new XMLOutputter(); System.out.println(xmlOut.outputString(doc)); Formatter errlog = new Formatter(); FeatureCollectionConfigBuilder fcb = new FeatureCollectionConfigBuilder(errlog); FeatureCollectionConfig config = fcb.readConfig(doc.getRootElement()); DateExtractor extractor = config.getDateExtractor(); String path = "[DATA_DIR]/native/grid/FNMOC/COAMPS/FNMOC_COAMPS_Equatorial_America_20141207_1200.grib1.ncx3"; MFile mfile = new MFileOS7(Paths.get(path), null); CalendarDate cd = extractor.getCalendarDate(mfile); assert cd != null : "date extractor failed"; System.out.printf("%s -> %s%n", path, cd); assert(cd.toString().equals("2014-12-07T12:00:00Z")); }
public DateExtractor getDateExtractor() { if (dateFormatMark != null) return new DateExtractorFromName(dateFormatMark, false); else { CollectionSpecParser sp = getCollectionSpecParser(null); if (sp.getDateFormatMark() != null) return new DateExtractorFromName(sp.getDateFormatMark(), true); } return new DateExtractorNone(); }
private Fmrc(String collectionSpec, Formatter errlog) throws IOException { this.manager = MFileCollectionManager.open(collectionSpec, null, errlog); this.config = new FeatureCollectionConfig(); this.config.spec = collectionSpec; }
CollectionSpecParser specp = config.getCollectionSpecParser(errlog); Path rootPath = Paths.get(specp.getRootDir()); boolean isGrib1 = config.type == FeatureCollectionType.GRIB1;
private Fmrc(String collectionSpec, Formatter errlog) throws IOException { this.manager = MFileCollectionManager.open(collectionSpec, collectionSpec, null, errlog); // LOOK no name this.config = new FeatureCollectionConfig(); this.config.spec = collectionSpec; }
long start = System.currentTimeMillis(); final Formatter errlog = new Formatter(); CollectionSpecParser specp = config.getCollectionSpecParser(errlog);
public static void main2(String[] args) throws IOException { org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger("test"); PartitionManager partition = new PartitionManagerFromIndexDirectory("NCDC-gfs4_all", new FeatureCollectionConfig(), new File("B:/ncdc/gfs4_all/"), NCX_SUFFIX, logger); Grib1PartitionBuilder builder = new Grib1PartitionBuilder("NCDC-gfs4_all", new File(partition.getRoot()), partition, logger); builder.createPartitionedIndex(CollectionUpdateType.nocheck, new Formatter()); }
/** * Update all the gbx indices in one directory, and the ncx index for that directory * * @param config FeatureCollectionConfig * @param dirPath directory path * @return true if collection was rewritten, exception on failure * @throws IOException */ static private boolean updateLeafCollection(boolean isGrib1, FeatureCollectionConfig config, CollectionUpdateType updateType, boolean isTop, Logger logger, Path dirPath) throws IOException { if (config.ptype == FeatureCollectionConfig.PartitionType.file) { return updateFilePartition(isGrib1, config, updateType, isTop, logger, dirPath); } else { Formatter errlog = new Formatter(); CollectionSpecParser specp = config.getCollectionSpecParser(errlog); try (DirectoryCollection dcm = new DirectoryCollection(config.collectionName, dirPath, isTop, config.olderThan, logger)) { dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName())); boolean changed = updateGribCollection(isGrib1, dcm, updateType, FeatureCollectionConfig.PartitionType.directory, logger, errlog); if (debug) System.out.printf(" GribCdmIndex.updateDirectoryPartition was updated=%s on %s%n", changed, dirPath); return changed; } } }
public static void main3(String[] args) throws IOException { org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger("test"); /* <featureCollection name="DGEX-CONUS_12km" featureType="GRIB" harvest="true" path="grib/NCEP/DGEX/CONUS_12km"> <collection spec="F:/data/grib/idd/dgex/ /.*grib2$" dateFormatMark="#DGEX_CONUS_12km_#yyyyMMdd_HHmm" timePartition="directory" olderThan="5 min"/> */ // String name, String path, FeatureCollectionType fcType, String spec, String dateFormatMark, String olderThan, String timePartition, String useIndexOnlyS, Element innerNcml //FeatureCollectionConfig config = new FeatureCollectionConfig("DGEX-test", "grib/NCEP/DGEX/CONUS_12km", FeatureCollectionType.GRIB2, // "Q:/cdmUnitTest/gribCollections/dgex/**/.*grib2$", "#DGEX_CONUS_12km_#yyyyMMdd_HHmm", null, "directory", null, null); // FeatureCollectionConfig config = new FeatureCollectionConfig("GFS_CONUS_80km", "grib/NCEP/GFS/CONUS_80km", FeatureCollectionType.GRIB1, // "Q:/cdmUnitTest/ncss/GFS/CONUS_80km/GFS_CONUS_80km_#yyyyMMdd_HHmm#.grib1", null, null, "file", null, null); //FeatureCollectionConfig config = new FeatureCollectionConfig("ds083.2_Aggregation", "ds083.2/Aggregation", FeatureCollectionType.GRIB1, // "Q:/cdmUnitTest/gribCollections/rdavm/ds083.2/grib1/**/.*grib1", "#fnl_#yyyyMMdd_HH_mm", null, "directory", null, null); /* <pdsHash> <useTableVersion>false</useTableVersion> </pdsHash> */ FeatureCollectionConfig config = new FeatureCollectionConfig("RFC", "grib/NPVU/RFC", FeatureCollectionType.GRIB1, "B:/motherlode/rfc/**/.*grib1$", null, "yyyyMMdd#.grib1#", null, "directory", null); // config.gribConfig.pdsHash.put("useTableVersion", false); // boolean isGrib1, MCollection dcm, CollectionUpdateType updateType, Formatter errlog, org.slf4j.Logger logger boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.test, logger); System.out.printf("changed = %s%n", changed); }
public MFileCollectionManager(FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) { super(config.collectionName != null ? config.collectionName : config.spec, logger); this.config = config; CollectionSpecParser sp = config.getCollectionSpecParser(errlog); this.root = sp.getRootDir(); CompositeMFileFilter filters = new CompositeMFileFilter(); if (null != sp.getFilter()) filters.addIncludeFilter(new WildcardMatchOnName(sp.getFilter())); olderThanInMsecs = parseOlderThanFilter(config.olderThan); if (config.dateFormatMark != null) dateExtractor = new DateExtractorFromName(config.dateFormatMark, false); else if (sp.getDateFormatMark() != null) dateExtractor = new DateExtractorFromName(sp.getDateFormatMark(), true); else dateExtractor = new DateExtractorNone(); scanList.add(new CollectionConfig(sp.getRootDir(), sp.getRootDir(), sp.wantSubdirs(), filters, null)); if (config.protoConfig != null) protoChoice = config.protoConfig.choice; if (config.updateConfig != null) { this.recheck = makeRecheck(config.updateConfig.recheckAfter); // static means never rescan on checkState; let it be externally triggered. if ((config.updateConfig.recheckAfter == null) && (config.updateConfig.rescan == null) && (config.updateConfig.deleteAfter == null)) setStatic(true); } if (this.auxInfo == null) this.auxInfo = new HashMap<>(10); this.auxInfo.put(FeatureCollectionConfig.AUX_CONFIG, config); }