public Partition(MCollection dcm) { FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); if (config == null) logger.warn("Partition missing FeatureCollectionConfig {}", dcm); this.dcm = dcm; this.name = dcm.getCollectionName(); this.lastModified = dcm.getLastModified(); this.directory = StringUtil2.replace(dcm.getRoot(), '\\', "/"); this.partitionDate = dcm.getPartitionDate(); String indexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/"); if (partitionDate == null) { partitionDate = getDateExtractor().getCalendarDateFromPath(indexFilename); // LOOK dicey } // now remove the directory if (indexFilename.startsWith(directory)) { indexFilename = indexFilename.substring(directory.length()); if (indexFilename.startsWith("/")) indexFilename = indexFilename.substring(1); } filename = indexFilename; }
public void consume(MFile mfile) { MCollection part = new CollectionSingleFile(mfile, logger); part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); try { boolean changed = updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.file, logger, errlog); if (changed) anyChange.set(true); } catch (IllegalStateException t) { logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage()); partition.removePartition(part); // keep on truckin; can happen if directory is empty } catch (Throwable t) { logger.error("Error making partition " + part.getRoot(), t); partition.removePartition(part); } } });
@Override public void close() { org.close(); }
public PartitionManagerFromIndexList(MCollection dcm, List<MFile> partFiles, org.slf4j.Logger logger) { super(dcm.getCollectionName(), logger); this.config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); this.root = dcm.getRoot(); this.partIndexFiles = partFiles; }
public int compare(MCollection o1, MCollection o2) { return o1.getCollectionName().compareTo(o2.getCollectionName()); } });
static public GribCollectionImmutable openGribCollectionFromDataFile(boolean isGrib1, MFile mfile, CollectionUpdateType updateType, FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) throws IOException { MCollection dcm = new CollectionSingleFile(mfile, logger); dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (isGrib1) { Grib1CollectionBuilder builder = new Grib1CollectionBuilder(dcm.getCollectionName(), dcm, logger); // LOOK ignoring partition type boolean changed = (builder.updateNeeded(updateType) && builder.createIndex(FeatureCollectionConfig.PartitionType.file, errlog)); } else { Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm.getCollectionName(), dcm, logger); boolean changed = (builder.updateNeeded(updateType) && builder.createIndex(FeatureCollectionConfig.PartitionType.file, errlog)); } // the index file should now exist, open it GribCollectionImmutable result = openCdmIndex(dcm.getIndexFilename(NCX_SUFFIX), config, true, logger); if (result != null) return result; // if open fails, force recreate the index if (updateType == CollectionUpdateType.never) return null; // not allowed to write if (updateType == CollectionUpdateType.always) return null;// already tried to force write, give up return openGribCollectionFromDataFile(isGrib1, mfile, CollectionUpdateType.always, config, errlog, logger); }
for (MFile f : manager.getFilesSorted()) { if (logger.isDebugEnabled()) logger.debug("Fmrc: "+config.spec+": file="+f.getPath()); return new FmrcInv("fmrc:"+manager.getCollectionName(), fmrList, config.fmrcConfig.regularize);
public GribCollectionBuilder(boolean isGrib1, String name, MCollection dcm, org.slf4j.Logger logger) { this.dcm = dcm; this.logger = logger; this.isGrib1 = isGrib1; this.name = StringUtil2.replace(name, ' ', "_"); this.directory = new File(dcm.getRoot()); }
protected void reallyScan(java.util.Map<String, MFile> map) throws IOException { getController(); // make sure a controller is instantiated // run through all scanners and collect MFile instances into the Map for (MCollection mc : scanList) { // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); continue; } int count = 0; while (iter.hasNext()) { MFile mfile = iter.next(); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++; } logger.debug("{} : was scanned nfiles= {} ", collectionName, count); } if (map.size() == 0) { logger.warn("MFileCollectionManager: No files found for {}", collectionName); } }
public Grib2CollectionBuilder(String name, MCollection dcm, org.slf4j.Logger logger) { super(false, name, dcm, logger); FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); gribConfig = config.gribConfig; }
dcmp.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, partitionManager.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG)); result.addPartition(dcmp); logger.debug(" Using canonical partition {}", canon.getDcm().getCollectionName());
@Override public Iterable<MCollection> makePartitions(CollectionUpdateType forceCollection) throws IOException { if (forceCollection == null) forceCollection = CollectionUpdateType.test; DirectoryBuilder builder = new DirectoryBuilder(topCollection, collectionDir, null, suffix); builder.constructChildren(indexReader, forceCollection); List<MCollection> result = new ArrayList<>(); for (DirectoryBuilder child : builder.getChildren()) { MCollection dc = null; try { dc = DirectoryBuilder.factory(config, child.getDir(), false, indexReader, suffix, logger); // DirectoryPartitions or DirectoryCollections if (!wasRemoved( dc)) result.add(dc); lastModified = Math.max(lastModified, dc.getLastModified()); } catch (Throwable ioe) { logger.warn("DirectoryBuilder on "+child.getDir()+" failed: skipping", ioe); if (dc != null) dc.close(); } } // sort collection by name Collections.sort(result, new Comparator<MCollection>() { public int compare(MCollection o1, MCollection o2) { return o1.getCollectionName().compareTo(o2.getCollectionName()); } }); return result; }
@Override public MCollection next() { MFile nextFile = iter.next(); // try { MCollection result = new CollectionSingleIndexFile( nextFile, logger); result.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); return result; //} catch (IOException e) { // logger.error("PartitionManagerFromList failed on "+nextFile.getPath(), e); // throw new RuntimeException(e); // } }
static private boolean updateGribCollection(boolean isGrib1, MCollection dcm, CollectionUpdateType updateType, FeatureCollectionConfig.PartitionType ptype, Logger logger, Formatter errlog) throws IOException { if (debug) System.out.printf("GribCdmIndex.updateGribCollection %s %s%n", dcm.getCollectionName(), updateType); if (!isUpdateNeeded(dcm.getIndexFilename(NCX_SUFFIX), updateType, (isGrib1 ? GribCollectionType.GRIB1 : GribCollectionType.GRIB2), logger)) return false; boolean changed; if (isGrib1) { // existing case handles correctly - make seperate index for each runtime (OR) partition == runtime Grib1CollectionBuilder builder = new Grib1CollectionBuilder(dcm.getCollectionName(), dcm, logger); changed = builder.updateNeeded(updateType) && builder.createIndex(ptype, errlog); } else { Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm.getCollectionName(), dcm, logger); changed = builder.updateNeeded(updateType) && builder.createIndex(ptype, errlog); } return changed; }
@Override public Iterable<MCollection> makePartitions(CollectionUpdateType forceCollection) throws IOException { List<MCollection> result = new ArrayList<>(100); try (CloseableIterator<MFile> iter = getFileIterator()) { while (iter.hasNext()) { MCollection part = new CollectionSingleFile(iter.next(), logger); result.add( part); lastModified = Math.max(lastModified, part.getLastModified()); } } return result; }
public void update() throws IOException { datasets = new ArrayList<>(); for (MFile f : manager.getFilesSorted()) datasets.add(new Dataset(f)); if (manager.hasDateExtractor()) { if (datasets.size() == 1) { Dataset ds = datasets.get(0); if (ds.start != null) dateRange = CalendarDateRange.of(ds.start, ds.start); // LOOK ?? } else if (datasets.size() > 1) { for (int i = 0; i < datasets.size() - 1; i++) { Dataset d1 = datasets.get(i); Dataset d2 = datasets.get(i + 1); d1.setDateRange(CalendarDateRange.of(d1.start, d2.start)); if (i == datasets.size() - 2) // last one d2.setDateRange(new CalendarDateRange(d2.start, d1.getDateRange().getDurationInSecs())); } Dataset first = datasets.get(0); Dataset last = datasets.get(datasets.size() - 1); dateRange = CalendarDateRange.of(first.getDateRange().getStart(), last.getDateRange().getEnd()); } } }
@Override public CloseableIterator<MFile> getFileIterator() throws IOException { return new MFileIterator(org.getFileIterator(), filter); } }
Dataset(MFile f) { this.location = f.getPath(); this.start = manager.extractDate(f); }
@Test public void testTimePartition() throws IOException { FeatureCollectionConfig config = new FeatureCollectionConfig("ds627.1", "test/ds627.1", FeatureCollectionType.GRIB1, TestDir.cdmUnitTestDir + "gribCollections/rdavm/ds627.1/.*gbx9", null, "#ei.mdfa.fc12hr.sfc.regn128sc.#yyyyMMddhh", null, "year", null); Formatter errlog = new Formatter(); CollectionSpecParser specp = new CollectionSpecParser(config.spec, errlog); try (TimePartition tp = new TimePartition(config, specp, logger)) { tp.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); int countP = 0; for (MCollection mc : tp.makePartitions(CollectionUpdateType.always)) { System.out.printf("%s%n", mc); countP++; int count = 0; for (MFile mfile : mc.getFilesSorted()) { System.out.printf(" %s%n", mfile); count++; } assert count == 12 : count; } assert countP == 34 : countP; } }
public PartitionManagerFromIndexList(MCollection dcm, List<MFile> partFiles, org.slf4j.Logger logger) { super(dcm.getCollectionName(), logger); this.config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); this.root = dcm.getRoot(); this.partIndexFiles = partFiles; }