static public MCollection open(String collectionName, String collectionSpec, String olderThan, Formatter errlog) throws IOException { if (collectionSpec.startsWith(CATALOG)) return new CollectionManagerCatalog(collectionName, collectionSpec.substring(CATALOG.length()), olderThan, errlog); else if (collectionSpec.startsWith(DIR)) return new DirectoryCollection(collectionName, collectionSpec.substring(DIR.length()), olderThan, null); else if (collectionSpec.startsWith(FILE)) return new CollectionSingleFile(MFileOS7.getExistingFile(collectionSpec.substring(FILE.length())), null); else if (collectionSpec.startsWith(LIST)) return new CollectionList(collectionName, collectionSpec.substring(LIST.length()), null); else if (collectionSpec.startsWith(GLOB)) return new CollectionGlob(collectionName, collectionSpec.substring(GLOB.length()), null); else return MFileCollectionManager.open(collectionName, collectionSpec, olderThan, errlog); }
static public MCollection open(String collectionName, String collectionSpec, String olderThan, Formatter errlog) throws IOException { if (collectionSpec.startsWith(CATALOG)) return new CollectionManagerCatalog(collectionName, collectionSpec.substring(CATALOG.length()), olderThan, errlog); else if (collectionSpec.startsWith(DIR)) return new DirectoryCollection(collectionName, collectionSpec.substring(DIR.length()), true, olderThan, null); else if (collectionSpec.startsWith(FILE)) { MFile file = MFileOS7.getExistingFile(collectionSpec.substring(FILE.length())); if (file == null) throw new FileNotFoundException(collectionSpec.substring(FILE.length())); return new CollectionSingleFile(file, null); }else if (collectionSpec.startsWith(LIST)) return new CollectionList(collectionName, collectionSpec.substring(LIST.length()), null); else if (collectionSpec.startsWith(GLOB)) return new CollectionGlob(collectionName, collectionSpec.substring(GLOB.length()), null); else return MFileCollectionManager.open(collectionName, collectionSpec, olderThan, errlog); }
@Override public Iterable<MCollection> makePartitions(CollectionUpdateType forceCollection) throws IOException { List<MCollection> result = new ArrayList<>(100); try (CloseableIterator<MFile> iter = getFileIterator()) { while (iter.hasNext()) { MCollection part = new CollectionSingleFile(iter.next(), logger); result.add( part); lastModified = Math.max(lastModified, part.getLastModified()); } } return result; }
public void consume(MFile mfile) { MCollection part = new CollectionSingleFile(mfile, logger); part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); try { boolean changed = updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.file, logger, errlog); if (changed) anyChange.set(true); } catch (IllegalStateException t) { logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage()); partition.removePartition(part); // keep on truckin; can happen if directory is empty } catch (Throwable t) { logger.error("Error making partition " + part.getRoot(), t); partition.removePartition(part); } } });
@Override public Iterable<MCollection> makePartitions(CollectionUpdateType forceCollection) throws IOException { List<MCollection> result = new ArrayList<>(100); try (CloseableIterator<MFile> iter = getFileIterator()) { while (iter.hasNext()) { MCollection part = new CollectionSingleFile(iter.next(), logger); if (!wasRemoved(part)) result.add(part); lastModified = Math.max(lastModified, part.getLastModified()); } } return result; }
partition.iterateOverMFileCollection(new DirectoryCollection.Visitor() { public void consume(MFile mfile) { MCollection part = new CollectionSingleFile(mfile, logger); part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
static public GribCollectionImmutable openGribCollectionFromDataFile(boolean isGrib1, MFile mfile, CollectionUpdateType updateType, FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) throws IOException { MCollection dcm = new CollectionSingleFile(mfile, logger); dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (isGrib1) { Grib1CollectionBuilder builder = new Grib1CollectionBuilder(dcm.getCollectionName(), dcm, logger); // LOOK ignoring partition type boolean changed = (builder.updateNeeded(updateType) && builder.createIndex(FeatureCollectionConfig.PartitionType.file, errlog)); } else { Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm.getCollectionName(), dcm, logger); boolean changed = (builder.updateNeeded(updateType) && builder.createIndex(FeatureCollectionConfig.PartitionType.file, errlog)); } // the index file should now exist, open it GribCollectionImmutable result = openCdmIndex(dcm.getIndexFilename(NCX_SUFFIX), config, true, logger); if (result != null) return result; // if open fails, force recreate the index if (updateType == CollectionUpdateType.never) return null; // not allowed to write if (updateType == CollectionUpdateType.always) return null;// already tried to force write, give up return openGribCollectionFromDataFile(isGrib1, mfile, CollectionUpdateType.always, config, errlog, logger); }