public GribCollectionMutable makeGribCollection() throws IOException { GribCollectionMutable result = GribCdmIndex.openMutableGCFromIndex(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), config, false, true, logger); if (result == null) { logger.error("Failed on openMutableGCFromIndex {}", dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX)); return null; } lastModified = result.lastModified; fileSize = result.fileSize; if (result.masterRuntime != null) partitionDate = result.masterRuntime.getFirstDate(); return result; }
public boolean updateNeeded(CollectionUpdateType ff) throws IOException { if (ff == CollectionUpdateType.never) return false; if (ff == CollectionUpdateType.always) return true; File collectionIndexFile = GribIndexCache.getExistingFileOrCache(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX)); if (collectionIndexFile == null) return true; if (ff == CollectionUpdateType.nocheck) return false; return needsUpdate(ff, collectionIndexFile); }
private boolean needsUpdate(CollectionUpdateType ff, File collectionIndexFile) throws IOException { long collectionLastModified = collectionIndexFile.lastModified(); Set<String> newFileSet = new HashSet<>(); for (MCollection dcm : partitionManager.makePartitions(CollectionUpdateType.test)) { String partitionIndexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/"); File partitionIndexFile = GribIndexCache.getExistingFileOrCache(partitionIndexFilename); if (partitionIndexFile == null) // make sure each partition has an index return true; if (collectionLastModified < partitionIndexFile.lastModified()) // and the partition index is earlier than the collection index return true; newFileSet.add(partitionIndexFilename); } if (ff == CollectionUpdateType.testIndexOnly) return false; // now see if any files were deleted GribCdmIndex reader = new GribCdmIndex(logger); List<MFile> oldFiles = new ArrayList<>(); reader.readMFiles(collectionIndexFile.toPath(), oldFiles); Set<String> oldFileSet = new HashSet<>(); for (MFile oldFile : oldFiles) { if (!newFileSet.contains(oldFile.getPath())) return true; // got deleted - must recreate the index oldFileSet.add(oldFile.getPath()); } // now see if any files were added for (String newFilename : newFileSet) { if (!oldFileSet.contains(newFilename)) return true; // got added - must recreate the index } return false; }
public Partition(MCollection dcm) { FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); if (config == null) logger.warn("Partition missing FeatureCollectionConfig {}", dcm); this.dcm = dcm; this.name = dcm.getCollectionName(); this.lastModified = dcm.getLastModified(); this.directory = StringUtil2.replace(dcm.getRoot(), '\\', "/"); this.partitionDate = dcm.getPartitionDate(); String indexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/"); if (partitionDate == null) { partitionDate = getDateExtractor().getCalendarDateFromPath(indexFilename); // LOOK dicey } // now remove the directory if (indexFilename.startsWith(directory)) { indexFilename = indexFilename.substring(directory.length()); if (indexFilename.startsWith("/")) indexFilename = indexFilename.substring(1); } filename = indexFilename; }
static private boolean updateGribCollection(boolean isGrib1, MCollection dcm, CollectionUpdateType updateType, FeatureCollectionConfig.PartitionType ptype, Logger logger, Formatter errlog) throws IOException { if (debug) System.out.printf("GribCdmIndex.updateGribCollection %s %s%n", dcm.getCollectionName(), updateType); if (!isUpdateNeeded(dcm.getIndexFilename(NCX_SUFFIX), updateType, (isGrib1 ? GribCollectionType.GRIB1 : GribCollectionType.GRIB2), logger)) return false; boolean changed; if (isGrib1) { // existing case handles correctly - make seperate index for each runtime (OR) partition == runtime Grib1CollectionBuilder builder = new Grib1CollectionBuilder(dcm.getCollectionName(), dcm, logger); changed = builder.updateNeeded(updateType) && builder.createIndex(ptype, errlog); } else { Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm.getCollectionName(), dcm, logger); changed = builder.updateNeeded(updateType) && builder.createIndex(ptype, errlog); } return changed; }
static public GribCollectionImmutable openGribCollectionFromDataFile(boolean isGrib1, MFile mfile, CollectionUpdateType updateType, FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) throws IOException { MCollection dcm = new CollectionSingleFile(mfile, logger); dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (isGrib1) { Grib1CollectionBuilder builder = new Grib1CollectionBuilder(dcm.getCollectionName(), dcm, logger); // LOOK ignoring partition type boolean changed = (builder.updateNeeded(updateType) && builder.createIndex(FeatureCollectionConfig.PartitionType.file, errlog)); } else { Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm.getCollectionName(), dcm, logger); boolean changed = (builder.updateNeeded(updateType) && builder.createIndex(FeatureCollectionConfig.PartitionType.file, errlog)); } // the index file should now exist, open it GribCollectionImmutable result = openCdmIndex(dcm.getIndexFilename(NCX_SUFFIX), config, true, logger); if (result != null) return result; // if open fails, force recreate the index if (updateType == CollectionUpdateType.never) return null; // not allowed to write if (updateType == CollectionUpdateType.always) return null;// already tried to force write, give up return openGribCollectionFromDataFile(isGrib1, mfile, CollectionUpdateType.always, config, errlog, logger); }