public GribCollectionBuilder(boolean isGrib1, String name, MCollection dcm, org.slf4j.Logger logger) { this.dcm = dcm; this.logger = logger; this.isGrib1 = isGrib1; this.name = StringUtil2.replace(name, ' ', "_"); this.directory = new File(dcm.getRoot()); }
public CollectionFiltered(String name, MCollection org, MFileFilter filter) { super(name, null); this.org = org; this.filter = filter; setRoot( org.getRoot()); }
protected void makeCollection() { Formatter errlog = new Formatter(); if (config.spec.startsWith(MFileCollectionManager.CATALOG)) { // LOOK CHANGE THIS datasetCollection = new CollectionManagerCatalog(config.name, config.spec, null, errlog); } else { datasetCollection = new MFileCollectionManager(config, errlog, this.logger); } topDirectory = datasetCollection.getRoot(); String errs = errlog.toString(); if (errs.length() > 0) logger.warn("MFileCollectionManager parse error = {} ", errs); }
public void consume(MFile mfile) { MCollection part = new CollectionSingleFile(mfile, logger); part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); try { boolean changed = updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.file, logger, errlog); if (changed) anyChange.set(true); } catch (IllegalStateException t) { logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage()); partition.removePartition(part); // keep on truckin; can happen if directory is empty } catch (Throwable t) { logger.error("Error making partition " + part.getRoot(), t); partition.removePartition(part); } } });
public PartitionManagerFromIndexList(MCollection dcm, List<MFile> partFiles, org.slf4j.Logger logger) { super(dcm.getCollectionName(), logger); this.config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); this.root = dcm.getRoot(); this.partIndexFiles = partFiles; }
public PartitionManagerFromIndexList(MCollection dcm, List<MFile> partFiles, org.slf4j.Logger logger) { super(dcm.getCollectionName(), logger); this.config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); this.root = dcm.getRoot(); this.partIndexFiles = partFiles; }
public Partition(MCollection dcm) { FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); if (config == null) logger.warn("Partition missing FeatureCollectionConfig {}", dcm); this.dcm = dcm; this.name = dcm.getCollectionName(); this.lastModified = dcm.getLastModified(); this.directory = StringUtil2.replace(dcm.getRoot(), '\\', "/"); this.partitionDate = dcm.getPartitionDate(); String indexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/"); if (partitionDate == null) { partitionDate = getDateExtractor().getCalendarDateFromPath(indexFilename); // LOOK dicey } // now remove the directory if (indexFilename.startsWith(directory)) { indexFilename = indexFilename.substring(directory.length()); if (indexFilename.startsWith("/")) indexFilename = indexFilename.substring(1); } filename = indexFilename; }
updateDirectoryCollectionRecurse(isGrib1, (DirectoryPartition) part, config, updateType, logger); } else { Path partPath = Paths.get(part.getRoot()); updateLeafCollection(isGrib1, config, updateType, false, logger, partPath); // LOOK why not using part ?? logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage()); logger.error("Error making partition " + part.getRoot(), t); dpart.removePartition(part);
static private boolean updateTimePartition(boolean isGrib1, TimePartition tp, CollectionUpdateType updateType, Logger logger) throws IOException { if (debug) System.out.printf("GribCdmIndex.updateTimePartition %s %s%n", tp.getRoot(), updateType); if (!isUpdateNeeded(tp.getIndexFilename(NCX_SUFFIX), updateType, (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger)) return false; long start = System.currentTimeMillis(); Formatter errlog = new Formatter(); for (MCollection part : tp.makePartitions(updateType)) { try { updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.timePeriod, logger, errlog); } catch (Throwable t) { logger.warn("Error making partition " + part.getRoot(), t); tp.removePartition(part); // keep on truckin; can happen if directory is empty } } // loop over component grib collections try { boolean changed = updatePartition(isGrib1, tp, updateType, logger, errlog); long took = System.currentTimeMillis() - start; errlog.format(" INFO updateTimePartition %s took %d msecs%n", tp.getRoot(), took); if (debug) System.out.printf("GribCdmIndex.updateTimePartition complete (%s) on %s errlog=%s%n", changed, tp.getRoot(), errlog); return changed; } catch (IllegalStateException t) { logger.warn("Error making partition {} '{}'", tp.getRoot(), t.getMessage()); return false; } catch (Throwable t) { logger.error("Error making partition " + tp.getRoot(), t); return false; } }