private Fmrc(FeatureCollectionConfig config, Formatter errlog) { this.manager = new MFileCollectionManager(config, errlog, null); this.config = config; }
public void addCollection(String spec, String olderThan) throws IOException { datasetManager = MFileCollectionManager.open(spec, olderThan, null); }
@Override public boolean scanIfNeeded() throws IOException { if (map == null && !isStatic()) return true; return isScanNeeded() && scan(true); }
public long getLastModified() { try { boolean wantScan = datasetManager.isScanNeeded(); datasetManager.scanIfNeeded(); } catch (IOException e) { logger.error("Aggregation scan failed, e"); } return datasetManager.getLastChanged(); }
public long getLastModified() { try { datasetManager.scanIfNeeded(); } catch (IOException e) { logger.error("Aggregation scan failed, e"); } return datasetManager.getLastChanged(); }
private List<CollectionManager> makePartitionsFromSubdirs() throws IOException { this.type = Type.directory; MController controller = MFileCollectionManager.getController(); // make sure loaded MFileCollectionManager dcm = new MFileCollectionManager(name, mcs, cdate, this.logger); dcm.setDateExtractor(dateExtractor); if (config != null && config.gribConfig != null) dcm.putAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG, config.gribConfig); dcm.scan(false); result.add(dcm);
public void addDirectoryScanFmrc(String dirName, String suffix, String regexpPatternString, String subdirs, String olderThan, String runMatcher, String forecastMatcher, String offsetMatcher) throws IOException { // only one this.runMatcher = runMatcher; //this.forecastMatcher = forecastMatcher; //this.offsetMatcher = offsetMatcher; // this.enhance = NetcdfDataset.getDefaultEnhanceMode(); isDate = true; //DatasetScanner d = new DatasetScanner(null, dirName, suffix, regexpPatternString, subdirs, olderThan); //datasetManager.addDirectoryScan(d); datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, null); if (runMatcher != null) { DateExtractor dateExtractor = new DateExtractorFromName(runMatcher, false); datasetManager.setDateExtractor(dateExtractor); } }
if (!hasScans()) { logger.debug("{}: scan not needed, no scanners", collectionName); return false; if (map == null && !isStatic()) { logger.debug("{}: scan needed, never scanned", collectionName); return true; Date lastCheckedDate = new Date(getLastScanned()); Date need = recheck.add(lastCheckedDate); if (now.before(need)) {
String olderThan = scanElem.getAttributeValue("olderThan"); datasetManager = MFileCollectionManager.openWithRecheck(ncmlLocation, recheck); datasetManager.addDirectoryScan(dirLocation, suffix, regexpPatternString, subdirs, olderThan, null); dateExtractor = new DateExtractorFromName(runDateMatcher, false); datasetManager.setDateExtractor(dateExtractor);
@Test @Ignore("tests fail on jenkins due to file permisssions") public void testScanFromConfig() throws IOException { //public FeatureCollectionConfig(String name, FeatureCollectionType fcType, String spec, String dateFormatMark, String olderThan, String recheckAfter, // String timePartition, String useIndexOnlyS, Element innerNcml) { // public FeatureCollectionConfig(String name, String path, FeatureCollectionType fcType, String spec, // String dateFormatMark, String olderThan, // String timePartition, String useIndexOnlyS, Element innerNcml) { FeatureCollectionConfig config = new FeatureCollectionConfig("testScanFromConfig", "path", FeatureCollectionType.FMRC, TestDir.cdmUnitTestDir + "agg/updating/.*nc$", null, null, "10 sec", null, null); assert touch(TestDir.cdmUnitTestDir + "agg/updating/extra.nc"); // count scanned files Formatter f = new Formatter(System.out); MFileCollectionManager dcm = new MFileCollectionManager(config, f, null); dcm.scan(true); List<MFile> fileList = (List<MFile>) Misc.getList(dcm.getFilesSorted()); assert fileList.size() == 2 : dcm; }
public CalendarDateRange update() throws IOException { datasets = new ArrayList<>(); manager.scan(false); for (MFile f : manager.getFilesSorted()) datasets.add(new Dataset(f)); if (manager.hasDateExtractor()) { if (datasets.size() == 1) { Dataset ds = datasets.get(0); if (ds.start != null) dateRange = CalendarDateRange.of(ds.start, ds.start); // LOOK ?? } else if (datasets.size() > 1) { for (int i = 0; i < datasets.size() - 1; i++) { Dataset d1 = datasets.get(i); Dataset d2 = datasets.get(i + 1); d1.setDateRange(CalendarDateRange.of(d1.start, d2.start)); if (i == datasets.size() - 2) // last one d2.setDateRange(new CalendarDateRange(d2.start, d1.getDateRange().getDurationInSecs())); } Dataset first = datasets.get(0); Dataset last = datasets.get(datasets.size() - 1); dateRange = CalendarDateRange.of(first.getDateRange().getStart(), last.getDateRange().getEnd()); } } return dateRange; }
@Override public synchronized Iterable<MFile> getFilesSorted() { if (map == null) try { scanFirstTime(); // never scanned } catch (IOException e) { e.printStackTrace(); return Collections.emptyList(); } List<MFile> result = new ArrayList<>(map.values()); if (hasDateExtractor()) { Collections.sort(result, new DateSorter()); } else { Collections.sort(result); } return result; }
if (dimName != null) out.print("dimName='" + dimName + "' "); if (datasetManager.getRecheck() != null) out.print("recheckEvery='" + datasetManager.getRecheck() + "' "); out.print(">\n"); long time = datasetManager.getLastScanned(); if (time == 0) time = System.currentTimeMillis(); // no scans (eg all static) will have a 0 logger.debug("Aggregation persisted = " + cacheFile.getPath() + " lastModified= " + new Date(datasetManager.getLastScanned()));
public void finish(CancelTask cancelTask) throws IOException { datasetManager.scan(true); // Make the list of Datasets, by scanning if needed. cacheDirty = true; makeDatasets(cancelTask); //ucar.unidata.io.RandomAccessFile.setDebugAccess( true); buildNetcdfDataset(cancelTask); //ucar.unidata.io.RandomAccessFile.setDebugAccess( false); }
realLocationRunTimeMap.put(realLocation, runTime); datasetManager = MFileCollectionManager.openWithRecheck(ncmlLocation, recheck); datasetManager.setFilesAndRunDate(realLocationRunTimeMap); String olderThan = scanElem.getAttributeValue("olderThan"); datasetManager = MFileCollectionManager.openWithRecheck(ncmlLocation, recheck); datasetManager.addDirectoryScan(dirLocation, suffix, regexpPatternString, subdirs, olderThan, null); dateExtractor = new DateExtractorFromName(runDateMatcher, false); datasetManager.setDateExtractor(dateExtractor);
protected void reallyScan(java.util.Map<String, MFile> map) throws IOException { getController(); // make sure a controller is instantiated if (hasScans()) {
protected void reallyScan(java.util.Map<String, MFile> map) throws IOException { getController(); // make sure a controller is instantiated // run through all scanners and collect MFile instances into the Map int count = 0; for (CollectionConfig mc : scanList) { long start = System.currentTimeMillis(); // System.out.printf("MFileCollectionManager reallyScan %s %s%n", mc.getDirectoryName(), CalendarDate.present()); // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? Iterator<MFile> iter = (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC wants subdir /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ if (iter == null) { logger.error(collectionName + ": Invalid collection= " + mc); continue; } while (iter.hasNext()) { MFile mfile = iter.next(); mfile.setAuxInfo(mc.getAuxInfo()); map.put(mfile.getPath(), mfile); count++; } long took = (System.currentTimeMillis() - start) / 1000; // System.out.printf("MFileCollectionManager reallyScan %s took %d secs%n", collectionName, took); if (logger.isDebugEnabled()) { long took2 = (System.currentTimeMillis() - start) / 1000; logger.debug("{} : was scanned nfiles= {} took={} secs", collectionName, count, took2); } } if (map.size() == 0) { logger.warn("MFileCollectionManager: No files found for {}", collectionName); } }
for (MFile cd : datasetManager.getFilesSorted()) { datasets.add( makeDataset(cd));
public void addDirectoryScanFmrc(String dirName, String suffix, String regexpPatternString, String subdirs, String olderThan, String runMatcher, String forecastMatcher, String offsetMatcher) throws IOException { // only one this.runMatcher = runMatcher; //this.forecastMatcher = forecastMatcher; //this.offsetMatcher = offsetMatcher; // this.enhance = NetcdfDataset.getDefaultEnhanceMode(); isDate = true; //DatasetScanner d = new DatasetScanner(null, dirName, suffix, regexpPatternString, subdirs, olderThan); //datasetManager.addDirectoryScan(d); datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, null); if (runMatcher != null) { DateExtractor dateExtractor = new DateExtractorFromName(runMatcher, false); datasetManager.setDateExtractor(dateExtractor); } }
if (!hasScans()) { logger.debug("{}: scan not needed, no scanners", collectionName); return false; if (map == null && !isStatic()) { logger.debug("{}: scan needed, never scanned", collectionName); return true; Date lastCheckedDate = new Date(getLastScanned()); Date need = recheck.add(lastCheckedDate); if (now.before(need)) {