@Override public long getLastModified() { return mfile.getLastModified(); }
@Override public long getLastChanged() { return mfile.getLastModified(); }
public boolean accept(MFile dataset) { long lastModified = dataset.getLastModified(); if (lastModified < 0) return true; // means dont know - can happen for remote files long now = System.currentTimeMillis(); if (now - lastModified > lastModifiedLimitInMillis) return true; return false; }
public boolean accept(MFile dataset) { long lastModified = dataset.getLastModified(); if (lastModified < 0) return true; // means dont know - can happen for remote files long now = System.currentTimeMillis(); if (now - lastModified > lastModifiedLimitInMillis) return true; return false; }
public boolean accept(MFile dataset) { long lastModified = dataset.getLastModified(); if (lastModified < 0) return true; // means dont know - can happen for remote files long now = System.currentTimeMillis(); if (now - lastModified > lastModifiedLimitInMillis) return true; return false; }
public CollectionSingleFile(MFile file, org.slf4j.Logger logger) { super(file.getName(), logger); mfiles.add(file); Path p = Paths.get(file.getPath()); if (p.getParent() != null) this.root = p.getParent().toString(); else this.root = System.getProperty("user.dir"); this.lastModified = file.getLastModified(); }
public CollectionSingleFile(MFile file, org.slf4j.Logger logger) { super(file.getName(), logger); mfiles.add(file); Path p = Paths.get(file.getPath()); if (p.getParent() != null) this.root = p.getParent().toString(); else this.root = System.getProperty("user.dir"); this.lastModified = file.getLastModified(); }
public boolean hasntChangedSince(MFile file, long when) { String idxPath = file.getPath(); if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; File idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return true; if (idxFile.lastModified() < file.getLastModified()) return true; if (0 < when && idxFile.lastModified() < when) return true; return false; } };
public boolean hasChangedSince(MFile file, long when) { String idxPath = file.getPath(); if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; File idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return true; long idxLastModified = idxFile.lastModified(); if (idxLastModified < file.getLastModified()) return true; if (0 < when && when < idxLastModified) return true; return false; } public boolean hasntChangedSince(MFile file, long when) {
private boolean scanFirstTime() throws IOException { Map<String, MFile> newMap = new HashMap<>(); if (!hasScans()) { map = newMap; return false; } reallyScan(newMap); // deleteOld(newMap); // ?? hmmmmm LOOK this seems wrong; maintainence in background ?? generally collection doesnt exist // implement olderThan if (olderThanInMsecs > 0) { long olderThan = System.currentTimeMillis() - olderThanInMsecs; // new files must be older than this. Iterator<MFile> iter = newMap.values().iterator(); // need iterator so we can remove() while (iter.hasNext()) { MFile newFile = iter.next(); String path = newFile.getPath(); if (newFile.getLastModified() > olderThan) { // the file is too new iter.remove(); logger.debug("{}: scan found new Dataset but its too recently modified = {}", collectionName, path); } } } map = newMap; this.lastScanned = System.currentTimeMillis(); this.lastChanged.set(this.lastScanned); logger.debug("{} : initial scan found n datasets = {} ", collectionName, map.keySet().size()); return map.keySet().size() > 0; }
public static List<GcMFile> makeFiles(File directory, List<MFile> files, Set<Integer> allFileSet) { List<GcMFile> result = new ArrayList<>(files.size()); String dirPath = StringUtil2.replace(directory.getPath(), '\\', "/"); for (int index : allFileSet) { MFile file = files.get(index); String filename; if (file.getPath().startsWith(dirPath)) { filename = file.getPath().substring(dirPath.length()); if (filename.startsWith("/")) filename = filename.substring(1); } else filename = file.getPath(); // when does this happen ?? result.add( new GcMFile(directory, filename, file.getLastModified(), file.getLength(), index)); } return result; }
private boolean scanFirstTime() throws IOException { Map<String, MFile> newMap = new HashMap<>(); if (!hasScans()) { map = newMap; return false; } reallyScan(newMap); // deleteOld(newMap); // ?? hmmmmm LOOK this seems wrong; maintainence in background ?? generally collection doesnt exist // implement olderThan if (olderThanInMsecs > 0) { long olderThan = System.currentTimeMillis() - olderThanInMsecs; // new files must be older than this. Iterator<MFile> iter = newMap.values().iterator(); // need iterator so we can remove() while (iter.hasNext()) { MFile newFile = iter.next(); String path = newFile.getPath(); if (newFile.getLastModified() > olderThan) { // the file is too new iter.remove(); logger.debug("{}: scan found new Dataset but its too recently modified = {}", collectionName, path); } } } map = newMap; this.lastScanned = System.currentTimeMillis(); this.lastChanged.set(this.lastScanned); logger.debug("{} : initial scan found n datasets = {} ", collectionName, map.keySet().size()); return map.keySet().size() > 0; }
private boolean scanFirstTime() throws IOException { Map<String, MFile> newMap = new HashMap<String, MFile>(); if (!hasScans()) { map = newMap; return false; } reallyScan(newMap); // deleteOld(newMap); // ?? hmmmmm LOOK this seems wrong; maintainence in background ?? generally collection doesnt exist // implement olderThan if (olderThanInMsecs > 0) { long olderThan = System.currentTimeMillis() - olderThanInMsecs; // new files must be older than this. Iterator<MFile> iter = newMap.values().iterator(); // need iterator so we can remove() while (iter.hasNext()) { MFile newFile = iter.next(); String path = newFile.getPath(); if (newFile.getLastModified() > olderThan) { // the file is too new iter.remove(); logger.debug("{}: scan found new Dataset but its too recently modified = {}", collectionName, path); } } } synchronized (this) { map = newMap; this.lastScanned = System.currentTimeMillis(); this.lastChanged.set(this.lastScanned); } logger.debug("{} : initial scan found n datasets = {} ", collectionName, map.keySet().size()); return map.keySet().size() > 0; }
MFile oldFile = oldMap.get(path); if (oldFile != null) { if (newFile.getLastModified() > oldFile.getLastModified()) { // the file has changed since last time nchange++; logger.debug("{}: scan found Dataset changed= {}", collectionName, path); } else if (changeChecker != null && changeChecker.hasntChangedSince(newFile, oldFile.getLastModified())) { // the ancilliary file hasnt changed nchange++; logger.debug("{}: scan changeChecker found Dataset changed= {}", collectionName, path); if (olderThan > 0 && newFile.getLastModified() > olderThan) { // the file is too new iter.remove(); logger.debug("{}: scan found new Dataset but its too recently modified = {}", collectionName, path);
public static GribIndex open(boolean isGrib1, MFile mfile) throws IOException { GribIndex index = isGrib1 ? new Grib1Index() : new Grib2Index(); if (!index.readIndex(mfile.getPath(), mfile.getLastModified(), CollectionUpdateType.never)) { return null; } return index; }
long fileModifiedSecs = mfile.getLastModified() / 1000; // ignore msecs long xmlModifiedSecs = inv.getLastModified() / 1000; // ignore msecs if (xmlModifiedSecs >= fileModifiedSecs) { // LOOK if fileDate is -1, will always succeed if (log.isDebugEnabled()) log.debug(" cache ok "+new Date(inv.getLastModified())+" >= "+new Date(mfile.getLastModified())+" for " + mfile.getName()); return inv; // ok, use it } else { if (log.isInfoEnabled()) log.info(" cache out of date "+new Date(inv.getLastModified())+" < "+new Date(mfile.getLastModified())+" for " + mfile.getName()); String xmlString = inv.writeXML( new Date(mfile.getLastModified())); ((CollectionManagerAbstract)cm).putMetadata(mfile, "fmrInv.xml", xmlString.getBytes(CDM.utf8Charset)); if (log.isDebugEnabled()) log.debug(" added xmlFile "+ mfile.getPath()+".fmrInv.xml to cache");
long fileModifiedSecs = mfile.getLastModified() / 1000; // ignore msecs long xmlModifiedSecs = inv.getLastModified() / 1000; // ignore msecs if (xmlModifiedSecs >= fileModifiedSecs) { // LOOK if fileDate is -1, will always succeed if (logger.isDebugEnabled()) logger.debug(" cache ok "+new Date(inv.getLastModified())+" >= "+new Date(mfile.getLastModified())+" for " + mfile.getName()); return inv; // ok, use it } else { if (logger.isInfoEnabled()) logger.info(" cache out of date "+new Date(inv.getLastModified())+" < "+new Date(mfile.getLastModified())+" for " + mfile.getName()); String xmlString = inv.writeXML( new Date(mfile.getLastModified())); ((CollectionManagerAbstract)cm).putMetadata(mfile, "fmrInv.xml", xmlString.getBytes(CDM.utf8Charset)); if (logger.isDebugEnabled()) logger.debug(" added xmlFile "+ mfile.getPath()+".fmrInv.xml to cache");
long fileModifiedSecs = mfile.getLastModified() / 1000; // ignore msecs long xmlModifiedSecs = inv.getLastModified() / 1000; // ignore msecs if (xmlModifiedSecs >= fileModifiedSecs) { // LOOK if fileDate is -1, will always succeed if (log.isDebugEnabled()) log.debug(" cache ok "+new Date(inv.getLastModified())+" >= "+new Date(mfile.getLastModified())+" for " + mfile.getName()); return inv; // ok, use it } else { if (log.isInfoEnabled()) log.info(" cache out of date "+new Date(inv.getLastModified())+" < "+new Date(mfile.getLastModified())+" for " + mfile.getName()); String xmlString = inv.writeXML( new Date(mfile.getLastModified())); ((CollectionManagerAbstract)cm).putMetadata(mfile, "fmrInv.xml", xmlString.getBytes(CDM.utf8Charset)); if (log.isDebugEnabled()) log.debug(" added xmlFile "+ mfile.getPath()+".fmrInv.xml to cache");
/** * Create a gbx9 index from a single grib1 or grib2 file. * Use the existing index if it already exists. * * @param isGrib1 true if grib1 * @param mfile the grib file * @param force force writing index * @return the resulting GribIndex * @throws IOException on io error */ public static GribIndex readOrCreateIndexFromSingleFile(boolean isGrib1, MFile mfile, CollectionUpdateType force, org.slf4j.Logger logger) throws IOException { GribIndex index = isGrib1 ? new Grib1Index() : new Grib2Index(); if (!index.readIndex(mfile.getPath(), mfile.getLastModified(), force)) { // heres where the index date is checked against the data file index.makeIndex(mfile.getPath(), null); logger.debug(" Index written: {} == {} records", mfile.getName() + GBX9_IDX, index.getNRecords()); } else if (debug) { logger.debug(" Index read: {} == {} records", mfile.getName() + GBX9_IDX, index.getNRecords()); } return index; }
if (now - mfile.getLastModified() < config.addLatest.lastModLimit) continue; ds.put(Dataset.UrlPath, urlPath); CalendarDate date = CalendarDate.of(mfile.getLastModified()); ds.put(Dataset.Dates, new DateType(date).setType("modified")); // <date type="modified">2011-09-02T20:50:58.288Z</date> ds.put(Dataset.Id, parentId + mfile.getName());