/** * Method to confirm that identifier can be deleted from {@link Backend} * * @param identifier * @return */ public boolean confirmDelete(DataIdentifier identifier) { if (isInUse(identifier)) { LOG.debug("identifier [{}] is inUse confirmDelete= false ", identifier); return false; } String fileName = getFileName(identifier); long lastModified = asyncWriteCache.getLastModified(fileName); if (lastModified != 0) { LOG.debug( "identifier [{}] is asyncWriteCache map confirmDelete= false ", identifier); return false; } if (asyncTouchCache.get(identifier) != null) { LOG.debug( "identifier [{}] is asyncTouchCache confirmDelete = false ", identifier); return false; } return true; }
/** * Return the length of record from {@link LocalCache} if available, * otherwise retrieve it from {@link Backend}. */ public long getLength(final DataIdentifier identifier) throws DataStoreException { String fileName = getFileName(identifier); Long length = recLenCache.get(identifier); if (length != null) { LOG.trace(" identifier [{}] length fetched from recLengthCache", identifier); return length; } else if ((length = cache.getFileLength(fileName)) != null) { LOG.trace(" identifier [{}] length fetched from local cache", identifier); recLenCache.put(identifier, length); return length; } else { length = backend.getLength(identifier); LOG.debug(" identifier [{}] length fetched from backend", identifier); recLenCache.put(identifier, length); asyncDownload(identifier); return length; } }
@Override public void onSuccess(AsyncUploadResult result) { DataIdentifier identifier = result.getIdentifier(); File file = result.getFile(); String fileName = getFileName(identifier); try { LOG.debug("Upload completed for [{}]", identifier); // remove from failed upload map if any. uploadRetryMap.remove(identifier); AsyncUploadCacheResult cachedResult = asyncWriteCache.remove(fileName); if (cachedResult.doRequiresDelete()) { // added record already marked for delete deleteRecord(identifier); } else { // async upload took lot of time. // getRecord to touch if required. getRecord(identifier); } } catch (IOException ie) { LOG.warn("Cannot remove pending file upload. Dataidentifer [ " + identifier + "], file [" + file.getAbsolutePath() + "]", ie); } catch (DataStoreException dse) { LOG.warn("Cannot remove pending file upload. Dataidentifer [ " + identifier + "], file [" + file.getAbsolutePath() + "]", dse); } }
@Override public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException { String fileName = getFileName(identifier); try { if (getLength(identifier) > -1) { LOG.trace("getRecord: [{}] retrieved using getLength", identifier); if (minModifiedDate > 0) { touchInternal(identifier); } usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } else if (asyncWriteCache.hasEntry(fileName, minModifiedDate > 0)) { LOG.trace("getRecord: [{}] retrieved from asyncUploadmap", identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } } catch (IOException ioe) { throw new DataStoreException("error in getting record [" + identifier + "]", ioe); } throw new DataStoreException("Record not found: " + identifier); }
if (getDataStore().confirmDelete(id)) { getDataStore().deleteFromCache(id); if (LOG.isInfoEnabled()) { LOG.info("Deleting old file "
@Override public Set<DataIdentifier> deleteAllOlderThan(final long min) throws DataStoreException { log("deleteAllOlderThan " + min); Set<DataIdentifier> tobeDeleted = new HashSet<DataIdentifier>(); for (Map.Entry<DataIdentifier, Long> entry : timeMap.entrySet()) { DataIdentifier identifier = entry.getKey(); long timestamp = entry.getValue(); if (timestamp < min && !store.isInUse(identifier) && store.confirmDelete(identifier)) { store.deleteFromCache(identifier); tobeDeleted.add(identifier); } } for (DataIdentifier identifier : tobeDeleted) { timeMap.remove(identifier); data.remove(identifier); } return tobeDeleted; }
long length = 0; try { temporary = newTemporaryFile(); DataIdentifier tempId = new DataIdentifier(temporary.getName()); usesIdentifier(tempId); encodeHexString(digest.digest())); LOG.debug("Digest of [{}], length =[{}] took [{}]ms ", new Object[] { identifier, length, (currTime - startTime) }); String fileName = getFileName(identifier); AsyncUploadCacheResult result = null; synchronized (this) { usesIdentifier(identifier);
LOG.info("path=[{}], tmpPath=[{}]", path, tmpDir.getAbsolutePath()); directory = new File(path); mkdirs(directory); mkdirs(new File(homeDir)); if (!mkdirs(tmpDir)) { FileUtils.cleanDirectory(tmpDir); LOG.info("tmp=[{}] cleaned.", tmpDir.getPath()); asyncWriteCacheInitStatus = false; backend = createBackend(); backend.init(this, path, config); String markerFileName = getMarkerFile(); if (markerFileName != null && !"".equals(markerFileName.trim())) { if (!markerFile.exists()) { LOG.info("load files from local cache"); uploadFilesFromCache(); try { markerFile.createNewFile(); LOG.info("Initialization of asyncWriteCache failed. " + "Re-loading all files from local cache"); uploadFilesFromCache(); asyncWriteCache.reset();
public DataRecord getRecordIfStored(DataIdentifier identifier) throws DataStoreException { String fileName = getFileName(identifier); try { if (asyncWriteCache.hasEntry(fileName, minModifiedDate > 0)) { "getRecordIfStored: [{}] retrieved from asyncuploadmap", identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } else if (recLenCache.containsKey(identifier)) { identifier); if (minModifiedDate > 0) { touchInternal(identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } else { recLenCache.put(identifier, length); if (minModifiedDate > 0) { touchInternal(identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } catch (DataStoreException ignore) {
/** * Derive file name from identifier. */ private static String getFileName(DataIdentifier identifier) { String name = identifier.toString(); return getFileName(name); }
@Override public synchronized int deleteAllOlderThan(long min) throws DataStoreException { Set<DataIdentifier> diSet = backend.deleteAllOlderThan(min); // remove entries from local cache for (DataIdentifier identifier : diSet) { recLenCache.remove(identifier); cache.delete(getFileName(identifier)); } try { for (String fileName : asyncWriteCache.deleteOlderThan(min)) { diSet.add(getIdentifier(fileName)); } } catch (IOException e) { throw new DataStoreException(e); } LOG.info( "deleteAllOlderThan exit. Deleted [{}]records. Number of records deleted [{}]", diSet, diSet.size()); return diSet.size(); }
/** * Upload file from {@link LocalCache} to {@link Backend}. * * @param f * file to uploaded. * @throws DataStoreException */ private void uploadFileToBackEnd(File f, boolean updateAsyncUploadCache) throws DataStoreException { try { DataIdentifier identifier = new DataIdentifier(f.getName()); usesIdentifier(identifier); if (!backend.exists(identifier)) { backend.write(identifier, f); } if (updateAsyncUploadCache) { String fileName = getFileName(identifier); asyncWriteCache.remove(fileName); } LOG.debug("uploaded [{}]", f.getName()); } catch (IOException ioe) { throw new DataStoreException(ioe); } }
/** * Creates an {@link Executor}. * This method is invoked during the initialization for asynchronous write/touch job executions. * @return an {@link Executor} */ protected Executor createAsyncWriteExecutor() { Executor asyncExecutor; if (dataStore.getAsyncUploadLimit() > 0 && getAsyncWritePoolSize() > 0) { asyncExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(getAsyncWritePoolSize(), new NamedThreadFactory(getClass().getSimpleName() + "-write-worker")); } else { asyncExecutor = new ImmediateExecutor(); } return asyncExecutor; }
/** * Retrieves all identifiers from {@link Backend}. */ @Override public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException { Set<DataIdentifier> ids = new HashSet<DataIdentifier>(); for (String fileName : asyncWriteCache.getAll()) { ids.add(getIdentifier(fileName)); } Iterator<DataIdentifier> itr = backend.getAllIdentifiers(); while (itr.hasNext()) { ids.add(itr.next()); } return ids.iterator(); }
if (getDataStore().confirmDelete(id)) { getDataStore().deleteFromCache(id); if (LOG.isInfoEnabled()) { LOG.info("Deleting old file "
@Override public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException { String fileName = getFileName(identifier); try { if (getLength(identifier) > -1) { LOG.trace("getRecord: [{}] retrieved using getLength", identifier); if (minModifiedDate > 0) { touchInternal(identifier); } usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } else if (asyncWriteCache.hasEntry(fileName, minModifiedDate > 0)) { LOG.trace("getRecord: [{}] retrieved from asyncUploadmap", identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } } catch (IOException ioe) { throw new DataStoreException("error in getting record [" + identifier + "]", ioe); } throw new DataStoreException("Record not found: " + identifier); }
long length = 0; try { temporary = newTemporaryFile(); DataIdentifier tempId = new DataIdentifier(temporary.getName()); usesIdentifier(tempId); encodeHexString(digest.digest())); LOG.debug("Digest of [{}], length =[{}] took [{}]ms ", new Object[] { identifier, length, (currTime - startTime) }); String fileName = getFileName(identifier); AsyncUploadCacheResult result = null; synchronized (this) { usesIdentifier(identifier);
LOG.info("path=[{}], tmpPath=[{}]", path, tmpDir.getAbsolutePath()); directory = new File(path); mkdirs(directory); mkdirs(new File(homeDir)); if (!mkdirs(tmpDir)) { FileUtils.cleanDirectory(tmpDir); LOG.info("tmp=[{}] cleaned.", tmpDir.getPath()); asyncWriteCacheInitStatus = false; backend = createBackend(); backend.init(this, path, config); String markerFileName = getMarkerFile(); if (markerFileName != null && !"".equals(markerFileName.trim())) { if (!markerFile.exists()) { LOG.info("load files from local cache"); uploadFilesFromCache(); try { markerFile.createNewFile(); LOG.info("Initialization of asyncWriteCache failed. " + "Re-loading all files from local cache"); uploadFilesFromCache(); asyncWriteCache.reset();
public DataRecord getRecordIfStored(DataIdentifier identifier) throws DataStoreException { String fileName = getFileName(identifier); try { if (asyncWriteCache.hasEntry(fileName, minModifiedDate > 0)) { "getRecordIfStored: [{}] retrieved from asyncuploadmap", identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } else if (recLenCache.containsKey(identifier)) { identifier); if (minModifiedDate > 0) { touchInternal(identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } else { recLenCache.put(identifier, length); if (minModifiedDate > 0) { touchInternal(identifier); usesIdentifier(identifier); return new CachingDataRecord(this, identifier); } catch (DataStoreException ignore) {
/** * Derive file name from identifier. */ private static String getFileName(DataIdentifier identifier) { String name = identifier.toString(); return getFileName(name); }