public MemoryDatabase() { databaseVersions = new ArrayList<DatabaseVersion>(); // Caches fullDatabaseVersionCache = new DatabaseVersion(); filenameHistoryCache = new HashMap<String, PartialFileHistory>(); databaseVersionIdCache = new HashMap<VectorClock, DatabaseVersion>(); contentChecksumFileHistoriesCache = new HashMap<FileChecksum, List<PartialFileHistory>>(); }
private void indexWithoutNewFiles(List<File> files, List<File> deletedFiles, Queue<DatabaseVersion> databaseVersionQueue) { DatabaseVersion newDatabaseVersion = new DatabaseVersion(); removeDeletedFiles(newDatabaseVersion, deletedFiles); logger.log(Level.FINE, "Added database version with only deletions: " + newDatabaseVersion); databaseVersionQueue.offer(newDatabaseVersion); }
private void updateFullDatabaseVersionCache() { fullDatabaseVersionCache = new DatabaseVersion(); for (DatabaseVersion databaseVersion : databaseVersions) { updateFullDatabaseVersionCache(databaseVersion); } }
@Override public void run() { try { logger.log(Level.INFO, "Starting Indexing."); indexer.index(files, deletedFiles, databaseVersionQueue); } catch (IOException e) { // TODO: Store this exception as a "result"? e.printStackTrace(); } // Signal end-of-stream. logger.log(Level.INFO, "Stopping indexing. Signal end of stream with empty databaseversion"); databaseVersionQueue.offer(new DatabaseVersion()); }
private void prepareResume() throws Exception { Collection<Long> versionsToResume = transferManager.loadPendingTransactionList(); boolean hasVersionsToResume = versionsToResume != null && versionsToResume.size() > 0; if (hasVersionsToResume) { logger.log(Level.INFO, "Found local transaction to resume."); logger.log(Level.INFO, "Attempting to find transactionRemoteFile"); remoteTransactionsToResume = attemptResumeTransactions(versionsToResume); Collection<DatabaseVersion> remoteDatabaseVersionsToResume = attemptResumeDatabaseVersions(versionsToResume); resuming = remoteDatabaseVersionsToResume != null && remoteTransactionsToResume != null && remoteDatabaseVersionsToResume.size() == remoteTransactionsToResume.size(); if (resuming) { databaseVersionQueue.addAll(remoteDatabaseVersionsToResume); databaseVersionQueue.add(new DatabaseVersion()); // Empty database version is the stopping marker transactionRemoteFileToResume = attemptResumeTransactionRemoteFile(); } else { transferManager.clearResumableTransactions(); } } else { transferManager.clearResumableTransactions(); } }
private void indexWithNewFiles(List<File> files, List<File> deletedFiles, Queue<DatabaseVersion> databaseVersionQueue) throws IOException { boolean isFirstFile = true; int filesCount = files.size(); while (!files.isEmpty()) { DatabaseVersion newDatabaseVersion = new DatabaseVersion(); // Create the DeduperListener that will receive MultiChunks and store them in the DatabaseVersion object DeduperListener deduperListener = new IndexerDeduperListener(newDatabaseVersion); // Signal the start of indexing if we are about to deduplicate the first file if (isFirstFile) { deduperListener.onStart(files.size()); removeDeletedFiles(newDatabaseVersion, deletedFiles); // Add deletions in first database version isFirstFile = false; } // Find and index new files deduper.deduplicate(files, deduperListener); if (!newDatabaseVersion.getFileHistories().isEmpty()) { logger.log(Level.FINE, "Processed new database version: " + newDatabaseVersion); databaseVersionQueue.offer(newDatabaseVersion); int remainingFilesCount = filesCount - files.size(); eventBus.post(new UpIndexMidSyncExternalEvent(config.getLocalDir().toString(), filesCount, remainingFilesCount)); } //else { (comment-only else case) // Just chunks and multichunks, no filehistory. Since this means the file was being // written/vanished during operations, it makes no sense to upload it. If the user // wants it indexed, Up can be run again. //} } }
databaseVersion = new DatabaseVersion();
protected DatabaseVersion createDatabaseVersionFromRow(ResultSet resultSet, boolean excludeChunkData, int fileHistoryMaxCount) throws SQLException { DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet); DatabaseVersion databaseVersion = new DatabaseVersion(); databaseVersion.setHeader(databaseVersionHeader); // Add chunk/multichunk/filecontent data if (!excludeChunkData) { Map<ChunkChecksum, ChunkEntry> chunks = chunkDao.getChunks(databaseVersionHeader.getVectorClock()); Map<MultiChunkId, MultiChunkEntry> multiChunks = multiChunkDao.getMultiChunks(databaseVersionHeader.getVectorClock()); Map<FileChecksum, FileContent> fileContents = fileContentDao.getFileContents(databaseVersionHeader.getVectorClock()); for (ChunkEntry chunk : chunks.values()) { databaseVersion.addChunk(chunk); } for (MultiChunkEntry multiChunk : multiChunks.values()) { databaseVersion.addMultiChunk(multiChunk); } for (FileContent fileContent : fileContents.values()) { databaseVersion.addFileContent(fileContent); } } // Add file histories Map<FileHistoryId, PartialFileHistory> fileHistories = fileHistoryDao .getFileHistoriesWithFileVersions(databaseVersionHeader.getVectorClock(), fileHistoryMaxCount); for (PartialFileHistory fileHistory : fileHistories.values()) { databaseVersion.addFileHistory(fileHistory); } return databaseVersion; }
@Override public DatabaseVersion clone() { DatabaseVersion clonedDatabaseVersion = new DatabaseVersion(); clonedDatabaseVersion.setHeader(getHeader()); for (ChunkEntry chunkEntry : getChunks()) { clonedDatabaseVersion.addChunk(chunkEntry); } for (MultiChunkEntry multiChunkEntry : getMultiChunks()) { clonedDatabaseVersion.addMultiChunk(multiChunkEntry); } for (FileContent fileContent : getFileContents()) { clonedDatabaseVersion.addFileContent(fileContent); } for (PartialFileHistory fileHistory : getFileHistories()) { clonedDatabaseVersion.addFileHistory(fileHistory); } return clonedDatabaseVersion; }
public static DatabaseVersion createDatabaseVersion(DatabaseVersionHeader basedOnDatabaseVersionHeader, Date date) { VectorClock vectorClock = (basedOnDatabaseVersionHeader != null) ? basedOnDatabaseVersionHeader.getVectorClock().clone() : new VectorClock(); vectorClock.incrementClock("someclient"); DatabaseVersion databaseVersion = new DatabaseVersion(); databaseVersion.setClient("someclient"); databaseVersion.setTimestamp(date); databaseVersion.setVectorClock(vectorClock); return databaseVersion; } }