public RemoteTransaction(Config config, TransferManager transferManager) { this(config, transferManager, new TransactionTO(config.getMachineName())); }
private NormalizedPath findConflictFilename(NormalizedPath conflictingPath) throws Exception { String conflictUserName = (config.getDisplayName() != null) ? config.getDisplayName() : config.getMachineName(); boolean conflictUserNameEndsWithS = conflictUserName.endsWith("s"); String conflictDate = new SimpleDateFormat("d MMM yy, h-mm a").format(new Date()); String conflictFilenameSuffix; if (conflictUserNameEndsWithS) { conflictFilenameSuffix = String.format("%s' conflicted copy, %s", conflictUserName, conflictDate); } else { conflictFilenameSuffix = String.format("%s's conflicted copy, %s", conflictUserName, conflictDate); } return conflictingPath.withSuffix(conflictFilenameSuffix, false); }
protected boolean otherRemoteOperationsRunning(String... operationIdentifiers) throws StorageException { logger.log(Level.INFO, "Looking for other running remote operations ..."); Map<String, ActionRemoteFile> actionRemoteFiles = transferManager.list(ActionRemoteFile.class); boolean otherRemoteOperationsRunning = false; List<String> disallowedOperationIdentifiers = Arrays.asList(operationIdentifiers); for (ActionRemoteFile actionRemoteFile : actionRemoteFiles.values()) { String operationName = actionRemoteFile.getOperationName(); String machineName = actionRemoteFile.getClientName(); boolean isOwnActionFile = machineName.equals(config.getMachineName()); boolean isOperationAllowed = !disallowedOperationIdentifiers.contains(operationName); boolean isOutdatedActionFile = isOutdatedActionFile(actionRemoteFile); if (!isOwnActionFile) { if (!isOutdatedActionFile) { if (isOperationAllowed) { logger.log(Level.INFO, "- Action file from other client, but allowed operation; not marking running; " + actionRemoteFile); } else { logger.log(Level.INFO, "- Action file from other client; --> marking operations running (!); " + actionRemoteFile); otherRemoteOperationsRunning = true; } } else { logger.log(Level.INFO, "- Action file outdated; ignoring " + actionRemoteFile); } } } return otherRemoteOperationsRunning; }
@Override public LogOperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Log' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------"); ArrayList<LightweightDatabaseVersion> databaseVersions = new ArrayList<>(); Iterator<DatabaseVersion> databaseVersionsIterator = localDatabase.getLastDatabaseVersions(options.getMaxDatabaseVersionCount(), options.getStartDatabaseVersionIndex(), options.getMaxFileHistoryCount()); while (databaseVersionsIterator.hasNext()) { DatabaseVersion databaseVersion = databaseVersionsIterator.next(); LightweightDatabaseVersion lightweightDatabaseVersion = createLightweightDatabaseVersion(databaseVersion); databaseVersions.add(lightweightDatabaseVersion); } return new LogOperationResult(databaseVersions); }
private void cleanActionFiles() throws StorageException { logger.log(Level.INFO, "Cleaning own old action files ..."); Map<String, ActionRemoteFile> actionRemoteFiles = transferManager.list(ActionRemoteFile.class); for (ActionRemoteFile actionRemoteFile : actionRemoteFiles.values()) { String machineName = actionRemoteFile.getClientName(); boolean isOwnActionFile = machineName.equals(config.getMachineName()); boolean isOutdatedActionFile = isOutdatedActionFile(actionRemoteFile); if (isOwnActionFile) { logger.log(Level.INFO, "- Deleting own action file " + actionRemoteFile + " ..."); transferManager.delete(actionRemoteFile); } else if (isOutdatedActionFile) { logger.log(Level.INFO, "- Action file from other client is OUTDATED; deleting " + actionRemoteFile + " ..."); transferManager.delete(actionRemoteFile); } else { logger.log(Level.INFO, "- Action file is current; ignoring " + actionRemoteFile + " ..."); } } }
/** * This function returns a list of all remote transaction files that belong to the client. If blocking transactions exist, * this methods returns null, because we are not allowed to proceed. */ public List<TransactionRemoteFile> getTransactionsByClient(String client) throws StorageException { Objects.requireNonNull(config, "Cannot get transactions if config is null."); Map<TransactionTO, TransactionRemoteFile> transactions = retrieveRemoteTransactions(); List<TransactionRemoteFile> transactionsByClient = new ArrayList<TransactionRemoteFile>(); for (TransactionTO potentiallyResumableTransaction : transactions.keySet()) { boolean isCancelledOwnTransaction = potentiallyResumableTransaction.getMachineName().equals(config.getMachineName()); if (isCancelledOwnTransaction) { transactionsByClient.add(transactions.get(potentiallyResumableTransaction)); } else { // Check for blocking transactions for (ActionTO action : potentiallyResumableTransaction.getActions()) { if (action.getType().equals(ActionType.DELETE)) { return null; } } } } return transactionsByClient; }
private TransactionRemoteFile attemptResumeTransactionRemoteFile() throws StorageException, BlockingTransfersException { TransactionRemoteFile transactionRemoteFile = null; // They look for the matching transaction on the remote. List<TransactionRemoteFile> transactions = transferManager.getTransactionsByClient(config.getMachineName()); // If there are blocking transactions, they stop completely. // Not sure yet what these blocking structures are. if (transactions == null) { // We have blocking transactions stopBecauseOfBlockingTransactions(); throw new BlockingTransfersException(); } // There is no sign of the transaction on the remote. Clean up the local transaction. if (transactions.size() != 1) { logger.log(Level.INFO, "Unable to find (unique) transactionRemoteFile. Not resuming."); transferManager.clearResumableTransactions(); } // Remote transaction file found. else { transactionRemoteFile = transactions.get(0); } return transactionRemoteFile; }
@Override public LsRemoteOperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Remote Status' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------"); eventBus.post(new LsRemoteStartSyncExternalEvent(config.getLocalDir().getAbsolutePath())); TransferManager transferManager = createTransferManager(loadedTransferManager); List<DatabaseRemoteFile> knownDatabases = localDatabase.getKnownDatabases(); List<DatabaseRemoteFile> unknownRemoteDatabases = listUnknownRemoteDatabases(transferManager, knownDatabases); transferManager.disconnect(); boolean hasChanges = unknownRemoteDatabases.size() > 0; eventBus.post(new LsRemoteEndSyncExternalEvent(config.getLocalDir().getAbsolutePath(), hasChanges)); return new LsRemoteOperationResult(new ArrayList<>(unknownRemoteDatabases)); }
localDatabase.markDatabaseVersionDirty(databaseVersionHeader.getVectorClock()); boolean isOwnDatabaseVersionHeader = config.getMachineName().equals(databaseVersionHeader.getClient()); String remoteFileToPruneClientName = config.getMachineName(); long remoteFileToPruneVersion = databaseVersionHeader.getVectorClock().getClock(config.getMachineName()); DatabaseRemoteFile remoteFileToPrune = new DatabaseRemoteFile(remoteFileToPruneClientName, remoteFileToPruneVersion);
boolean isCancelledOwnTransaction = potentiallyCancelledTransaction.getMachineName().equals(config.getMachineName());
/** * This methods takes a Map containing DatabaseVersions (headers only) and loads these headers into {@link DatabaseBranches}. * In addition, the local branch is added to this. The resulting DatabaseBranches will contain all headers exactly once, * for the client that created that version. * * @param localBranch {@link DatabaseBranch} containing the locally known headers. * @param remoteDatabaseHeaders Map from {@link DatabaseRemoteFile}s (important for client names) to the {@link DatabaseVersion}s that are * contained in these files. * * @return DatabaseBranches filled with all the headers that originated from either of the parameters. */ private DatabaseBranches populateDatabaseBranches(DatabaseBranch localBranch, SortedMap<DatabaseRemoteFile, List<DatabaseVersion>> remoteDatabaseHeaders) { DatabaseBranches allBranches = new DatabaseBranches(); allBranches.put(config.getMachineName(), localBranch.clone()); for (DatabaseRemoteFile remoteDatabaseFile : remoteDatabaseHeaders.keySet()) { // Populate branches DatabaseBranch remoteClientBranch = allBranches.getBranch(remoteDatabaseFile.getClientName(), true); for (DatabaseVersion remoteDatabaseVersion : remoteDatabaseHeaders.get(remoteDatabaseFile)) { DatabaseVersionHeader header = remoteDatabaseVersion.getHeader(); remoteClientBranch.add(header); } } logger.log(Level.INFO, "Populated unknown branches: " + allBranches); return allBranches; }
Long lastLocalValue = lastVectorClock.getClock(config.getMachineName()); Long lastDirtyLocalValue = localDatabase.getMaxDirtyVectorClock(config.getMachineName()); newVectorClock.setClock(config.getMachineName(), newLocalValue);
@Override public StatusOperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Status' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------"); if (options != null && options.isForceChecksum()) { logger.log(Level.INFO, "Force checksum ENABLED."); } if (options != null && !options.isDelete()) { logger.log(Level.INFO, "Delete missing files DISABLED."); } // Get local database logger.log(Level.INFO, "Querying current file tree from database ..."); eventBus.post(new StatusStartSyncExternalEvent(config.getLocalDir().getAbsolutePath())); // Path to actual file version final Map<String, FileVersion> filesInDatabase = localDatabase.getCurrentFileTree(); // Find local changes logger.log(Level.INFO, "Analyzing local folder " + config.getLocalDir() + " ..."); ChangeSet localChanges = findLocalChanges(filesInDatabase); if (!localChanges.hasChanges()) { logger.log(Level.INFO, "- No changes to local database"); } // Return result StatusOperationResult statusResult = new StatusOperationResult(); statusResult.setChangeSet(localChanges); eventBus.post(new StatusEndSyncExternalEvent(config.getLocalDir().getAbsolutePath(), localChanges.hasChanges())); return statusResult; }
@Override public LsOperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Ls' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------"); String pathExpression = parsePathExpression(options.getPathExpression(), options.isFileHistoryId()); Set<FileType> fileTypes = options.getFileTypes(); List<FileVersion> fileList = localDatabase.getFileList(pathExpression, options.getDate(), options.isFileHistoryId(), options.isRecursive(), options.isDeleted(), fileTypes); Map<FileHistoryId, PartialFileHistory> fileHistories = null; if (options.isFetchHistories()) { fileHistories = fetchFileHistories(fileList); } return new LsOperationResult(fileList, fileHistories); }
long newestLocalDatabaseVersion = getNewestDatabaseFileVersion(config.getMachineName(), localDatabase.getKnownDatabases()); DatabaseRemoteFile remoteDeltaDatabaseFile = new DatabaseRemoteFile(config.getMachineName(), newestLocalDatabaseVersion + 1); File localDeltaDatabaseFile = config.getCache().getDatabaseFile(remoteDeltaDatabaseFile.getName());
@Override public RestoreOperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Restore' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------");
public AbstractTransferOperation(Config config, String operationName) { super(config); this.eventBus = LocalEventBus.getInstance(); try { // Do NOT reuse TransferManager for action file renewal; see #140 TransferManager actionFileTransferManager = TransferManagerFactory .build(config) .withFeature(ReadAfterWriteConsistent.class) .withFeature(Retriable.class) .asDefault(); TransactionAwareFeatureTransferManager regularFileTransferManager = TransferManagerFactory .build(config) .withFeature(ReadAfterWriteConsistent.class) .withFeature(Retriable.class) .withFeature(PathAware.class) .withFeature(TransactionAware.class) .as(TransactionAware.class); this.actionHandler = new ActionFileHandler(actionFileTransferManager, operationName, config.getMachineName()); this.transferManager = regularFileTransferManager; } catch (StorageException e) { logger.log(Level.SEVERE, "Unable to create AbstractTransferOperation: Unable to create TransferManager", e); throw new RuntimeException("Unable to create AbstractTransferOperation: Unable to create TransferManager: " + e.getMessage()); } }
@Override public UpOperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Sync up' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------");
@Test public void testConfigHelperLoadConfigTO() throws Exception { // Setup Config testConfig = TestConfigUtil.createTestLocalConfig(); // Run ConfigTO loadedConfigTO = ConfigHelper.loadConfigTO(testConfig.getLocalDir()); // Test assertNotNull(loadedConfigTO); assertEquals(testConfig.getDisplayName(), loadedConfigTO.getDisplayName()); assertEquals(testConfig.getMachineName(), loadedConfigTO.getMachineName()); assertEquals(testConfig.getMasterKey(), loadedConfigTO.getMasterKey()); // Tear down TestConfigUtil.deleteTestLocalConfigAndData(testConfig); }
@Test public void testConfigHelperLoadConfig() throws Exception { // Setup Config testConfig = TestConfigUtil.createTestLocalConfig(); // Run Config loadedConfig = ConfigHelper.loadConfig(testConfig.getLocalDir()); // Test assertNotNull(loadedConfig); assertEquals(testConfig.getAppDir(), loadedConfig.getAppDir()); assertEquals(testConfig.getCacheDir(), loadedConfig.getCacheDir()); assertNotNull(loadedConfig.getChunker()); assertEquals(testConfig.getChunker().toString(), loadedConfig.getChunker().toString()); assertNotNull(loadedConfig.getCache()); assertNotNull(loadedConfig.getConnection()); assertEquals(testConfig.getDatabaseDir(), loadedConfig.getDatabaseDir()); assertEquals(testConfig.getDatabaseFile(), loadedConfig.getDatabaseFile()); assertEquals(testConfig.getDisplayName(), loadedConfig.getDisplayName()); assertEquals(testConfig.getLocalDir(), loadedConfig.getLocalDir()); assertEquals(testConfig.getLogDir(), loadedConfig.getLogDir()); assertEquals(testConfig.getMachineName(), loadedConfig.getMachineName()); assertEquals(testConfig.getMasterKey(), loadedConfig.getMasterKey()); assertNotNull(loadedConfig.getMultiChunker()); assertNotNull(loadedConfig.getRepoId()); assertNotNull(loadedConfig.getTransformer()); // Tear down TestConfigUtil.deleteTestLocalConfigAndData(testConfig); }