/** * Builds a {@link WaveletDeltaRecord} and applies it to the wavelet container. * The delta must be non-empty. */ protected WaveletDeltaRecord applyDelta( ByteStringMessage<ProtocolAppliedWaveletDelta> appliedDelta, WaveletDelta transformed) throws InvalidProtocolBufferException, OperationException { TransformedWaveletDelta transformedDelta = AppliedDeltaUtil.buildTransformedDelta(appliedDelta, transformed); WaveletDeltaRecord deltaRecord = new WaveletDeltaRecord(transformed.getTargetVersion(), appliedDelta, transformedDelta); waveletState.appendDelta(deltaRecord); return deltaRecord; }
public static DBObject serialize(WaveletDeltaRecord waveletDelta, String waveId, String waveletId) { BasicDBObject mongoWaveletDelta = new BasicDBObject(); mongoWaveletDelta.append(FIELD_WAVE_ID, waveId); mongoWaveletDelta.append(FIELD_WAVELET_ID, waveletId); mongoWaveletDelta.append(FIELD_APPLIEDATVERSION, serialize(waveletDelta.getAppliedAtVersion())); mongoWaveletDelta.append(FIELD_APPLIED, waveletDelta.getAppliedDelta().getByteArray()); mongoWaveletDelta.append(FIELD_TRANSFORMED, serialize(waveletDelta.getTransformedDelta())); return mongoWaveletDelta; }
/** * Reads the last complete record in the deltas file and truncates any trailing junk. */ private void initializeEndVersionAndTruncateTrailingJunk() throws IOException { long numRecords = index.length(); if (numRecords >= 1) { endVersion = getDeltaByEndVersion(numRecords).getResultingVersion(); } else { endVersion = null; } // The file's position should be at the end. Truncate any // trailing junk such as from a partially completed write. file.setLength(file.getFilePointer()); } }
@Override public void appendDelta(WaveletDeltaRecord deltaRecord) throws OperationException { HashedVersion currentVersion = getCurrentVersion(); Preconditions.checkArgument(currentVersion.equals(deltaRecord.getAppliedAtVersion()), "Applied version %s doesn't match current version %s", deltaRecord.getAppliedAtVersion(), currentVersion); if (deltaRecord.getAppliedAtVersion().getVersion() == 0) { Preconditions.checkState(lastPersistedVersion.get() == null); snapshot = WaveletDataUtil.buildWaveletFromFirstDelta(getWaveletName(), deltaRecord.getTransformedDelta()); } else { WaveletDataUtil.applyWaveletDelta(deltaRecord.getTransformedDelta(), snapshot); } // Now that we built the snapshot without any exceptions, we record the delta. cachedDeltas.put(deltaRecord.getAppliedAtVersion(), deltaRecord); }
/** * Read a delta to the file. Does not move the file pointer before writing. Returns number of * bytes written. */ private long writeDelta(WaveletDeltaRecord delta) throws IOException { // We'll write zeros in place of the header and come back & write it at the end. long headerPointer = file.getFilePointer(); file.write(new byte[DeltaHeader.HEADER_LENGTH]); int appliedLength = writeAppliedDelta(delta.getAppliedDelta()); int transformedLength = writeTransformedWaveletDelta(delta.getTransformedDelta()); long endPointer = file.getFilePointer(); file.seek(headerPointer); writeDeltaHeader(new DeltaHeader(DELTA_PROTOCOL_VERSION, appliedLength, transformedLength)); file.seek(endPointer); return endPointer - headerPointer; }
@Override public TransformedWaveletDelta getTransformedDeltaByEndVersion(final HashedVersion endVersion) { Preconditions.checkArgument(endVersion.getVersion() > 0, "end version %s is not positive", endVersion); Entry<HashedVersion, WaveletDeltaRecord> transformedEntry = cachedDeltas.lowerEntry(endVersion); final WaveletDeltaRecord cachedDelta = transformedEntry != null ? transformedEntry.getValue() : null; if (snapshot == null) { return null; } else { WaveletDeltaRecord deltaRecord = getDeltaRecordByEndVersion(endVersion); TransformedWaveletDelta delta; if (deltaRecord == null && cachedDelta != null && cachedDelta.getResultingVersion().equals(endVersion)) { delta = cachedDelta.getTransformedDelta(); } else { delta = deltaRecord != null ? deltaRecord.getTransformedDelta() : null; } return delta; } }
public void testDeltasAccessibleByBeginVersion() throws Exception { appendDeltas(d1, d2, d3); assertEquals(d1.getTransformedDelta(), target.getTransformedDelta(V0)); assertEquals(d1.getAppliedDelta(), target.getAppliedDelta(V0)); assertEquals(d2.getTransformedDelta(), target.getTransformedDelta(d1.getResultingVersion())); assertEquals(d2.getAppliedDelta(), target.getAppliedDelta(d1.getResultingVersion())); assertEquals(d3.getTransformedDelta(), target.getTransformedDelta(d2.getResultingVersion())); assertEquals(d3.getAppliedDelta(), target.getAppliedDelta(d2.getResultingVersion())); // Wrong hashes return null. assertNull(target.getTransformedDelta(HashedVersion.unsigned(0))); assertNull(target.getAppliedDelta(HashedVersion.unsigned(0))); }
public void testWriteToNewWavelet() throws Exception { Pair<DeltaStore,WaveletDeltaRecord> pair = newDeltaStoreWithRecord(WAVE1_WAVELET1); DeltaStore store = pair.first; WaveletDeltaRecord record = pair.second; DeltasAccess wavelet = store.open(WAVE1_WAVELET1); assertFalse(wavelet.isEmpty()); assertEquals(WAVE1_WAVELET1, wavelet.getWaveletName()); assertEquals(record.getResultingVersion(), wavelet.getEndVersion()); assertEquals(record, wavelet.getDelta(0)); assertEquals(record, wavelet.getDeltaByEndVersion(record.getResultingVersion().getVersion())); assertEquals(record.getAppliedAtVersion(), wavelet.getAppliedAtVersion(0)); assertEquals(record.getAppliedDelta(), wavelet.getAppliedDelta(0)); assertEquals(record.getTransformedDelta(), wavelet.getTransformedDelta(0)); wavelet.close(); }
@Override public WaveletDeltaRecord submitRequest(WaveletName waveletName, ProtocolSignedDelta signedDelta) throws OperationException, InvalidProtocolBufferException, InvalidHashException, PersistenceException, WaveletStateException { awaitLoad(); acquireWriteLock(); try { checkStateOk(); HashedVersion before = getCurrentVersion(); WaveletDeltaRecord result = transformAndApplyLocalDelta(signedDelta); HashedVersion after = getCurrentVersion(); // Only publish and persist the delta if it wasn't transformed away // (right now it never is since the current OT algorithm doesn't transform ops away) // and wasn't a duplicate of a previously applied delta. if (!after.equals(before)) { Preconditions.checkState(!result.isEmpty()); Preconditions.checkState(result.getAppliedAtVersion().equals(before)); ImmutableSet<String> domainsToNotify = domainsOf(Iterables.concat( accessSnapshot().getParticipants(), participantsRemovedBy(result.getTransformedDelta()))); notifyOfDeltas(ImmutableList.of(result), domainsToNotify); // We always persist a local delta immediately after it's applied // and after it's broadcast on the wave bus and to remote servers. persist(result.getResultingVersion(), domainsToNotify); } return result; } finally { releaseWriteLock(); } }
@Override public TransformedWaveletDelta apply(WaveletDeltaRecord record) { return record.getTransformedDelta(); } };
private static void readDeltasInRange(WaveletDeltaRecordReader reader, ConcurrentNavigableMap<HashedVersion, WaveletDeltaRecord> cachedDeltas, HashedVersion startVersion, HashedVersion endVersion, Receiver<WaveletDeltaRecord> receiver) throws IOException { WaveletDeltaRecord delta = getDelta(reader, cachedDeltas, startVersion); Preconditions.checkArgument(delta != null && delta.getAppliedAtVersion().equals(startVersion), "invalid start version"); for (;;) { if (!receiver.put(delta)) { return; } if (delta.getResultingVersion().getVersion() >= endVersion.getVersion()) { break; } delta = getDelta(reader, cachedDeltas, delta.getResultingVersion()); if (delta == null) { break; } } Preconditions.checkArgument(delta != null && delta.getResultingVersion().equals(endVersion), "invalid end version"); }
@Override public ByteStringMessage<ProtocolAppliedWaveletDelta> getAppliedDelta(long version) { WaveletDeltaRecord delta = getDelta(version); return (delta != null) ? delta.getAppliedDelta() : null; }
@Override public HashedVersion getAppliedAtVersion(long version) throws InvalidProtocolBufferException { WaveletDeltaRecord delta = getDelta(version); return (delta != null) ? delta.getAppliedAtVersion() : null; }
public void testDeltasAccessibleByBeginVersion() throws Exception { appendDeltas(d1, d2, d3); assertEquals(d1.getTransformedDelta(), target.getTransformedDelta(V0)); assertEquals(d1.getAppliedDelta(), target.getAppliedDelta(V0)); assertEquals(d2.getTransformedDelta(), target.getTransformedDelta(d1.getResultingVersion())); assertEquals(d2.getAppliedDelta(), target.getAppliedDelta(d1.getResultingVersion())); assertEquals(d3.getTransformedDelta(), target.getTransformedDelta(d2.getResultingVersion())); assertEquals(d3.getAppliedDelta(), target.getAppliedDelta(d2.getResultingVersion())); // Wrong hashes return null. assertNull(target.getTransformedDelta(HashedVersion.unsigned(0))); assertNull(target.getAppliedDelta(HashedVersion.unsigned(0))); }
public void testWriteToNewWavelet() throws Exception { Pair<DeltaStore,WaveletDeltaRecord> pair = newDeltaStoreWithRecord(WAVE1_WAVELET1); DeltaStore store = pair.first; WaveletDeltaRecord record = pair.second; DeltasAccess wavelet = store.open(WAVE1_WAVELET1); assertFalse(wavelet.isEmpty()); assertEquals(WAVE1_WAVELET1, wavelet.getWaveletName()); assertEquals(record.getResultingVersion(), wavelet.getEndVersion()); assertEquals(record, wavelet.getDelta(0)); assertEquals(record, wavelet.getDeltaByEndVersion(record.getResultingVersion().getVersion())); assertEquals(record.getAppliedAtVersion(), wavelet.getAppliedAtVersion(0)); assertEquals(record.getAppliedDelta(), wavelet.getAppliedDelta(0)); assertEquals(record.getTransformedDelta(), wavelet.getTransformedDelta(0)); wavelet.close(); }
/** Picks out the transformed deltas from a list of delta records. */ private static ImmutableList<TransformedWaveletDelta> transformedDeltasOf( Iterable<WaveletDeltaRecord> deltaRecords) { ImmutableList.Builder<TransformedWaveletDelta> transformedDeltas = ImmutableList.builder(); for (WaveletDeltaRecord deltaRecord : deltaRecords) { transformedDeltas.add(deltaRecord.getTransformedDelta()); } return transformedDeltas.build(); }
@Override public boolean put(WaveletDeltaRecord delta) { return receiver.put(delta.getAppliedDelta()); } });
@Override public HashedVersion getHashedVersion(long version) { final Entry<HashedVersion, WaveletDeltaRecord> cachedEntry = lookupCached(cachedDeltas, version); if (version == 0) { return versionZero; } else if (snapshot == null) { return null; } else if (version == snapshot.getVersion()) { return snapshot.getHashedVersion(); } else { WaveletDeltaRecord delta; try { delta = lookup(version); } catch (IOException e) { throw new RuntimeIOException(new IOException(format("Version : %d", version), e)); } if (delta == null && cachedEntry != null) { return cachedEntry.getKey(); } else { return delta != null ? delta.getAppliedAtVersion() : null; } } }
public void testDeltasAccesssibleByEndVersion() throws Exception { appendDeltas(d1, d2, d3); for (WaveletDeltaRecord d : Arrays.asList(d1, d2, d3)) { assertEquals(d.getTransformedDelta(), target.getTransformedDeltaByEndVersion(d.getResultingVersion())); assertEquals(d.getAppliedDelta(), target.getAppliedDeltaByEndVersion(d.getResultingVersion())); } // Wrong hashes return null. assertNull(target.getTransformedDeltaByEndVersion( HashedVersion.unsigned(d1.getResultingVersion().getVersion()))); assertNull(target.getAppliedDeltaByEndVersion( HashedVersion.unsigned(d1.getResultingVersion().getVersion()))); }
private void checkHistoryForDeltasWithInterrupt(final int interruptIndex, WaveletDeltaRecord... deltas) { HashedVersion beginVersion = deltas[0].getAppliedAtVersion(); HashedVersion endVersion = deltas[interruptIndex].getTransformedDelta().getResultingVersion(); expected.add(deltas[i].getTransformedDelta()); Lists.newArrayListWithExpectedSize(interruptIndex+1); for (int i=0; i <= interruptIndex; i++) { expected.add(deltas[i].getAppliedDelta());