@VisibleForTesting protected SequencedItemList<Operation> createInMemoryLog() { return new SequencedItemList<>(); }
@Override public boolean add(Operation item) { if (this.addCount.incrementAndGet() == this.corruptAtIndex) { // Still add the item, but report that we haven't added it. return false; } return super.add(item); } }
private void triggerTailReads() { this.executor.execute(() -> { // Gather all the eligible tail reads. List<TailRead> toTrigger; synchronized (this.tailReads) { Operation lastOp = this.inMemoryOperationLog.getLast(); if (lastOp != null) { long seqNo = lastOp.getSequenceNumber(); toTrigger = this.tailReads.stream().filter(e -> e.afterSequenceNumber < seqNo).collect(Collectors.toList()); } else { toTrigger = Collections.emptyList(); } } // Trigger all of them (no need to unregister them; the unregister handle is already wired up). for (TailRead tr : toTrigger) { tr.future.complete(Futures.runOrFail( () -> this.inMemoryOperationLog.read(tr.afterSequenceNumber, tr.maxCount), tr.future)); } }); }
/** * Tests the functionality of the truncate() method. */ @Test public void testTruncate() { SequencedItemList<Item> list = new SequencedItemList<>(); for (int i = 0; i < ITEM_COUNT; i++) { list.add(new Item(i)); } // Truncate 25% of items. list.truncate(ITEM_COUNT / 4 - 1); checkRange("Truncate 25%", ITEM_COUNT / 4, ITEM_COUNT - 1, list.read(START, ITEM_COUNT)); // Truncate the same 25% of items - verify no change. list.truncate(ITEM_COUNT / 4 - 1); checkRange("Re-truncate 25%", ITEM_COUNT / 4, ITEM_COUNT - 1, list.read(START, ITEM_COUNT)); // Truncate all items. list.truncate(END); Iterator<Item> readResult = list.read(START, ITEM_COUNT * 2); Assert.assertFalse("List should be empty.", readResult.hasNext()); }
/** * Tests the functionality of the clear() method. */ @Test public void testClear() { SequencedItemList<Item> list = new SequencedItemList<>(); for (int i = 0; i < ITEM_COUNT; i++) { list.add(new Item(i)); } list.clear(); Iterator<Item> readResult = list.read(START, ITEM_COUNT * 2); Assert.assertFalse("List should be empty.", readResult.hasNext()); }
/** * Tests the combination of the basic append() method and read(). */ @Test public void testAddRead() { SequencedItemList<Item> list = new SequencedItemList<>(); for (int i = 0; i < ITEM_COUNT; i++) { list.add(new Item(i)); } //Read 1/2 items Iterator<Item> readResult = list.read(START, ITEM_COUNT / 2); checkRange("Read first 50%", 0, ITEM_COUNT / 2 - 1, readResult); // Read all items readResult = list.read(START, ITEM_COUNT); checkRange("Read all items", 0, ITEM_COUNT - 1, readResult); // Try to read more items. readResult = list.read(START, ITEM_COUNT * 2); checkRange("Read more items than list has", 0, ITEM_COUNT - 1, readResult); // Read 25% of items, starting at the middle point. readResult = list.read(ITEM_COUNT / 2 - 1, ITEM_COUNT / 4); checkRange("Read 25% starting at 50%", ITEM_COUNT / 2, ITEM_COUNT / 2 + ITEM_COUNT / 4 - 1, readResult); }
SequencedItemList<Operation> opLog = new SequencedItemList<>(); ArrayList<TestReadIndex.MethodInvocation> methodInvocations = new ArrayList<>(); TestReadIndex readIndex = new TestReadIndex(methodInvocations::add); Iterator<Operation> logIterator = opLog.read(-1, operations.size()); int currentIndex = -1; int currentReadIndex = -1;
if (this.ackEffective.get()) { this.log.truncate(upToSequenceNumber); this.metadata.removeTruncationMarkers(upToSequenceNumber); synchronized (this.lock) { Operation last = this.log.getLast(); if (this.waitFullyAcked != null && (last == null || last.getSequenceNumber() <= upToSequenceNumber)) { callback = this.waitFullyAcked;
Iterator<Entry> iterator() { ensureEnabled(); return this.entries.read(Long.MIN_VALUE, Integer.MAX_VALUE); }
void truncate(long upToSequence, String clientId) throws DataLogWriterNotPrimaryException { ensureLock(clientId); ensureEnabled(); this.entries.truncate(upToSequence); }
Entry getLast() { return this.entries.getLast(); }
/** * Clears the list. */ public void clear() { synchronized (this.lock) { // Mark every node as truncated. while (this.head != null) { this.head = trim(this.head); } this.tail = null; } }
SequencedItemList<Item> list = new SequencedItemList<>(); for (int i = 0; i < ITEM_COUNT; i++) { list.add(new Item(i)); Iterator<Item> addIterator = list.read(START, ITEM_COUNT * 2); Item firstValue = addIterator.next(); Assert.assertEquals("Unexpected first value, pre-modification.", 0, firstValue.getSequenceNumber()); list.add(new Item(ITEM_COUNT)); checkRange("Post-addition.", 1, ITEM_COUNT, addIterator); Iterator<Item> truncateIterator = list.read(START, ITEM_COUNT * 2); list.truncate(10); Assert.assertFalse("Unexpected value from hasNext when list has been truncated.", truncateIterator.hasNext()); AssertExtensions.assertThrows( Iterator<Item> midTruncateIterator = list.read(START, ITEM_COUNT * 2); Assert.assertTrue("Unexpected value from hasNext when not been truncated.", midTruncateIterator.hasNext()); list.truncate(20); AssertExtensions.assertThrows( "Unexpected behavior from next() when current element has been truncated (after hasNext() and before next()).",
/** * Tests the functionality of the addIf() method. */ @Test public void testAddIf() { SequencedItemList<Item> list = new SequencedItemList<>(); for (int i = 0; i < ITEM_COUNT; i++) { final Item currentValue = new Item(i); // Happy case. boolean resultValue = list.add(currentValue); Assert.assertTrue("Unexpected return value from addIf for successful append.", resultValue); // Unhappy case resultValue = list.add(currentValue); Assert.assertFalse("Unexpected return value from addIf for unsuccessful append.", resultValue); } Iterator<Item> readResult = list.read(START, ITEM_COUNT * 2); checkRange("AddIf", 0, ITEM_COUNT - 1, readResult); }
@Override public CompletableFuture<Iterator<Operation>> read(long afterSequenceNumber, int maxCount, Duration timeout) { Exceptions.checkNotClosed(this.closed.get(), this); ErrorInjector<Exception> asyncErrorInjector; synchronized (this.lock) { ErrorInjector.throwSyncExceptionIfNeeded(this.readSyncErrorInjector); asyncErrorInjector = this.readAsyncErrorInjector; } return ErrorInjector .throwAsyncExceptionIfNeeded(asyncErrorInjector, () -> { Iterator<Operation> logReadResult = this.log.read(afterSequenceNumber, maxCount); if (logReadResult.hasNext()) { // Result is readily available; return it. return CompletableFuture.completedFuture(logReadResult); } else { // Result is not yet available; wait for an add and then retry the read. return waitForAdd(afterSequenceNumber, timeout) .thenComposeAsync(v1 -> this.read(afterSequenceNumber, maxCount, timeout), this.executor); } }); }
@Override public CompletableFuture<Void> truncate(long upToSequenceNumber, Duration timeout) { ensureRunning(); Preconditions.checkArgument(this.metadata.isValidTruncationPoint(upToSequenceNumber), "Invalid Truncation Point. Must refer to a MetadataCheckpointOperation."); // The SequenceNumber we were given points directly to a MetadataCheckpointOperation. We must not remove it! // Instead, it must be the first operation that does survive, so we need to adjust our SeqNo to the one just // before it. long actualTruncationSequenceNumber = upToSequenceNumber - 1; // Find the closest Truncation Marker (that does not exceed it). LogAddress truncationFrameAddress = this.metadata.getClosestTruncationMarker(actualTruncationSequenceNumber); if (truncationFrameAddress == null) { // Nothing to truncate. return CompletableFuture.completedFuture(null); } TimeoutTimer timer = new TimeoutTimer(timeout); log.info("{}: Truncate (OperationSequenceNumber = {}, DataFrameAddress = {}).", this.traceObjectId, upToSequenceNumber, truncationFrameAddress); // Before we do any real truncation, we need to mini-snapshot the metadata with only those fields that are updated // asynchronously for us (i.e., not via normal Log Operations) such as the Storage State. That ensures that this // info will be readily available upon recovery without delay. return add(new StorageMetadataCheckpointOperation(), timer.getRemaining()) .thenComposeAsync(v -> this.durableDataLog.truncate(truncationFrameAddress, timer.getRemaining()), this.executor) .thenRunAsync(() -> { // Truncate InMemory Transaction Log. int count = this.inMemoryOperationLog.truncate(actualTruncationSequenceNumber); // Remove old truncation markers. this.metadata.removeTruncationMarkers(actualTruncationSequenceNumber); this.operationProcessor.getMetrics().operationLogTruncate(count); }, this.executor); }
/** * Returns a CompletableFuture that will be completed when the TestWriterDataSource becomes empty. */ CompletableFuture<Void> waitFullyAcked() { synchronized (this.lock) { if (this.waitFullyAcked == null) { // Nobody else is waiting for the DataSource to empty out. if (this.log.getLast() == null) { // We are already empty; return a completed future. return CompletableFuture.completedFuture(null); } else { // Not empty yet; create an uncompleted Future and store it. this.waitFullyAcked = new CompletableFuture<>(); } } return this.waitFullyAcked; } }
/** * Truncates items from the beginning of the list up to, and including, the element with the given Sequence Number. * * @param upToSequenceNumber The Sequence Number to truncate up to. * @return The number of truncated items. */ public int truncate(long upToSequenceNumber) { int count = 0; synchronized (this.lock) { // We truncate by finding the new head and simply pointing our head reference to it, as well as disconnecting // its predecessor node from it. We also need to mark every truncated node as such - this will instruct ongoing // reads to stop serving truncated data. while (this.head != null && this.head.item.getSequenceNumber() <= upToSequenceNumber) { this.head = trim(this.head); count++; } if (this.head == null) { this.tail = null; } } return count; }
ensureRunning(); log.debug("{}: Read (AfterSequenceNumber = {}, MaxCount = {}).", this.traceObjectId, afterSequenceNumber, maxCount); Iterator<Operation> logReadResult = this.inMemoryOperationLog.read(afterSequenceNumber, maxCount); if (logReadResult.hasNext()) { Operation lastOp; synchronized (this.tailReads) { lastOp = this.inMemoryOperationLog.getLast(); if (lastOp == null || lastOp.getSequenceNumber() <= afterSequenceNumber) { logReadResult = this.inMemoryOperationLog.read(afterSequenceNumber, maxCount); assert logReadResult.hasNext() : String.format("Unable to read anything after SeqNo %d, even though last operation SeqNo == %d",
EntryCollection(int maxAppendSize) { this.entries = new SequencedItemList<>(); this.writeLock = new AtomicReference<>(); this.epoch = new AtomicLong(); this.maxAppendSize = maxAppendSize; this.enabled = new AtomicBoolean(true); }