private SegmentHandle readHandle(String segmentName) { return InMemoryStorage.newHandle(segmentName, true); }
private SegmentHandle writeHandle(String segmentName) { return InMemoryStorage.newHandle(segmentName, false); }
@Override public CompletableFuture<SegmentHandle> openRead(String streamSegmentName) { return CompletableFuture.completedFuture(InMemoryStorage.newHandle(streamSegmentName, true)); }
context.storage.setWriteInterceptor((segmentName, offset, data, length, storage) -> { if (writeCount.incrementAndGet() % failWriteEvery == 0) { return storage.write(InMemoryStorage.newHandle(segmentName, false), offset, data, length, TIMEOUT) .thenRun(() -> { writeFailCount.incrementAndGet(); context.storage.setSealInterceptor((segmentName, storage) -> { if (sealCount.incrementAndGet() % failSealEvery == 0) { return storage.seal(InMemoryStorage.newHandle(segmentName, false), TIMEOUT) .thenRun(() -> { sealFailCount.incrementAndGet(); context.storage.setConcatInterceptor((targetSegment, offset, sourceSegment, storage) -> { if (mergeCount.incrementAndGet() % failMergeEvery == 0) { return storage.concat(InMemoryStorage.newHandle(targetSegment, false), offset, sourceSegment, TIMEOUT) .thenRun(() -> { mergeFailCount.incrementAndGet();
futures.add(s.concat(InMemoryStorage.newHandle(segment1, false), 0, segment2, TIMEOUT)); waitOn.get(createSegment1Key).release(); AssertExtensions.assertThrows( futures.add(s.write(InMemoryStorage.newHandle(segment1, false), 0, new ByteArrayInputStream(new byte[0]), 0, TIMEOUT)); futures.add(s.write(InMemoryStorage.newHandle(segment2, false), 0, new ByteArrayInputStream(new byte[0]), 0, TIMEOUT)); AssertExtensions.assertThrows( "Write(target) was invoked while concat was running",
futures.add(s.delete(InMemoryStorage.newHandle(segmentName, false), TIMEOUT)); AssertExtensions.assertThrows( "Second operation was invoked while the first one was still running.",
final String segmentName = "Segment"; final String concatSourceName = "Concat"; val handle = InMemoryStorage.newHandle(segmentName, false); AtomicReference<Object> toReturn = new AtomicReference<>(); AtomicReference<BiConsumer<String, String>> validator = new AtomicReference<>();
/** * Tests the flush() method when it has a StreamSegmentSealOperation but the Segment is already sealed in Storage. */ @Test public void testSealAlreadySealed() throws Exception { // Add some appends and seal, and then flush together. Verify that everything got flushed in one go. @Cleanup TestContext context = new TestContext(DEFAULT_CONFIG); context.storage.create(context.segmentAggregator.getMetadata().getName(), TIMEOUT).join(); context.segmentAggregator.initialize(TIMEOUT).join(); // Generate and add a Seal Operation. StorageOperation sealOp = generateSealAndUpdateMetadata(SEGMENT_ID, context); context.segmentAggregator.add(sealOp); // Seal the segment in Storage, behind the scenes. context.storage.seal(InMemoryStorage.newHandle(context.segmentAggregator.getMetadata().getName(), false), TIMEOUT).join(); // Call flush and verify no exception is thrown. context.segmentAggregator.flush(TIMEOUT).join(); // Verify data - even though already sealed, make sure the metadata is updated accordingly. Assert.assertTrue("Segment is not marked in metadata as sealed in storage post flush.", context.segmentAggregator.getMetadata().isSealedInStorage()); }
SegmentHandle corruptedSegmentHandle = InMemoryStorage.newHandle(context.metadata.getStreamSegmentMetadata(segmentIds.get(0)).getName(), false); Supplier<Exception> exceptionSupplier = () -> {
private void verifyFinalOutput(HashMap<Long, ByteArrayOutputStream> segmentContents, Collection<Long> transactionIds, TestContext context) { // Verify all Transactions are deleted. for (long transactionId : transactionIds) { SegmentMetadata metadata = context.metadata.getStreamSegmentMetadata(transactionId); Assert.assertTrue("Transaction not marked as deleted in metadata: " + transactionId, metadata.isDeleted()); Assert.assertFalse("Transaction was not deleted from storage after being merged: " + transactionId, context.storage.exists(metadata.getName(), TIMEOUT).join()); verifyAttributes(metadata, context); } for (long segmentId : segmentContents.keySet()) { SegmentMetadata metadata = context.metadata.getStreamSegmentMetadata(segmentId); Assert.assertNotNull("Setup error: No metadata for segment " + segmentId, metadata); Assert.assertFalse("Setup error: Not expecting a Transaction segment in the final list: " + segmentId, context.transactionIds.containsKey(metadata.getId())); Assert.assertEquals("Metadata does not indicate that all bytes were copied to Storage for segment " + segmentId, metadata.getLength(), metadata.getStorageLength()); Assert.assertEquals("Metadata.Sealed disagrees with Metadata.SealedInStorage for segment " + segmentId, metadata.isSealed(), metadata.isSealedInStorage()); SegmentProperties sp = context.storage.getStreamSegmentInfo(metadata.getName(), TIMEOUT).join(); Assert.assertEquals("Metadata.StorageLength disagrees with Storage.Length for segment " + segmentId, metadata.getStorageLength(), sp.getLength()); Assert.assertEquals("Metadata.Sealed/SealedInStorage disagrees with Storage.Sealed for segment " + segmentId, metadata.isSealedInStorage(), sp.isSealed()); byte[] expected = segmentContents.get(segmentId).toByteArray(); byte[] actual = new byte[expected.length]; int actualLength = context.storage.read(InMemoryStorage.newHandle(metadata.getName(), true), 0, actual, 0, actual.length, TIMEOUT).join(); Assert.assertEquals("Unexpected number of bytes read from Storage for segment " + segmentId, metadata.getStorageLength(), actualLength); Assert.assertArrayEquals("Unexpected data written to storage for segment " + segmentId, expected, actual); Assert.assertEquals("Unexpected truncation offset for segment " + segmentId, metadata.getStartOffset(), context.storage.getTruncationOffset(metadata.getName())); verifyAttributes(metadata, context); } }
Assert.assertEquals("Unexpected truncation offset in Storage.", lastTruncateOp.getStreamSegmentOffset(), context.storage.getTruncationOffset(context.segmentAggregator.getMetadata().getName())); context.storage.read(InMemoryStorage.newHandle(context.segmentAggregator.getMetadata().getName(), false), 0, actualData, 0, actualData.length, TIMEOUT).join();
Assert.assertTrue("Segment is not sealed in storage post flush.", storageInfo.isSealed()); Assert.assertTrue("Segment is not marked in metadata as sealed in storage post flush.", context.segmentAggregator.getMetadata().isSealedInStorage()); context.storage.read(InMemoryStorage.newHandle(context.segmentAggregator.getMetadata().getName(), false), 0, actualData, 0, actualData.length, TIMEOUT).join(); checkAttributes(context); Assert.assertArrayEquals("Unexpected data written to storage.", expectedData, actualData);
Assert.assertEquals("Unexpected truncation offset in Storage.", truncateOp.getStreamSegmentOffset(), context.storage.getTruncationOffset(context.segmentAggregator.getMetadata().getName())); context.storage.read(InMemoryStorage.newHandle(context.segmentAggregator.getMetadata().getName(), false), 0, actualData, 0, actualData.length, TIMEOUT).join();