/** * Reads all of the cachedEntries from the ReadResult and puts their content into the cachedEntries list. * Upon encountering a non-cached entry, it stops iterating and returns it. */ private ReadResultEntry collectCachedEntries(long initialOffset, ReadResult readResult, ArrayList<ReadResultEntryContents> cachedEntries) { long expectedOffset = initialOffset; while (readResult.hasNext()) { ReadResultEntry entry = readResult.next(); if (entry.getType() == Cache) { Preconditions.checkState(entry.getStreamSegmentOffset() == expectedOffset, "Data returned from read was not contiguous."); ReadResultEntryContents content = entry.getContent().getNow(null); expectedOffset += content.getLength(); cachedEntries.add(content); } else { return entry; } } return null; }
/** * Reads from the Segment between the given offsets. This method assumes all the data is readily available in the cache, * otherwise it will block synchronously for Storage retrieval. * * @param segment The Segment to read from. * @param startOffset The offset to start reading from. * @param endOffset The offset to stop reading at. * @param timer Timer for the operation. * @return An Enumeration of InputStreams representing the read data. */ private InputStream readFromInMemorySegment(DirectSegmentAccess segment, long startOffset, long endOffset, TimeoutTimer timer) { long readOffset = startOffset; long remainingLength = endOffset - startOffset; ArrayList<InputStream> inputs = new ArrayList<>(); while (remainingLength > 0) { int readLength = (int) Math.min(remainingLength, Integer.MAX_VALUE); try (ReadResult readResult = segment.read(readOffset, readLength, timer.getRemaining())) { inputs.addAll(readResult.readRemaining(readLength, timer.getRemaining())); assert readResult.getConsumedLength() == readLength : "Expecting a full read (from memory)."; remainingLength -= readResult.getConsumedLength(); readOffset += readResult.getConsumedLength(); } } return new SequenceInputStream(Iterators.asEnumeration(inputs.iterator())); }
private void close(Throwable failureCause) { if (!this.closed.getAndSet(true)) { this.readResult.close(); if (failureCause == null) { // We are closing normally with no processing exception. this.entryHandler.processResultComplete(); } else { // An exception was encountered; this must be reported to the entry handler. this.entryHandler.processError(Exceptions.unwrap(failureCause)); } } }
private void checkSegmentReads(String segmentName, AtomicLong expectedCurrentOffset, long segmentLength, StreamSegmentStore store, byte[] expectedData) throws Exception { @Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset.get(), (int) (segmentLength - expectedCurrentOffset.get()), TIMEOUT).join(); Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext()); // A more thorough read check is done in StreamSegmentContainerTests; here we just check if the data was merged correctly. while (readResult.hasNext()) { ReadResultEntry readEntry = readResult.next(); AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength()); Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset.get(), readEntry.getStreamSegmentOffset()); if (!readEntry.getContent().isDone()) { readEntry.requestContent(TIMEOUT); } readEntry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType()); ReadResultEntryContents readEntryContents = readEntry.getContent().join(); byte[] actualData = new byte[readEntryContents.getLength()]; StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length); AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset.get(), actualData, 0, readEntryContents.getLength()); expectedCurrentOffset.addAndGet(readEntryContents.getLength()); } Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed()); }
public static void verifyReadResult(ReadResult readResult, SegmentProperties si, int expectedStartOffset, int expectedLength, byte[] writtenData) throws Exception { Assert.assertEquals("Unexpected ReadResult.StartOffset.", expectedStartOffset, readResult.getStreamSegmentStartOffset()); while (readResult.hasNext()) { val entry = readResult.next(); if (entry.getStreamSegmentOffset() < si.getStartOffset()) { Assert.assertEquals("Expected a truncated ReadResultEntry.", ReadResultEntryType.Truncated, entry.getType()); Assert.assertFalse("Not expecting ReadResult to have any other entries.", readResult.hasNext()); break; } else if (entry.getStreamSegmentOffset() == si.getLength()) { Assert.assertEquals("Expected an EndOfSegment ReadResultEntry.", ReadResultEntryType.EndOfStreamSegment, entry.getType()); Assert.assertFalse("Not expecting ReadResult to have any other entries.", readResult.hasNext()); break; } else { Assert.assertEquals("Expected a Storage ReadResultEntry.", ReadResultEntryType.Storage, entry.getType()); entry.requestContent(TIMEOUT); val contents = entry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); AssertExtensions.assertGreaterThanOrEqual("Empty read entry contents.", 0, contents.getLength()); val readData = StreamHelpers.readAll(contents.getData(), contents.getLength()); AssertExtensions.assertArrayEquals("Unexpected data read back.", writtenData, (int) entry.getStreamSegmentOffset(), readData, 0, readData.length); } } Assert.assertEquals("Unexpected consumed length.", expectedLength, readResult.getConsumedLength()); }
private CompletableFuture<ReadResultEntry> fetchNextEntry() { // Get the next item. We don't really rely on hasNext; we just use the fact that next() returns null // if there is nothing else to read. ReadResultEntry currentEntry = this.readResult.next(); if (currentEntry != null && currentEntry.getType() != ReadResultEntryType.EndOfStreamSegment) { // We have something to retrieve. CompletableFuture<ReadResultEntryContents> entryContentsFuture = currentEntry.getContent(); if (entryContentsFuture.isDone()) { // Result is readily available. return CompletableFuture.completedFuture(currentEntry); } else if (this.entryHandler.shouldRequestContents(currentEntry.getType(), currentEntry.getStreamSegmentOffset())) { // ReadResultEntry that does not have data readily available and we were instructed to request the content. currentEntry.requestContent(this.entryHandler.getRequestContentTimeout()); return entryContentsFuture.thenApply(v -> currentEntry); } } return null; }
@Override public CompletableFuture<ReadItem> readExact(String segmentName, Object address) { Exceptions.checkNotNullOrEmpty(segmentName, "segmentName"); Preconditions.checkArgument(address instanceof Address, "Unexpected address type."); Address a = (Address) address; return this.store .read(segmentName, a.offset, a.length, this.testConfig.getTimeout()) .thenApplyAsync(readResult -> { byte[] data = new byte[a.length]; readResult.readRemaining(data, this.testConfig.getTimeout()); return new SegmentStoreReadItem(new Event(new ByteArraySegment(data), 0), address); }, this.executor); }
private static void checkReadIndex(Map<String, ByteArrayOutputStream> segmentContents, Map<String, Long> lengths, Map<String, Long> truncationOffsets, TestContext context) throws Exception { waitForOperationsInReadIndex(context.container); for (String segmentName : segmentContents.keySet()) { long segmentLength = lengths.get(segmentName); long startOffset = truncationOffsets.getOrDefault(segmentName, 0L); val si = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join(); Assert.assertEquals("Unexpected Metadata StartOffset for segment " + segmentName, startOffset, si.getStartOffset()); Assert.assertEquals("Unexpected Metadata Length for segment " + segmentName, segmentLength, si.getLength()); byte[] expectedData = segmentContents.get(segmentName).toByteArray(); long expectedCurrentOffset = startOffset; @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, (int) (segmentLength - startOffset), TIMEOUT).join(); Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext()); // A more thorough read check is done in testSegmentRegularOperations; here we just check if the data was merged correctly. while (readResult.hasNext()) { ReadResultEntry readEntry = readResult.next(); AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength()); Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset()); Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally()); Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType()); ReadResultEntryContents readEntryContents = readEntry.getContent().join(); byte[] actualData = new byte[readEntryContents.getLength()]; StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length); AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength()); expectedCurrentOffset += readEntryContents.getLength(); } Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed()); } }
/** * Blocks until all operations processed so far have been added to the ReadIndex and InMemoryOperationLog. * This is needed to simplify test verification due to the fact that the the OperationProcessor commits operations to * the ReadIndex and InMemoryOperationLog asynchronously, after those operations were ack-ed. This method makes use * of the fact that the OperationProcessor/MemoryStateUpdater will still commit such operations in sequence; it * creates a new segment, writes 1 byte to it and issues a read (actual/future) and waits until it's completed - when * it is, it is guaranteed that everything prior to that has been committed. */ private static void waitForOperationsInReadIndex(SegmentContainer container) throws Exception { TimeoutTimer timer = new TimeoutTimer(TIMEOUT); String segmentName = "test" + System.nanoTime(); container.createStreamSegment(segmentName, null, timer.getRemaining()) .thenCompose(v -> container.append(segmentName, new byte[1], null, timer.getRemaining())) .thenCompose(v -> container.read(segmentName, 0, 1, timer.getRemaining())) .thenCompose(rr -> { ReadResultEntry rre = rr.next(); rre.requestContent(TIMEOUT); return rre.getContent().thenRun(rr::close); }) .thenCompose(v -> container.deleteStreamSegment(segmentName, timer.getRemaining())) .get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); }
@Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, 0, actualData.length, TIMEOUT).join(); actualLength = readResult.readRemaining(actualData, TIMEOUT); } catch (Exception ex) { ex = (Exception) Exceptions.unwrap(ex); ReadResult readResult = readOnlySegmentStore.read(segmentName, metadataProps.getStartOffset(), expectedLength, TIMEOUT).join(); actualLength = readResult.readRemaining(actualData, TIMEOUT);
/** * Reads the remaining contents of the ReadResult and returns an ordered List of InputStreams that contain its contents. * This will stop when either the given maximum length or the end of the ReadResult has been reached. * * @param maxLength The maximum number of bytes to read. * @param fetchTimeout A timeout to use when needing to fetch the contents of an entry that is not in the Cache. * @return A List containing InputStreams with the data read. */ default List<InputStream> readRemaining(int maxLength, Duration fetchTimeout) { int bytesRead = 0; ArrayList<InputStream> result = new ArrayList<>(); while (hasNext() && bytesRead < maxLength) { ReadResultEntry entry = next(); if (entry.getType() == ReadResultEntryType.EndOfStreamSegment || entry.getType() == ReadResultEntryType.Future) { // Reached the end. break; } else if (!entry.getContent().isDone()) { entry.requestContent(fetchTimeout); } result.add(entry.getContent().join().getData()); } return result; } }
@Cleanup ReadResult readResult = store.read(segmentName, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT).join(); Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext()); while (readResult.hasNext()) { ReadResultEntry readEntry = readResult.next(); Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset()); readResult.hasNext()); val first = truncatedResult.next(); Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType()); Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed());
@Cleanup ReadResult rr = context.readIndex.read(notSealedId, context.metadata.getStreamSegmentMetadata(notSealedId).getLength(), 1, TIMEOUT); ReadResultEntry fe = rr.next(); Assert.assertEquals("Expecting a Future Read.", ReadResultEntryType.Future, fe.getType()); Assert.assertFalse("Not expecting Future Read to be completed.", fe.getContent().isDone());
/** * Reads the remaining contents of the ReadResult into the given array. This will stop when the given target has been * filled or when the current end of the Segment has been reached. * * @param target A byte array where the ReadResult will be read into. * @param fetchTimeout A timeout to use when needing to fetch the contents of an entry that is not in the Cache. * @return The number of bytes read. */ @VisibleForTesting @SneakyThrows(IOException.class) default int readRemaining(byte[] target, Duration fetchTimeout) { int bytesRead = 0; while (hasNext() && bytesRead < target.length) { ReadResultEntry entry = next(); if (entry.getType() == ReadResultEntryType.EndOfStreamSegment || entry.getType() == ReadResultEntryType.Future) { // Reached the end. break; } else if (!entry.getContent().isDone()) { entry.requestContent(fetchTimeout); } ReadResultEntryContents contents = entry.getContent().join(); StreamHelpers.readAll(contents.getData(), target, bytesRead, Math.min(contents.getLength(), target.length - bytesRead)); bytesRead += contents.getLength(); } return bytesRead; }
while (readResult.hasNext()) { ReadResultEntry readEntry = readResult.next(); if (readEntry.getStreamSegmentOffset() >= segmentLength) { Assert.assertEquals("Unexpected value for isEndOfStreamSegment when reaching the end of sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType()); Assert.assertTrue("ReadResult was not closed when reaching the end of sealed segment" + segmentName, readResult.isClosed());
@Cleanup val readResult = dsa.read(1, readBuffer.length, TIMEOUT); val firstEntry = readResult.next(); firstEntry.requestContent(TIMEOUT); val entryContents = firstEntry.getContent().join();
void performReadIndexChecks(Collection<OperationWithCompletion> operations, ReadIndex readIndex) throws Exception { AbstractMap<Long, Integer> expectedLengths = getExpectedLengths(operations); AbstractMap<Long, InputStream> expectedData = getExpectedContents(operations); for (Map.Entry<Long, InputStream> e : expectedData.entrySet()) { int expectedLength = expectedLengths.getOrDefault(e.getKey(), -1); @Cleanup ReadResult readResult = readIndex.read(e.getKey(), 0, expectedLength, TIMEOUT); int readLength = 0; while (readResult.hasNext()) { ReadResultEntryContents entry = readResult.next().getContent().join(); int length = entry.getLength(); readLength += length; int streamSegmentOffset = expectedLengths.getOrDefault(e.getKey(), 0); expectedLengths.put(e.getKey(), streamSegmentOffset + length); AssertExtensions.assertStreamEquals(String.format("Unexpected data returned from ReadIndex. StreamSegmentId = %d, Offset = %d.", e.getKey(), streamSegmentOffset), e.getValue(), entry.getData(), length); } Assert.assertEquals("Not enough bytes were read from the ReadIndex for StreamSegment " + e.getKey(), expectedLength, readLength); } }
@Cleanup ReadResult truncatedResult = context.readIndex.read(segmentId, 0, 1, TIMEOUT); val first = truncatedResult.next(); Assert.assertEquals("Read request for a truncated offset did not start with a Truncated ReadResultEntryType.", ReadResultEntryType.Truncated, first.getType()); @Cleanup ReadResult readResult = context.readIndex.read(segmentId, expectedCurrentOffset, (int) (segmentLength - expectedCurrentOffset), TIMEOUT); Assert.assertTrue(testId + ": Empty read result for segment " + segmentId, readResult.hasNext()); while (readResult.hasNext()) { ReadResultEntry readEntry = readResult.next(); AssertExtensions.assertGreaterThan(testId + ": getRequestedReadLength should be a positive integer for segment " + segmentId, 0, readEntry.getRequestedReadLength()); Assert.assertEquals(testId + ": Unexpected value from getStreamSegmentOffset for segment " + segmentId, expectedCurrentOffset, readEntry.getStreamSegmentOffset()); Assert.assertTrue(testId + ": ReadResult was not closed post-full-consumption for segment" + segmentId, readResult.isClosed());
/** * Tests the behavior of Future Reads on an empty index that is sealed. */ @Test public void testFutureReadsEmptyIndex() throws Exception { @Cleanup TestContext context = new TestContext(); // Create an empty segment. This is the easiest way to ensure the Read Index is empty. long segmentId = createSegment(0, context); @Cleanup val rr = context.readIndex.read(segmentId, 0, 1, TIMEOUT); val futureReadEntry = rr.next(); Assert.assertEquals("Unexpected entry type.", ReadResultEntryType.Future, futureReadEntry.getType()); Assert.assertFalse("ReadResultEntry is completed.", futureReadEntry.getContent().isDone()); // Seal the segment. This should complete all future reads. context.metadata.getStreamSegmentMetadata(segmentId).markSealed(); context.readIndex.triggerFutureReads(Collections.singleton(segmentId)); Assert.assertTrue("Expected future read to be failed after sealing.", futureReadEntry.getContent().isCompletedExceptionally()); AssertExtensions.assertSuppliedFutureThrows( "Expected future read to be failed with appropriate exception.", futureReadEntry::getContent, ex -> ex instanceof StreamSegmentSealedException); }
@Test public void testReadDirectlyFromStore() throws Exception { String segmentName = "testReadFromStore"; final int entries = 10; final byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; UUID clientId = UUID.randomUUID(); StreamSegmentStore segmentStore = serviceBuilder.createStreamSegmentService(); fillStoreForSegment(segmentName, clientId, data, entries, segmentStore); ReadResult result = segmentStore.read(segmentName, 0, entries * data.length, Duration.ZERO).get(); int index = 0; while (result.hasNext()) { ReadResultEntry entry = result.next(); ReadResultEntryType type = entry.getType(); assertTrue(type == ReadResultEntryType.Cache || type == ReadResultEntryType.Future); // Each ReadResultEntryContents may be of an arbitrary length - we should make no assumptions. // Also put a timeout when fetching the response in case we get back a Future read and it never completes. ReadResultEntryContents contents = entry.getContent().get(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); byte next; while ((next = (byte) contents.getData().read()) != -1) { byte expected = data[index % data.length]; assertEquals(expected, next); index++; } } assertEquals(entries * data.length, index); }