@Override public CompletableReadResultEntry apply(Long readOffset, Integer readLength) { if (readOffset < this.segmentInfo.getStartOffset()) { // We attempted to read from a truncated portion of the Segment. return new TruncatedReadResultEntry(readOffset, readLength, this.segmentInfo.getStartOffset()); } else if (readOffset >= this.segmentInfo.getLength()) { // We've reached the end of a Sealed Segment. return new EndOfStreamSegmentReadResultEntry(readOffset, readLength); } else { // Execute the read from Storage. if (readOffset + readLength > segmentInfo.getLength()) { // Adjust max read length to not exceed the last known offset of the Segment. readLength = (int) (segmentInfo.getLength() - readOffset); } readLength = Math.min(this.readBlockSize, readLength); return new StorageReadResultEntry(readOffset, readLength, this::fetchContents); } }
/** * Creates a new instance of the StreamSegmentMapOperation class for a non-transaction Segment. * * @param streamSegmentProperties Information about the StreamSegment. */ public StreamSegmentMapOperation(SegmentProperties streamSegmentProperties) { this.streamSegmentId = ContainerMetadata.NO_STREAM_SEGMENT_ID; this.streamSegmentName = streamSegmentProperties.getName(); this.startOffset = streamSegmentProperties.getStartOffset(); this.length = streamSegmentProperties.getLength(); this.sealed = streamSegmentProperties.isSealed(); this.attributes = streamSegmentProperties.getAttributes(); this.pinned = false; }
connection.send(new SegmentIsTruncated(nonCachedEntry.getStreamSegmentOffset(), segment, info.getStartOffset(), EMPTY_STACK_TRACE))) .exceptionally(e -> handleException(nonCachedEntry.getStreamSegmentOffset(), segment, operation, wrapCancellationException(e))); } else {
} catch (Exception ex) { ex = (Exception) Exceptions.unwrap(ex); if (!(ex instanceof StreamSegmentTruncatedException) || metadataProps.getStartOffset() == 0) { expectedLength = (int) (storageProps.getLength() - metadataProps.getStartOffset()); if (metadataProps.getStartOffset() < storageProps.getLength()) { @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, metadataProps.getStartOffset(), expectedLength, TIMEOUT).join(); actualLength = readResult.readRemaining(actualData, TIMEOUT);
private void write00(SegmentInfo s, RevisionDataOutput output) throws IOException { output.writeLong(s.getSegmentId()); SegmentProperties sp = s.getProperties(); output.writeUTF(sp.getName()); output.writeLong(sp.getLength()); output.writeLong(sp.getStartOffset()); output.writeBoolean(sp.isSealed()); // We only serialize Core Attributes. Extended Attributes can be retrieved from the AttributeIndex. output.writeMap(Attributes.getCoreNonNullAttributes(sp.getAttributes()), RevisionDataOutput::writeUUID, RevisionDataOutput::writeLong); }
@Override public void getStreamSegmentInfo(GetStreamSegmentInfo getStreamSegmentInfo) { String segmentName = getStreamSegmentInfo.getSegmentName(); final String operation = "getStreamSegmentInfo"; if (!verifyToken(segmentName, getStreamSegmentInfo.getRequestId(), getStreamSegmentInfo.getDelegationToken(), operation)) { return; } segmentStore.getStreamSegmentInfo(segmentName, TIMEOUT) .thenAccept(properties -> { if (properties != null) { StreamSegmentInfo result = new StreamSegmentInfo(getStreamSegmentInfo.getRequestId(), properties.getName(), true, properties.isSealed(), properties.isDeleted(), properties.getLastModified().getTime(), properties.getLength(), properties.getStartOffset()); log.trace("Read stream segment info: {}", result); connection.send(result); } else { log.trace("getStreamSegmentInfo could not find segment {}", segmentName); connection.send(new StreamSegmentInfo(getStreamSegmentInfo.getRequestId(), segmentName, false, true, true, 0, 0, 0)); } }) .exceptionally(e -> handleException(getStreamSegmentInfo.getRequestId(), segmentName, operation, e)); }
protected void assertEquals(String message, SegmentProperties expected, SegmentProperties actual) { Assert.assertEquals(message + " getName() mismatch.", expected.getName(), actual.getName()); Assert.assertEquals(message + " isDeleted() mismatch.", expected.isDeleted(), actual.isDeleted()); Assert.assertEquals(message + " getLength() mismatch.", expected.getLength(), actual.getLength()); Assert.assertEquals(message + " isSealed() mismatch.", expected.isSealed(), actual.isSealed()); Assert.assertEquals(message + " getStartOffset() mismatch.", expected.getStartOffset(), actual.getStartOffset()); }
long expectedStartOffset = startOffsets.getOrDefault(segmentName, 0L); long expectedLength = e.getValue(); Assert.assertEquals("Unexpected Start Offset for segment " + segmentName, expectedStartOffset, sp.getStartOffset()); Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength()); Assert.assertEquals("Unexpected value for isSealed for segment " + segmentName, expectSealed, sp.isSealed());
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
long expectedLength = lengths.get(segmentName); Assert.assertEquals("Unexpected StartOffset for segment " + segmentName, expectedStartOffset, sp.getStartOffset()); Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength()); Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
long expectedLength = lengths.get(segmentName); Assert.assertEquals("Unexpected StartOffset for non-truncated segment " + segmentName, 0, sp.getStartOffset()); Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength()); Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted());
private CompletableFuture<Void> waitForSegmentInStorage(SegmentProperties metadataProps, TestContext context) { if (metadataProps.getLength() == 0) { // Empty segments may or may not exist in Storage, so don't bother complicating ourselves with this. return CompletableFuture.completedFuture(null); } Function<SegmentProperties, Boolean> meetsConditions = storageProps -> storageProps.isSealed() == metadataProps.isSealed() && storageProps.getLength() >= metadataProps.getLength() && context.storageFactory.truncationOffsets.getOrDefault(metadataProps.getName(), 0L) >= metadataProps.getStartOffset(); AtomicBoolean canContinue = new AtomicBoolean(true); TimeoutTimer timer = new TimeoutTimer(TIMEOUT); return Futures.loop( canContinue::get, () -> Futures.exceptionallyExpecting( context.storage.getStreamSegmentInfo(metadataProps.getName(), TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException, StreamSegmentInformation.builder().name(metadataProps.getName()).build()) .thenCompose(storageProps -> { if (meetsConditions.apply(storageProps)) { canContinue.set(false); return CompletableFuture.completedFuture(null); } else if (!timer.hasRemaining()) { return Futures.failedFuture(new TimeoutException()); } else { return Futures.delayedFuture(Duration.ofMillis(10), executorService()); } }).thenRun(Runnables.doNothing()), executorService()); }
private static void checkReadIndex(Map<String, ByteArrayOutputStream> segmentContents, Map<String, Long> lengths, Map<String, Long> truncationOffsets, TestContext context) throws Exception { waitForOperationsInReadIndex(context.container); for (String segmentName : segmentContents.keySet()) { long segmentLength = lengths.get(segmentName); long startOffset = truncationOffsets.getOrDefault(segmentName, 0L); val si = context.container.getStreamSegmentInfo(segmentName, TIMEOUT).join(); Assert.assertEquals("Unexpected Metadata StartOffset for segment " + segmentName, startOffset, si.getStartOffset()); Assert.assertEquals("Unexpected Metadata Length for segment " + segmentName, segmentLength, si.getLength()); byte[] expectedData = segmentContents.get(segmentName).toByteArray(); long expectedCurrentOffset = startOffset; @Cleanup ReadResult readResult = context.container.read(segmentName, expectedCurrentOffset, (int) (segmentLength - startOffset), TIMEOUT).join(); Assert.assertTrue("Empty read result for segment " + segmentName, readResult.hasNext()); // A more thorough read check is done in testSegmentRegularOperations; here we just check if the data was merged correctly. while (readResult.hasNext()) { ReadResultEntry readEntry = readResult.next(); AssertExtensions.assertGreaterThan("getRequestedReadLength should be a positive integer for segment " + segmentName, 0, readEntry.getRequestedReadLength()); Assert.assertEquals("Unexpected value from getStreamSegmentOffset for segment " + segmentName, expectedCurrentOffset, readEntry.getStreamSegmentOffset()); Assert.assertTrue("getContent() did not return a completed future for segment" + segmentName, readEntry.getContent().isDone() && !readEntry.getContent().isCompletedExceptionally()); Assert.assertNotEquals("Unexpected value for isEndOfStreamSegment for non-sealed segment " + segmentName, ReadResultEntryType.EndOfStreamSegment, readEntry.getType()); ReadResultEntryContents readEntryContents = readEntry.getContent().join(); byte[] actualData = new byte[readEntryContents.getLength()]; StreamHelpers.readAll(readEntryContents.getData(), actualData, 0, actualData.length); AssertExtensions.assertArrayEquals("Unexpected data read from segment " + segmentName + " at offset " + expectedCurrentOffset, expectedData, (int) expectedCurrentOffset, actualData, 0, readEntryContents.getLength()); expectedCurrentOffset += readEntryContents.getLength(); } Assert.assertTrue("ReadResult was not closed post-full-consumption for segment" + segmentName, readResult.isClosed()); } }
Assert.assertEquals("Unexpected name.", segmentName, info.getName()); Assert.assertEquals("Unexpected length.", appendData.length, info.getLength()); Assert.assertEquals("Unexpected startOffset.", 1, info.getStartOffset()); Assert.assertEquals("Unexpected attribute count.", 2, info.getAttributes().size()); Assert.assertEquals("Unexpected attribute 1.", 1L, (long) info.getAttributes().get(attributeId1));
Map<UUID, Long> expectedAttributes = segmentState == null ? null : segmentState.getAttributes(); SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in metadata for StreamSegment " + name, expectedAttributes, sm); long expectedStartOffset = segmentState == null ? 0 : segmentState.getStartOffset(); Assert.assertEquals("Unexpected StartOffset in metadata for " + name, expectedStartOffset, sm.getStartOffset());
processor.truncateSegment(new WireCommands.TruncateSegment(3, streamSegmentName, truncateOffset, "")); assertEquals(truncateOffset, store.getStreamSegmentInfo(streamSegmentName, PravegaRequestProcessor.TIMEOUT) .join().getStartOffset()); .join().getStartOffset()); .join().getStartOffset());
private StreamSegmentMetadata toMetadata(long segmentId, SegmentProperties sp) { StreamSegmentMetadata sm = new StreamSegmentMetadata(sp.getName(), segmentId, CONTAINER_ID); sm.updateAttributes(sp.getAttributes()); sm.setLength(sp.getLength()); sm.setStartOffset(sp.getStartOffset()); if (sp.isSealed()) { sm.markSealed(); } if (sp.isDeleted()) { sm.markDeleted(); } return sm; }
sm.setStorageLength(0); sm.setLength(sp.getLength()); sm.setStartOffset(sp.getStartOffset()); if (sp.isSealed()) { sm.markSealed();
public static void verifyReadResult(ReadResult readResult, SegmentProperties si, int expectedStartOffset, int expectedLength, byte[] writtenData) throws Exception { Assert.assertEquals("Unexpected ReadResult.StartOffset.", expectedStartOffset, readResult.getStreamSegmentStartOffset()); while (readResult.hasNext()) { val entry = readResult.next(); if (entry.getStreamSegmentOffset() < si.getStartOffset()) { Assert.assertEquals("Expected a truncated ReadResultEntry.", ReadResultEntryType.Truncated, entry.getType()); Assert.assertFalse("Not expecting ReadResult to have any other entries.", readResult.hasNext()); break; } else if (entry.getStreamSegmentOffset() == si.getLength()) { Assert.assertEquals("Expected an EndOfSegment ReadResultEntry.", ReadResultEntryType.EndOfStreamSegment, entry.getType()); Assert.assertFalse("Not expecting ReadResult to have any other entries.", readResult.hasNext()); break; } else { Assert.assertEquals("Expected a Storage ReadResultEntry.", ReadResultEntryType.Storage, entry.getType()); entry.requestContent(TIMEOUT); val contents = entry.getContent().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); AssertExtensions.assertGreaterThanOrEqual("Empty read entry contents.", 0, contents.getLength()); val readData = StreamHelpers.readAll(contents.getData(), contents.getLength()); AssertExtensions.assertArrayEquals("Unexpected data read back.", writtenData, (int) entry.getStreamSegmentOffset(), readData, 0, readData.length); } } Assert.assertEquals("Unexpected consumed length.", expectedLength, readResult.getConsumedLength()); }