@Test(timeout = 20000) public void testReadSegmentTruncated() { // Set up PravegaRequestProcessor instance to execute read segment request against String streamSegmentName = "testReadSegment"; int readLength = 1000; StreamSegmentStore store = mock(StreamSegmentStore.class); ServerConnection connection = mock(ServerConnection.class); PravegaRequestProcessor processor = new PravegaRequestProcessor(store, mock(TableStore.class), connection); TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.Truncated, 0, readLength); List<ReadResultEntry> results = new ArrayList<>(); results.add(entry1); CompletableFuture<ReadResult> readResult = new CompletableFuture<>(); readResult.complete(new TestReadResult(0, readLength, results)); when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult); StreamSegmentInformation info = StreamSegmentInformation.builder() .name(streamSegmentName) .length(1234) .startOffset(123) .build(); when(store.getStreamSegmentInfo(streamSegmentName, PravegaRequestProcessor.TIMEOUT)) .thenReturn(CompletableFuture.completedFuture(info)); // Execute and Verify readSegment calling stack in connection and store is executed as design. processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, "")); verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT); verify(store).getStreamSegmentInfo(streamSegmentName, PravegaRequestProcessor.TIMEOUT); verify(connection).send(new WireCommands.SegmentIsTruncated(0, streamSegmentName, info.getStartOffset(), "")); verifyNoMoreInteractions(connection); verifyNoMoreInteractions(store); }
private SegmentProperties createSegmentProperty(String streamSegmentName, UUID txnId) { Map<UUID, Long> attributes = new HashMap<>(); attributes.put(Attributes.EVENT_COUNT, 10L); attributes.put(Attributes.CREATION_TIME, System.currentTimeMillis()); return StreamSegmentInformation.builder() .name(txnId == null ? streamSegmentName + "#." : streamSegmentName + "#transaction." + txnId) .sealed(true) .deleted(false) .lastModified(null) .startOffset(0) .length(100) .attributes(attributes) .build(); }
/** * Reads a range of bytes from a Segment in Storage. * * @param handle A SegmentHandle pointing to the Segment to read from. * @param startOffset The first offset within the Segment to read from. * @param maxReadLength The maximum number of bytes to read. * @param readBlockSize The maximum number of bytes to read at once (the returned StreamSegmentReadResult will be * broken down into Entries smaller than or equal to this size). * @param storage A ReadOnlyStorage to execute the reads against. * @return A StreamSegmentReadResult that can be used to process the data. This will be made up of ReadResultEntries * of the following types: Storage, Truncated or EndOfSegment. */ public static StreamSegmentReadResult read(SegmentHandle handle, long startOffset, int maxReadLength, int readBlockSize, ReadOnlyStorage storage) { Exceptions.checkArgument(startOffset >= 0, "startOffset", "startOffset must be a non-negative number."); Exceptions.checkArgument(maxReadLength >= 0, "maxReadLength", "maxReadLength must be a non-negative number."); Preconditions.checkNotNull(handle, "handle"); Preconditions.checkNotNull(storage, "storage"); String traceId = String.format("Read[%s]", handle.getSegmentName()); // Build a SegmentInfo using the information we are given. If startOffset or length are incorrect, the underlying // ReadOnlyStorage will throw appropriate exceptions at the caller. StreamSegmentInformation segmentInfo = StreamSegmentInformation.builder().name(handle.getSegmentName()) .startOffset(startOffset) .length(startOffset + maxReadLength) .build(); return new StreamSegmentReadResult(startOffset, maxReadLength, new SegmentReader(segmentInfo, handle, readBlockSize, storage), traceId); }
.startOffset(startOffset) .length(storageLength) .sealed(true) .builder() .name(mapOp.getStreamSegmentName() + "_pinned") .startOffset(startOffset) .length(storageLength) .sealed(true)
.builder() .name(segmentName) .startOffset(0L) .length(1L) .attributes(toAttributes(createAttributeUpdates(ATTRIBUTE_COUNT)))
.name(segmentName) .length(getSegmentLength.apply(segmentName)) .startOffset(getSegmentStartOffset.apply(segmentName)) .sealed(i % 2 == 0) .attributes(toAttributes(createAttributeUpdates(ATTRIBUTE_COUNT)))
val si1 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build(); val rr1 = StreamSegmentStorageReader.read(si1, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s); val firstEntry1 = rr1.next(); val si2 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build(); val rr2 = StreamSegmentStorageReader.read(si2, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s);
/** * Tests the getStreamSegmentInfo() method. */ @Test public void testGetStreamSegmentInfo() { @Cleanup val context = new TestContext(); context.container.startAsync().awaitRunning(); // Non-existent segment. AssertExtensions.assertSuppliedFutureThrows( "Unexpected exception when the segment does not exist.", () -> context.container.getStreamSegmentInfo(SEGMENT_NAME, TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException); // Create a segment, add some data, set some attributes, "truncate" it and then seal it. val storageInfo = context.storage.create(SEGMENT_NAME, TIMEOUT) .thenCompose(handle -> context.storage.write(handle, 0, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT)) .thenCompose(v -> context.storage.getStreamSegmentInfo(SEGMENT_NAME, TIMEOUT)).join(); val expectedInfo = StreamSegmentInformation.from(storageInfo) .startOffset(storageInfo.getLength() / 2) .attributes(ImmutableMap.of(UUID.randomUUID(), 100L, Attributes.EVENT_COUNT, 1L)) .build(); // Fetch the SegmentInfo from the ReadOnlyContainer and verify it is as expected. val actual = context.container.getStreamSegmentInfo(SEGMENT_NAME, TIMEOUT).join(); Assert.assertEquals("Unexpected Name.", expectedInfo.getName(), actual.getName()); Assert.assertEquals("Unexpected Length.", expectedInfo.getLength(), actual.getLength()); Assert.assertEquals("Unexpected Sealed status.", expectedInfo.isSealed(), actual.isSealed()); }
/** * Tests the read() method on a truncated segment. */ @Test public void testTruncatedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(SEGMENT_APPEND_SIZE) .sealed(false).build(); // 1. Read a truncated offset. val truncatedResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(truncatedResult, si, 0, 0, writtenData); // 2. Read a truncated offset. val nonTruncatedResult = StreamSegmentStorageReader.read(si, SEGMENT_APPEND_SIZE, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(nonTruncatedResult, si, SEGMENT_APPEND_SIZE, writtenData.length - SEGMENT_APPEND_SIZE, writtenData); }
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
@Override protected StreamSegmentMapOperation createOperation(Random random) { long length = MathHelpers.abs(random.nextLong()); val op = new StreamSegmentMapOperation(StreamSegmentInformation .builder() .name(super.getStreamSegmentName(random.nextLong())) .startOffset(length / 2) .length(length) .sealed(random.nextBoolean()) .deleted(random.nextBoolean()) .attributes(createAttributes(10)) .build()); op.markPinned(); return op; }
/** * Tests the read() method on a non-truncated, non-sealed segment. */ @Test public void testNormalRead() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(false).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
private StreamSegmentMapOperation createMap(String name) { return new StreamSegmentMapOperation(StreamSegmentInformation.builder() .name(name) .length(SEGMENT_LENGTH) .startOffset(SEGMENT_LENGTH / 2) .sealed(true) .attributes(createAttributes()) .build()); }
private void read00(RevisionDataInput input, SegmentInfo.SegmentInfoBuilder builder) throws IOException { builder.segmentId(input.readLong()); val infoBuilder = StreamSegmentInformation .builder() .name(input.readUTF()) .length(input.readLong()) .startOffset(input.readLong()) .sealed(input.readBoolean()); infoBuilder.attributes(input.readMap(RevisionDataInput::readUUID, RevisionDataInput::readLong)); builder.properties(infoBuilder.build()); } }
/** * Tests the read() method on a Sealed Segment. */ @Test public void testSealedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(true).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }