public static WireCommand readFrom(DataInput in, int length) throws IOException { String segment = in.readUTF(); long offset = in.readLong(); boolean atTail = in.readBoolean(); boolean endOfSegment = in.readBoolean(); int dataLength = in.readInt(); if (dataLength > length) { throw new BufferOverflowException(); } byte[] data = new byte[dataLength]; in.readFully(data); return new SegmentRead(segment, offset, atTail, endOfSegment, ByteBuffer.wrap(data)); }
@Override public void segmentIsSealed(WireCommands.SegmentIsSealed segmentIsSealed) { log.info("Received segmentSealed {}", segmentIsSealed); CompletableFuture<SegmentRead> future = grabFuture(segmentIsSealed.getSegment(), segmentIsSealed.getRequestId()); if (future != null) { future.complete(new WireCommands.SegmentRead(segmentIsSealed.getSegment(), segmentIsSealed.getRequestId(), true, true, ByteBuffer.allocate(0))); } }
@Test(timeout = 20000) public void testReadSegmentWithCancellationException() { // Set up PravegaRequestProcessor instance to execute read segment request against String streamSegmentName = "testReadSegment"; int readLength = 1000; StreamSegmentStore store = mock(StreamSegmentStore.class); ServerConnection connection = mock(ServerConnection.class); PravegaRequestProcessor processor = new PravegaRequestProcessor(store, mock(TableStore.class), connection); CompletableFuture<ReadResult> readResult = new CompletableFuture<>(); readResult.completeExceptionally(new CancellationException("cancel read")); // Simulate a CancellationException for a Read Segment. when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult); // Execute and Verify readSegment is calling stack in connection and store is executed as design. processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, "")); verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT); // Since the underlying store cancels the read request verify if an empty SegmentRead Wirecommand is sent as a response. verify(connection).send(new WireCommands.SegmentRead(streamSegmentName, 0, true, false, ByteBuffer.wrap(new byte[0]))); verifyNoMoreInteractions(connection); verifyNoMoreInteractions(store); }
@Test public void testIsSegmentReady() throws EndOfSegmentException, SegmentTruncatedException { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; int numEntries = SegmentInputStreamImpl.DEFAULT_BUFFER_SIZE / data.length; ByteBuffer wireData = ByteBuffer.allocate((data.length + WireCommands.TYPE_PLUS_LENGTH_SIZE) * numEntries); for (int i = 0; i < numEntries; i++) { wireData.putInt(WireCommandType.EVENT.getCode()); wireData.putInt(data.length); wireData.put(data); } wireData.flip(); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 3); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); assertFalse(stream.isSegmentReady()); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, true, false, wireData.slice())); for (int i = 0; i < numEntries; i++) { assertTrue(stream.isSegmentReady()); assertEquals(ByteBuffer.wrap(data), stream.read()); } assertFalse(stream.isSegmentReady()); assertBlocks(() -> stream.read(), () -> { fakeNetwork.complete(1, new WireCommands.SegmentRead(segment.getScopedName(), wireData.capacity(), false, false, createEventFromData(data))); }); assertFalse(stream.isSegmentReady()); }
@Test public void testLongerThanRequestedRead() throws EndOfSegmentException, SegmentTruncatedException { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; int numEntries = SegmentInputStreamImpl.DEFAULT_BUFFER_SIZE / data.length; ByteBuffer wireData = ByteBuffer.allocate((data.length + WireCommands.TYPE_PLUS_LENGTH_SIZE) * numEntries); for (int i = 0; i < numEntries; i++) { wireData.putInt(WireCommandType.EVENT.getCode()); wireData.putInt(data.length); wireData.put(data); } wireData.flip(); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 3); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, wireData.slice())); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); for (int i = 0; i < numEntries; i++) { assertEquals(ByteBuffer.wrap(data), stream.read()); } ByteBuffer read = assertBlocks(() -> stream.read(), () -> { fakeNetwork.complete(1, new WireCommands.SegmentRead(segment.getScopedName(), wireData.capacity(), false, false, createEventFromData(data))); }); assertEquals(ByteBuffer.wrap(data), read); }
@Test(timeout = 10000) public void testWrongOffsetReturned() throws ConnectionFailedException { Segment segment = new Segment("scope", "testWrongOffsetReturned", 0); byte[] good = new byte[] { 0, 1, 2, 3, 4 }; byte[] bad = new byte[] { 9, 8, 7, 6 }; PravegaNodeUri endpoint = new PravegaNodeUri("localhost", SERVICE_PORT); MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl(); MockController controller = new MockController(endpoint.getEndpoint(), endpoint.getPort(), connectionFactory); @Cleanup AsyncSegmentInputStreamImpl in = new AsyncSegmentInputStreamImpl(controller, connectionFactory, segment, ""); ClientConnection c = mock(ClientConnection.class); connectionFactory.provideConnection(endpoint, c); CompletableFuture<SegmentRead> readFuture = in.read(1234, 5678); AssertExtensions.assertBlocks(() -> readFuture.get(), () -> { ReplyProcessor processor = connectionFactory.getProcessor(endpoint); processor.segmentRead(new WireCommands.SegmentRead(segment.getScopedName(), 1235, false, false, ByteBuffer.wrap(bad))); processor.segmentRead(new WireCommands.SegmentRead(segment.getScopedName(), 1234, false, false, ByteBuffer.wrap(good))); }); verify(c).sendAsync(Mockito.eq(new WireCommands.ReadSegment(segment.getScopedName(), 1234, 5678, "")), Mockito.any(ClientConnection.CompletedCallback.class)); assertTrue(Futures.isSuccessful(readFuture)); assertEquals(ByteBuffer.wrap(good), readFuture.join().getData()); verifyNoMoreInteractions(c); }
@Test(timeout = 20000) public void testReadSegmentEmptySealed() { // Set up PravegaRequestProcessor instance to execute read segment request against String streamSegmentName = "testReadSegment"; int readLength = 1000; StreamSegmentStore store = mock(StreamSegmentStore.class); ServerConnection connection = mock(ServerConnection.class); PravegaRequestProcessor processor = new PravegaRequestProcessor(store, mock(TableStore.class), connection); TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.EndOfStreamSegment, 0, readLength); List<ReadResultEntry> results = new ArrayList<>(); results.add(entry1); CompletableFuture<ReadResult> readResult = new CompletableFuture<>(); readResult.complete(new TestReadResult(0, readLength, results)); when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult); // Execute and Verify readSegment calling stack in connection and store is executed as design. processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, "")); verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT); verify(connection).send(new WireCommands.SegmentRead(streamSegmentName, 0, false, true, ByteBuffer.wrap(new byte[0]))); verifyNoMoreInteractions(connection); verifyNoMoreInteractions(store); }
@Test public void testSetOffset() throws EndOfSegmentException, SegmentTruncatedException { byte[] data1 = new byte[]{0, 1, 2, 3, 4, 5}; byte[] data2 = new byte[]{6, 7, 8, 9}; ByteBuffer wireData1 = createEventFromData(data1); ByteBuffer wireData2 = createEventFromData(data2); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 5); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData1, 0, wireData1.remaining()))); ByteBuffer read = stream.read(); assertEquals(ByteBuffer.wrap(data1), read); fakeNetwork.complete(2, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData1, 0, wireData1.remaining()))); fakeNetwork.complete(3, new WireCommands.SegmentRead(segment.getScopedName(), wireData1.remaining(), false, false, ByteBufferUtils.slice(wireData2, 0, wireData2.remaining()))); stream.setOffset(0); read = stream.read(); assertEquals(ByteBuffer.wrap(data1), read); read = stream.read(); assertEquals(ByteBuffer.wrap(data2), read); }
@Test public void testReadWithEndOffsetWithSmallerReads() throws Exception { byte[] data = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); int wireDataSize = wireData.remaining(); //size of the data with header size. AsyncSegmentInputStream mockAsyncInputStream = mock(AsyncSegmentInputStream.class); when(mockAsyncInputStream.read(0, wireDataSize)) .thenReturn(CompletableFuture.completedFuture(new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, 2)))); when(mockAsyncInputStream.read(2, 16)) .thenReturn(CompletableFuture.completedFuture(new WireCommands.SegmentRead(segment.getScopedName(), 2, false, false, ByteBufferUtils.slice(wireData, 2, wireDataSize - 2)))); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(mockAsyncInputStream, 0, wireDataSize, SegmentInputStreamImpl.DEFAULT_BUFFER_SIZE); ByteBuffer read = stream.read(); assertEquals(ByteBuffer.wrap(data), read); //verify we are reading the data. verify(mockAsyncInputStream, times(1)).read(0L, wireDataSize); verify(mockAsyncInputStream, times(1)).read(2L, wireDataSize - 2); }
@Test public void testReadWithEndOffsetWithDataGreaterThanBuffer() throws Exception { byte[] data = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); int wireDataSize = wireData.remaining(); //size of the data with header size. int bufferSize = wireDataSize / 2; //buffer is half the data length AsyncSegmentInputStream mockAsyncInputStream = mock(AsyncSegmentInputStream.class); when(mockAsyncInputStream.read(0, bufferSize)) .thenReturn(CompletableFuture.completedFuture(new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, bufferSize)))); when(mockAsyncInputStream.read(bufferSize, wireDataSize - bufferSize)) .thenReturn(CompletableFuture.completedFuture(new WireCommands.SegmentRead(segment.getScopedName(), bufferSize, false, false, ByteBufferUtils.slice(wireData, bufferSize, wireDataSize - bufferSize)))); //Create a SegmentInputStream where the Buffer can hold only part of the data. @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(mockAsyncInputStream, 0, wireDataSize, bufferSize); ByteBuffer read = stream.read(); assertEquals(ByteBuffer.wrap(data), read); //verify we are reading the data. verify(mockAsyncInputStream, times(1)).read(0L, bufferSize); verify(mockAsyncInputStream, times(1)).read(bufferSize, wireDataSize - bufferSize); }
@Test(timeout = 10000) public void testTimeout() throws EndOfSegmentException, SegmentTruncatedException { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 7); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); assertBlocks(() -> stream.read(), () -> fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, wireData.slice()))); ByteBuffer read = stream.read(10); assertNull(read); fakeNetwork.completeExceptionally(1, new ConnectionFailedException()); AssertExtensions.assertThrows(ConnectionFailedException.class, () -> stream.read()); stream.read(10); assertNull(read); read = stream.read(10); assertNull(read); }
@Test public void testExceptionRecovery() throws EndOfSegmentException, SegmentTruncatedException { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 7); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, 2))); fakeNetwork.completeExceptionally(1, new ConnectionFailedException()); fakeNetwork.complete(2, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, 2))); fakeNetwork.complete(3, new WireCommands.SegmentRead(segment.getScopedName(), 2, false, false, ByteBufferUtils.slice(wireData, 2, 7))); fakeNetwork.complete(4, new WireCommands.SegmentRead(segment.getScopedName(), 9, false, false, ByteBufferUtils.slice(wireData, 9, 2))); fakeNetwork.complete(5, new WireCommands.SegmentRead(segment.getScopedName(), 11, false, false, ByteBufferUtils.slice(wireData, 11, wireData.capacity() - 11))); AssertExtensions.assertThrows(ConnectionFailedException.class, () -> stream.read()); ByteBuffer read = stream.read(); assertEquals(ByteBuffer.wrap(data), read); }
@Test(expected = EndOfSegmentException.class) public void testReadWithEndOffset() throws Exception { byte[] data = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); int wireDataSize = wireData.remaining(); //size of the data with header size. AsyncSegmentInputStream mockAsyncInputStream = mock(AsyncSegmentInputStream.class); when(mockAsyncInputStream.read(0, wireDataSize)) .thenReturn(CompletableFuture.completedFuture(new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, wireData.slice()))); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(mockAsyncInputStream, 0, wireDataSize, SegmentInputStreamImpl.DEFAULT_BUFFER_SIZE); ByteBuffer read = stream.read(); assertEquals(ByteBuffer.wrap(data), read); //verify we are reading the data. verify(mockAsyncInputStream, times(1)).read(0L, wireDataSize); //ensure there is one invocation. stream.read(); // this should throw EndOfSegmentExceptiono as we have reached the endOffset }
@Test public void testStreamTruncated() throws EndOfSegmentException { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 6); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, 2))); fakeNetwork.completeExceptionally(1, new SegmentTruncatedException()); fakeNetwork.complete(2, new WireCommands.SegmentRead(segment.getScopedName(), 2, false, false, ByteBufferUtils.slice(wireData, 2, 7))); fakeNetwork.complete(3, new WireCommands.SegmentRead(segment.getScopedName(), 9, false, false, ByteBufferUtils.slice(wireData, 9, 2))); fakeNetwork.complete(4, new WireCommands.SegmentRead(segment.getScopedName(), 11, false, false, ByteBufferUtils.slice(wireData, 11, wireData.capacity() - 11))); AssertExtensions.assertThrows(SegmentTruncatedException.class, () -> stream.read()); AssertExtensions.assertThrows(SegmentTruncatedException.class, () -> stream.read()); }
@Test public void testRead() { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 3); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); ByteBuffer read = assertBlocks(() -> stream.read(), () -> fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, wireData.slice()))); assertEquals(ByteBuffer.wrap(data), read); read = assertBlocks(() -> stream .read(), () -> fakeNetwork.complete(1, new WireCommands.SegmentRead(segment.getScopedName(), wireData.capacity(), false, false, wireData.slice()))); assertEquals(ByteBuffer.wrap(data), read); }
@Test public void testSmallerThanNeededRead() throws EndOfSegmentException, SegmentTruncatedException { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 5); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, 2))); fakeNetwork.complete(1, new WireCommands.SegmentRead(segment.getScopedName(), 2, false, false, ByteBufferUtils.slice(wireData, 2, 7))); fakeNetwork.complete(2, new WireCommands.SegmentRead(segment.getScopedName(), 9, false, false, ByteBufferUtils.slice(wireData, 9, 2))); fakeNetwork.complete(3, new WireCommands.SegmentRead(segment.getScopedName(), 11, false, false, ByteBufferUtils.slice(wireData, 11, wireData.capacity() - 11))); ByteBuffer read = stream.read(); assertEquals(ByteBuffer.wrap(data), read); }
@Test public void testBlockingEndOfSegment() { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 2); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); assertBlocks(() -> { assertEquals(ByteBuffer.wrap(data), stream.read()); }, () -> { fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, 0))); fakeNetwork.complete(1, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, true, wireData.slice())); }); }
@Test public void testSegmentRead() throws IOException { testCommand(new WireCommands.SegmentRead(testString1, l, true, false, buffer)); }
@Test(timeout = 5000) public void testClose() { byte[] data = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; ByteBuffer wireData = createEventFromData(data); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 2); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, false, ByteBufferUtils.slice(wireData, 0, wireData.remaining()))); EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); stream.close(); AssertExtensions.assertThrows(ObjectClosedException.class, () -> stream.read()); }
@Test public void testStreamTruncatedWithPartialEvent() throws EndOfSegmentException { ByteBuffer trailingData = ByteBuffer.wrap(new byte[] {0, 1}); TestAsyncSegmentInputStream fakeNetwork = new TestAsyncSegmentInputStream(segment, 1); @Cleanup EventSegmentReaderImpl stream = SegmentInputStreamFactoryImpl.getEventSegmentReader(fakeNetwork, 0); fakeNetwork.complete(0, new WireCommands.SegmentRead(segment.getScopedName(), 0, false, true, trailingData.slice())); AssertExtensions.assertThrows(EndOfSegmentException.class, () -> stream.read()); }