private void failConnection(Throwable e) { log.info("Failing connection for writer {} with exception {}", writerId, e.toString()); state.failConnection(Exceptions.unwrap(e)); reconnect(); }
/** * Establish a connection and wait for it to be setup. (Retries built in) */ CompletableFuture<ClientConnection> getConnection() throws SegmentSealedException { if (state.isClosed()) { throw new IllegalStateException("SegmentOutputStream is already closed", state.getException()); } if (state.isAlreadySealed()) { throw new SegmentSealedException(this.segmentName); } if (state.getConnection() == null) { reconnect(); } CompletableFuture<ClientConnection> future = new CompletableFuture<>(); state.setupConnection.register(future); return future; }
} catch (ConnectionFailedException e) { log.warn("Connection " + writerId + " failed due to: ", e); reconnect(); // As the message is inflight, this will perform the retransmission.
@Test(timeout = 10000) public void testConnectAndFailedSetupAppend() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); ScheduledExecutorService executor = mock(ScheduledExecutorService.class); implementAsDirectExecutor(executor); // Ensure task submitted to executor is run inline. cf.setExecutor(executor); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); doThrow(ConnectionFailedException.class).doNothing().when(connection).send(any(SetupAppend.class)); cf.provideConnection(uri, connection); @Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); verify(connection).send(new SetupAppend(2, cid, SEGMENT, "")); }
@Test(timeout = 10000) public void testConnectAndConnectionDrop() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); ScheduledExecutorService executor = mock(ScheduledExecutorService.class); implementAsDirectExecutor(executor); // Ensure task submitted to executor is run inline. cf.setExecutor(executor); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); @Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).connectionDropped(); // simulate a connection dropped //Ensure setup Append is invoked on the executor. verify(connection).send(new SetupAppend(2, cid, SEGMENT, "")); }
@Test public void testOverSizedWriteFails() throws ConnectionFailedException, SegmentSealedException { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); @Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); ByteBuffer data = ByteBuffer.allocate(PendingEvent.MAX_WRITE_SIZE + 1); CompletableFuture<Void> acked = new CompletableFuture<>(); try { output.write(PendingEvent.withoutHeader("routingKey", data, acked)); fail("Did not throw"); } catch (IllegalArgumentException e) { // expected } assertEquals(false, acked.isDone()); verifyNoMoreInteractions(connection); }
InOrder order = Mockito.inOrder(connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0));
@Test(timeout = 10000) public void testConnectAndFailedSetupAppendDueToTruncation() throws Exception { AtomicBoolean callbackInvoked = new AtomicBoolean(); Consumer<Segment> resendToSuccessorsCallback = segment -> { callbackInvoked.set(true); }; UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); ScheduledExecutorService executor = mock(ScheduledExecutorService.class); implementAsDirectExecutor(executor); // Ensure task submitted to executor is run inline. cf.setExecutor(executor); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); @Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, resendToSuccessorsCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).noSuchSegment(new WireCommands.NoSuchSegment(1, SEGMENT, "SomeException")); CompletableFuture<ClientConnection> connectionFuture = output.getConnection(); assertThrows(NoSuchSegmentException.class, () -> Futures.getThrowingException(connectionFuture)); assertTrue(callbackInvoked.get()); }
@Test(timeout = 10000) public void testConnectAndSend() throws SegmentSealedException, ConnectionFailedException { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); sendAndVerifyEvent(cid, connection, output, getBuffer("test"), 1); verifyNoMoreInteractions(connection); }
@Test(timeout = 10000) public void testConditionalSend() throws SegmentSealedException, ConnectionFailedException { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); sendAndVerifyEvent(cid, connection, output, getBuffer("test"), 1); verifyNoMoreInteractions(connection); }
SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0));
SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); output.write(PendingEvent.withoutHeader(null, getBuffer("test1"), new CompletableFuture<>()));
@Test(timeout = 10000) public void testSealedBeforeFlush() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); InOrder order = Mockito.inOrder(connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); ByteBuffer data = getBuffer("test"); CompletableFuture<Void> ack = new CompletableFuture<>(); output.write(PendingEvent.withoutHeader(null, data, ack)); order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null)); assertEquals(false, ack.isDone()); cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(1, SEGMENT, "SomeException")); output.getUnackedEventsOnSeal(); // this is invoked by the segmentSealedCallback. AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }
@Test(timeout = 10000) public void testConnectWithMultipleFailures() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); ScheduledExecutorService executor = mock(ScheduledExecutorService.class); implementAsDirectExecutor(executor); // Ensure task submitted to executor is run inline. cf.setExecutor(executor); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); @Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); //simulate a processing Failure and ensure SetupAppend is executed. cf.getProcessor(uri).processingFailure(new IOException()); verify(connection).send(new SetupAppend(2, cid, SEGMENT, "")); cf.getProcessor(uri).connectionDropped(); verify(connection).send(new SetupAppend(3, cid, SEGMENT, "")); cf.getProcessor(uri).wrongHost(new WireCommands.WrongHost(3, SEGMENT, "newHost", "SomeException")); verify(connection).send(new SetupAppend(4, cid, SEGMENT, "")); }
@Test(timeout = 10000) public void testClose() throws ConnectionFailedException, SegmentSealedException { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); ByteBuffer data = getBuffer("test"); CompletableFuture<Void> acked = new CompletableFuture<>(); output.write(PendingEvent.withoutHeader(null, data, acked)); verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null)); assertEquals(false, acked.isDone()); AssertExtensions.assertBlocks(() -> output.close(), () -> cf.getProcessor(uri).dataAppended(new WireCommands.DataAppended(cid, 1, 0))); assertEquals(false, acked.isCompletedExceptionally()); assertEquals(true, acked.isDone()); verify(connection, Mockito.atMost(1)).send(new WireCommands.KeepAlive()); verify(connection).close(); verifyNoMoreInteractions(connection); }
output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, TXN_SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, TXN_SEGMENT, cid, 0));
@Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0));
@Test(timeout = 10000) public void testFlushDuringTransactionAbort() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); InOrder order = Mockito.inOrder(connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(TXN_SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, TXN_SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, TXN_SEGMENT, cid, 0)); ByteBuffer data = getBuffer("test"); // Write an Event. CompletableFuture<Void> ack = new CompletableFuture<>(); output.write(PendingEvent.withoutHeader(null, data, ack)); order.verify(connection).send(new Append(TXN_SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null)); assertFalse(ack.isDone()); //writer is not complete until a response from Segment Store is received. // Validate that flush() is blocking until there is a response from Segment Store. AssertExtensions.assertBlocks(() -> { // A flush() should throw a SegmentSealedException. AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }, () -> { // Simulate a NoSuchSegment response from SegmentStore due to a Transaction abort. cf.getProcessor(uri).noSuchSegment(new WireCommands.NoSuchSegment(1, TXN_SEGMENT, "SomeException")); }); AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }
@Test(timeout = 10000) public void testNoSuchSegment() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); InOrder order = Mockito.inOrder(connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); ByteBuffer data = getBuffer("test"); //Write an Event. CompletableFuture<Void> ack = new CompletableFuture<>(); output.write(PendingEvent.withoutHeader(null, data, ack)); order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null)); assertEquals(false, ack.isDone()); //writer is not complete until a response from Segment Store is received. //Simulate a No Such Segment while waiting on flush. AssertExtensions.assertBlocks(() -> { AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }, () -> { cf.getProcessor(uri).noSuchSegment(new WireCommands.NoSuchSegment(1, SEGMENT, "SomeException")); output.getUnackedEventsOnSeal(); }); AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }
@Test(timeout = 10000) public void testSealedAfterFlush() throws Exception { UUID cid = UUID.randomUUID(); PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT); MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl(); cf.setExecutor(executorService()); MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf); ClientConnection connection = mock(ClientConnection.class); cf.provideConnection(uri, connection); InOrder order = Mockito.inOrder(connection); SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, ""); output.reconnect(); order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, "")); cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0)); ByteBuffer data = getBuffer("test"); CompletableFuture<Void> ack = new CompletableFuture<>(); output.write(PendingEvent.withoutHeader(null, data, ack)); order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null)); assertEquals(false, ack.isDone()); AssertExtensions.assertBlocks(() -> { AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }, () -> { cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(1, SEGMENT, "SomeException")); output.getUnackedEventsOnSeal(); }); AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush()); }