private void assertSendFailure(Class<? extends RuntimeException> expectedError) throws Exception { Future<RecordMetadata> future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); assertTrue(future.isDone()); try { future.get(); fail("Future should have raised " + expectedError.getSimpleName()); } catch (ExecutionException e) { assertTrue(expectedError.isAssignableFrom(e.getCause().getClass())); } }
private void doInitTransactions(TransactionManager transactionManager, ProducerIdAndEpoch producerIdAndEpoch) { transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE); sender.run(time.milliseconds()); sender.run(time.milliseconds()); prepareInitPidResponse(Errors.NONE, producerIdAndEpoch.producerId, producerIdAndEpoch.epoch); sender.run(time.milliseconds()); assertTrue(transactionManager.hasProducerId()); }
private void prepareAndReceiveInitProducerId(long producerId, Errors error) { short producerEpoch = 0; if (error != Errors.NONE) producerEpoch = RecordBatch.NO_PRODUCER_EPOCH; client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { return body instanceof InitProducerIdRequest && ((InitProducerIdRequest) body).transactionalId() == null; } }, new InitProducerIdResponse(0, error, producerId, producerEpoch)); sender.run(time.milliseconds()); }
@Test public void testDisconnectAndRetry() { // This is called from the initTransactions method in the producer as the first order of business. // It finds the coordinator and then gets a PID. transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, true, CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator, connection lost. prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator sender.run(time.milliseconds()); assertEquals(brokerNode, transactionManager.coordinator(CoordinatorType.TRANSACTION)); }
private void doInitTransactions(long pid, short epoch) { transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator sender.run(time.milliseconds()); assertEquals(brokerNode, transactionManager.coordinator(CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.NONE, false, pid, epoch); sender.run(time.milliseconds()); // get pid. assertTrue(transactionManager.hasProducerId()); }
private void verifyAddPartitionsFailsWithPartitionLevelError(final Errors error) throws InterruptedException { final long pid = 1L; final short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future; assertFalse(responseFuture.isDone()); prepareAddPartitionsToTxn(tp0, error); sender.run(time.milliseconds()); // attempt send addPartitions. assertTrue(transactionManager.hasError()); assertFalse(transactionManager.transactionContainsPartition(tp0)); }
@Test public void testTransactionalIdAuthorizationFailureInFindCoordinator() { TransactionalRequestResult initPidResult = transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED, false, CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator sender.run(time.milliseconds()); assertTrue(transactionManager.hasError()); assertTrue(transactionManager.lastError() instanceof TransactionalIdAuthorizationException); sender.run(time.milliseconds()); // one more run to fail the InitProducerId future assertTrue(initPidResult.isCompleted()); assertFalse(initPidResult.isSuccessful()); assertTrue(initPidResult.error() instanceof TransactionalIdAuthorizationException); assertFatalError(TransactionalIdAuthorizationException.class); }
@Test public void testTransactionalSplitBatchAndSend() throws Exception { ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(123456L, (short) 0); TopicPartition tp = new TopicPartition("testSplitBatchAndSend", 1); TransactionManager txnManager = new TransactionManager(logContext, "testSplitBatchAndSend", 60000, 100); setupWithTransactionState(txnManager); doInitTransactions(txnManager, producerIdAndEpoch); txnManager.beginTransaction(); txnManager.maybeAddPartitionToTransaction(tp); client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp, Errors.NONE))); sender.run(time.milliseconds()); testSplitBatchAndSend(txnManager, producerIdAndEpoch, tp); }
@Test public void testSenderShutdownWithPendingAddPartitions() throws Exception { long pid = 13131L; short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); FutureRecordMetadata sendFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future; prepareAddPartitionsToTxn(tp0, Errors.NONE); prepareProduceResponse(Errors.NONE, pid, epoch); sender.initiateClose(); sender.run(); assertTrue(sendFuture.isDone()); }
@Test public void testTransactionalIdAuthorizationFailureInInitProducerId() { final long pid = 13131L; TransactionalRequestResult initPidResult = transactionManager.initializeTransactions(); prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.TRANSACTION, transactionalId); sender.run(time.milliseconds()); // find coordinator sender.run(time.milliseconds()); assertEquals(brokerNode, transactionManager.coordinator(CoordinatorType.TRANSACTION)); prepareInitPidResponse(Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED, false, pid, RecordBatch.NO_PRODUCER_EPOCH); sender.run(time.milliseconds()); assertTrue(transactionManager.hasError()); assertTrue(initPidResult.isCompleted()); assertFalse(initPidResult.isSuccessful()); assertTrue(initPidResult.error() instanceof TransactionalIdAuthorizationException); assertFatalError(TransactionalIdAuthorizationException.class); }
@Test public void testUnsupportedFindCoordinator() { transactionManager.initializeTransactions(); client.prepareUnsupportedVersionResponse(body -> { FindCoordinatorRequest findCoordinatorRequest = (FindCoordinatorRequest) body; assertEquals(findCoordinatorRequest.coordinatorType(), CoordinatorType.TRANSACTION); assertEquals(findCoordinatorRequest.coordinatorKey(), transactionalId); return true; }); sender.run(time.milliseconds()); // InitProducerRequest is queued sender.run(time.milliseconds()); // FindCoordinator is queued after peeking InitProducerRequest assertTrue(transactionManager.hasFatalError()); assertTrue(transactionManager.lastError() instanceof UnsupportedVersionException); }
@Test public void shouldNotSendAbortTxnRequestWhenOnlyAddPartitionsRequestFailed() { final long pid = 13131L; final short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); prepareAddPartitionsToTxnResponse(Errors.TOPIC_AUTHORIZATION_FAILED, tp0, epoch, pid); sender.run(time.milliseconds()); // Send AddPartitionsRequest TransactionalRequestResult abortResult = transactionManager.beginAbort(); assertFalse(abortResult.isCompleted()); sender.run(time.milliseconds()); assertTrue(abortResult.isCompleted()); assertTrue(abortResult.isSuccessful()); }
@Test public void testTransactionalIdAuthorizationFailureInAddPartitions() { final long pid = 13131L; final short epoch = 1; final TopicPartition tp = new TopicPartition("foo", 0); doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp); prepareAddPartitionsToTxn(tp, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED); sender.run(time.milliseconds()); assertTrue(transactionManager.hasError()); assertTrue(transactionManager.lastError() instanceof TransactionalIdAuthorizationException); assertFatalError(TransactionalIdAuthorizationException.class); }
@Test public void testIsSendToPartitionAllowedWithInFlightPartitionAddAfterFatalError() { final long pid = 13131L; final short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); // Send the AddPartitionsToTxn request and leave it in-flight sender.run(time.milliseconds()); transactionManager.transitionToFatalError(new KafkaException()); assertFalse(transactionManager.isSendToPartitionAllowed(tp0)); assertTrue(transactionManager.hasFatalError()); }
@Test public void testIsSendToPartitionAllowedWithInFlightPartitionAddAfterAbortableError() { final long pid = 13131L; final short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); // Send the AddPartitionsToTxn request and leave it in-flight sender.run(time.milliseconds()); transactionManager.transitionToAbortableError(new KafkaException()); assertFalse(transactionManager.isSendToPartitionAllowed(tp0)); assertTrue(transactionManager.hasAbortableError()); }
@Test public void testAddPartitionToTransactionRetainsRetryBackoffForRegularRetriableError() { long pid = 13131L; short epoch = 1; TopicPartition partition = new TopicPartition("foo", 0); doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(partition); assertTrue(transactionManager.hasPartitionsToAdd()); assertFalse(transactionManager.isPartitionAdded(partition)); assertTrue(transactionManager.isPartitionPendingAdd(partition)); prepareAddPartitionsToTxn(partition, Errors.COORDINATOR_NOT_AVAILABLE); sender.run(time.milliseconds()); TransactionManager.TxnRequestHandler handler = transactionManager.nextRequestHandler(false); assertNotNull(handler); assertEquals(DEFAULT_RETRY_BACKOFF_MS, handler.retryBackoffMs()); }
@Test public void testAddPartitionToTransactionOverridesRetryBackoffForConcurrentTransactions() { long pid = 13131L; short epoch = 1; TopicPartition partition = new TopicPartition("foo", 0); doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(partition); assertTrue(transactionManager.hasPartitionsToAdd()); assertFalse(transactionManager.isPartitionAdded(partition)); assertTrue(transactionManager.isPartitionPendingAdd(partition)); prepareAddPartitionsToTxn(partition, Errors.CONCURRENT_TRANSACTIONS); sender.run(time.milliseconds()); TransactionManager.TxnRequestHandler handler = transactionManager.nextRequestHandler(false); assertNotNull(handler); assertEquals(20, handler.retryBackoffMs()); }
@Test public void testIsSendToPartitionAllowedWithAddedPartitionAfterAbortableError() { final long pid = 13131L; final short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); sender.run(time.milliseconds()); assertFalse(transactionManager.hasPartitionsToAdd()); transactionManager.transitionToAbortableError(new KafkaException()); assertTrue(transactionManager.isSendToPartitionAllowed(tp0)); assertTrue(transactionManager.hasAbortableError()); }
@Test public void testEndTxnNotSentIfIncompleteBatches() { long pid = 13131L; short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); prepareAddPartitionsToTxn(tp0, Errors.NONE); sender.run(time.milliseconds()); assertTrue(transactionManager.isPartitionAdded(tp0)); transactionManager.beginCommit(); assertNull(transactionManager.nextRequestHandler(true)); assertTrue(transactionManager.nextRequestHandler(false).isEndTxn()); }
@Test public void testIsSendToPartitionAllowedWithAddedPartitionAfterFatalError() { final long pid = 13131L; final short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid); sender.run(time.milliseconds()); assertFalse(transactionManager.hasPartitionsToAdd()); transactionManager.transitionToFatalError(new KafkaException()); assertFalse(transactionManager.isSendToPartitionAllowed(tp0)); assertTrue(transactionManager.hasFatalError()); }