private TxnOffsetCommitHandler txnOffsetCommitHandler(TransactionalRequestResult result, Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId) { for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) { OffsetAndMetadata offsetAndMetadata = entry.getValue(); CommittedOffset committedOffset = new CommittedOffset(offsetAndMetadata.offset(), offsetAndMetadata.metadata(), offsetAndMetadata.leaderEpoch()); pendingTxnOffsetCommits.put(entry.getKey(), committedOffset); } TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(transactionalId, consumerGroupId, producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, pendingTxnOffsetCommits); return new TxnOffsetCommitHandler(result, builder); }
/** * Checks if {@link OffsetAndMetadata} was committed by a {@link KafkaSpout} instance in this topology. * * @param tp The topic partition the commit metadata belongs to. * @param committedOffset {@link OffsetAndMetadata} info committed to Kafka * @param offsetManagers The offset managers. * @return true if this topology committed this {@link OffsetAndMetadata}, false otherwise */ public boolean isOffsetCommittedByThisTopology(TopicPartition tp, OffsetAndMetadata committedOffset, Map<TopicPartition, OffsetManager> offsetManagers) { try { if (processingGuarantee == ProcessingGuarantee.AT_LEAST_ONCE && offsetManagers.containsKey(tp) && offsetManagers.get(tp).hasCommitted()) { return true; } final CommitMetadata committedMetadata = JSON_MAPPER.readValue(committedOffset.metadata(), CommitMetadata.class); return committedMetadata.getTopologyId().equals(context.getStormId()); } catch (IOException e) { LOG.warn("Failed to deserialize expected commit metadata [{}]." + " This error is expected to occur once per partition, if the last commit to each partition" + " was by an earlier version of the KafkaSpout, or by a process other than the KafkaSpout. " + "Defaulting to behavior compatible with earlier version", committedOffset); LOG.trace("", e); return false; } }
offsetAndMetadata.leaderEpoch(), offsetAndMetadata.metadata()));
@Test public void testAtMostOnceModeCommitsBeforeEmit() throws Exception { //At-most-once mode must commit tuples before they are emitted to the topology to ensure that a spout crash won't cause replays. KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .build(); KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); //The spout should have emitted the tuple, and must have committed it before emit InOrder inOrder = inOrder(consumerMock, collectorMock); inOrder.verify(consumerMock).poll(anyLong()); inOrder.verify(consumerMock).commitSync(commitCapture.capture()); inOrder.verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList()); CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE); Map<TopicPartition, OffsetAndMetadata> committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(0L)); assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); }
@Test public void testNoGuaranteeModeCommitsPolledTuples() throws Exception { //When using the no guarantee mode, the spout must commit tuples periodically, regardless of whether they've been acked KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE) .setTupleTrackingEnforced(true) .build(); try (SimulatedTime time = new SimulatedTime()) { KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); ArgumentCaptor<KafkaSpoutMessageId> msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture()); assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue())); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + spoutConfig.getOffsetsCommitPeriodMs()); spout.nextTuple(); verify(consumerMock).commitAsync(commitCapture.capture(), isNull()); CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE); Map<TopicPartition, OffsetAndMetadata> committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(1L)); assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); } }
@Override public Long committedSafeOffset(TopicPartition tp) { OffsetAndMetadata rawOffsetAndMetadata = _kafkaConsumer.committed(tp); if (rawOffsetAndMetadata == null || rawOffsetAndMetadata.metadata().isEmpty()) { return null; } return rawOffsetAndMetadata.offset(); }
public static Map<TopicPartition, OffsetAndMetadata> from(Map<org.apache.kafka.common.TopicPartition, org.apache.kafka.clients.consumer.OffsetAndMetadata> offsets) { return offsets.entrySet().stream().collect(Collectors.toMap( e -> new TopicPartition(e.getKey().topic(), e.getKey().partition()), e -> new OffsetAndMetadata(e.getValue().offset(), e.getValue().metadata())) ); }
public static OffsetAndMetadata from(org.apache.kafka.clients.consumer.OffsetAndMetadata offsetAndMetadata) { if (offsetAndMetadata != null) { return new OffsetAndMetadata(offsetAndMetadata.offset(), offsetAndMetadata.metadata()); } else { return null; } }
/** * Checks if {@link OffsetAndMetadata} was committed by a {@link KafkaSpout} instance in this topology. * * @param tp The topic partition the commit metadata belongs to. * @param committedOffset {@link OffsetAndMetadata} info committed to Kafka * @param offsetManagers The offset managers. * @return true if this topology committed this {@link OffsetAndMetadata}, false otherwise */ public boolean isOffsetCommittedByThisTopology(TopicPartition tp, OffsetAndMetadata committedOffset, Map<TopicPartition, OffsetManager> offsetManagers) { try { if (processingGuarantee == ProcessingGuarantee.AT_LEAST_ONCE && offsetManagers.containsKey(tp) && offsetManagers.get(tp).hasCommitted()) { return true; } final CommitMetadata committedMetadata = JSON_MAPPER.readValue(committedOffset.metadata(), CommitMetadata.class); return committedMetadata.getTopologyId().equals(context.getStormId()); } catch (IOException e) { LOG.warn("Failed to deserialize expected commit metadata [{}]." + " This error is expected to occur once per partition, if the last commit to each partition" + " was by an earlier version of the KafkaSpout, or by a process other than the KafkaSpout. " + "Defaulting to behavior compatible with earlier version", committedOffset); LOG.trace("", e); return false; } }
@Override public void onComplete(Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap, Exception e) { if (_userCallback != null) { Map<TopicPartition, OffsetAndMetadata> userOffsetMap = topicPartitionOffsetAndMetadataMap; if (topicPartitionOffsetAndMetadataMap != null) { userOffsetMap = new HashMap<>(); for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : topicPartitionOffsetAndMetadataMap.entrySet()) { String rawMetadata = entry.getValue().metadata(); long userOffset = LiKafkaClientsUtils.offsetFromWrappedMetadata(rawMetadata); String userMetadata = LiKafkaClientsUtils.metadataFromWrappedMetadata(rawMetadata); userOffsetMap.put(entry.getKey(), new OffsetAndMetadata(userOffset, userMetadata)); } } _userCallback.onComplete(userOffsetMap, e); } }
if (committed == null || committed.metadata() == null || committed.metadata().isEmpty()) { checkState( nextId == 0 && writerId == null, ShardMetadata metadata = JSON_MAPPER.readValue(committed.metadata(), ShardMetadata.class); + "This mostly indicates groupId '%s' is used else where or in earlier runs. " + "Try another group id. Metadata for this shard on Kafka : '%s'", shard, spec.getSinkGroupId(), committed.metadata())); "Committed sequence id can not be lower than %s, partition metadata : %s", nextId - 1, committed.metadata());
consumerHighWatermark = LiKafkaClientsUtils.offsetFromWrappedMetadata(offsetAndMetadata.metadata());
offsets.forEach((topicPartition, offsetAndMetadata) -> { kafkaOffsets.put(new TopicPartition(topicPartition.topic(), topicPartition.partition()), new io.vertx.kafka.client.consumer.OffsetAndMetadata(offsetAndMetadata.offset() + 1, offsetAndMetadata.metadata())); });
if (committed == null || committed.metadata() == null || committed.metadata().isEmpty()) { checkState( nextId == 0 && writerId == null, ShardMetadata metadata = JSON_MAPPER.readValue(committed.metadata(), ShardMetadata.class); + "This mostly indicates groupId '%s' is used else where or in earlier runs. " + "Try another group id. Metadata for this shard on Kafka : '%s'", shard, spec.getSinkGroupId(), committed.metadata())); "Committed sequence id can not be lower than %s, partition metadata : %s", nextId - 1, committed.metadata());
private OffsetAndMetadata committedMain(TopicPartition partition, Duration timeout) { // Not handling large message here. The committed will be the actual committed value. // The returned metadata includes the user committed offset and the user committed metadata, separated by the // first comma. OffsetAndMetadata offsetAndMetadata; if (timeout == null) { offsetAndMetadata = _kafkaConsumer.committed(partition); } else { offsetAndMetadata = _kafkaConsumer.committed(partition, timeout); } if (offsetAndMetadata != null) { String rawMetadata = offsetAndMetadata.metadata(); Long userOffset = LiKafkaClientsUtils.offsetFromWrappedMetadata(rawMetadata); String userMetadata; if (userOffset == null) { userOffset = offsetAndMetadata.offset(); userMetadata = offsetAndMetadata.metadata(); } else { userMetadata = LiKafkaClientsUtils.metadataFromWrappedMetadata(rawMetadata); } offsetAndMetadata = new OffsetAndMetadata(userOffset, userMetadata); } return offsetAndMetadata; }
String wrappedMetadata = LiKafkaClientsUtils.wrapMetadataWithOffset(offsetsToCommit.get(tp).metadata(), highWatermarkToCommit); OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(safeOffsetToCommit, wrappedMetadata); safeOffsetsToCommit.put(tp, offsetAndMetadata);
@Override public void seekToCommitted(Collection<TopicPartition> partitions) { for (TopicPartition tp : partitions) { OffsetAndMetadata offsetAndMetadata = _kafkaConsumer.committed(tp); if (offsetAndMetadata == null) { throw new NoOffsetForPartitionException(tp); } _kafkaConsumer.seek(tp, offsetAndMetadata.offset()); _consumerRecordsProcessor.clear(tp); Long hw = LiKafkaClientsUtils.offsetFromWrappedMetadata(offsetAndMetadata.metadata()); if (hw == null) { hw = offsetAndMetadata.offset(); } _consumerRecordsProcessor.setPartitionConsumerHighWaterMark(tp, hw); } }
OffsetAndMetadata offsetAndMetadata = consumer.committed(tp); assertEquals(offsetAndMetadata.offset(), 0L); assertEquals(offsetAndMetadata.metadata(), ""); OffsetAndMetadata offsetAndMetadata = consumer.committed(tp); assertEquals(offsetAndMetadata.offset(), 1L); assertEquals(offsetAndMetadata.metadata(), "test"); OffsetAndMetadata offsetAndMetadata = consumer.committed(tp); assertEquals(offsetAndMetadata.offset(), 2L); assertEquals(offsetAndMetadata.metadata(), "test,test");