/** * Get the offset of the <i>next record</i> that will be fetched (if a record with that offset exists). * This method may issue a remote call to the server if there is no current position for the given partition. * <p> * This call will block until either the position could be determined or an unrecoverable error is * encountered (in which case it is thrown to the caller), or the timeout specified by {@code default.api.timeout.ms} expires * (in which case a {@link org.apache.kafka.common.errors.TimeoutException} is thrown to the caller). * * @param partition The partition to get the position for * @return The current position of the consumer (that is, the offset of the next record to be fetched) * @throws IllegalStateException if the provided TopicPartition is not assigned to this consumer * @throws org.apache.kafka.clients.consumer.InvalidOffsetException if no offset is currently defined for * the partition * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this * function is called * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while * this function is called * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the topic or to the * configured groupId. See the exception for more details * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors * @throws org.apache.kafka.common.errors.TimeoutException if the position cannot be determined before the * timeout specified by {@code default.api.timeout.ms} expires */ @Override public long position(TopicPartition partition) { return position(partition, Duration.ofMillis(defaultApiTimeoutMs)); }
private boolean sampleLoadingFinished(Map<TopicPartition, Long> endOffsets) { for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) { long position = _consumer.position(entry.getKey()); if (position < entry.getValue()) { LOG.debug("Partition {} is still lagging. Current position: {}, LEO: {}", entry.getKey(), position, entry.getValue()); return false; } } return true; }
@Override public Long getPosition(StreamPartition<Integer> partition) { return consumer.position(new TopicPartition(partition.getStream(), partition.getPartitionId())); }
public static long getEarliestOffset(KafkaConsumer consumer, String topic, int partitionId) { TopicPartition topicPartition = new TopicPartition(topic, partitionId); consumer.assign(Arrays.asList(topicPartition)); consumer.seekToBeginning(Arrays.asList(topicPartition)); return consumer.position(topicPartition); }
public static long getLatestOffset(KafkaConsumer consumer, String topic, int partitionId) { TopicPartition topicPartition = new TopicPartition(topic, partitionId); consumer.assign(Arrays.asList(topicPartition)); consumer.seekToEnd(Arrays.asList(topicPartition)); return consumer.position(topicPartition); }
@Override public Long getLatestSequenceNumber(StreamPartition<Integer> partition) { Long currPos = consumer.position(new TopicPartition(partition.getStream(), partition.getPartitionId())); seekToLatest(Collections.singleton(partition)); Long nextPos = consumer.position(new TopicPartition(partition.getStream(), partition.getPartitionId())); seek(partition, currPos); return nextPos; }
@Override public Long getEarliestSequenceNumber(StreamPartition<Integer> partition) { Long currPos = consumer.position(new TopicPartition(partition.getStream(), partition.getPartitionId())); seekToEarliest(Collections.singleton(partition)); Long nextPos = consumer.position(new TopicPartition(partition.getStream(), partition.getPartitionId())); seek(partition, currPos); return nextPos; }
@Override public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { waitClientRunning(); if (!running) { return Lists.newArrayList(); } ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); currentOffsets.clear(); for (TopicPartition topicPartition : records.partitions()) { currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition)); } if (!records.isEmpty()) { List<Message> messages = new ArrayList<>(); for (ConsumerRecord<String, Message> record : records) { messages.add(record.value()); } return messages; } return Lists.newArrayList(); }
@Override public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { waitClientRunning(); if (!running) { return Lists.newArrayList(); } ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout)); currentOffsets.clear(); for (TopicPartition topicPartition : records.partitions()) { currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition)); } if (!records.isEmpty()) { List<FlatMessage> flatMessages = new ArrayList<>(); for (ConsumerRecord<String, String> record : records) { String flatMessageJson = record.value(); FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); flatMessages.add(flatMessage); } return flatMessages; } return Lists.newArrayList(); }
final long leastAvailableOffset = consumer.position(topicPartition);
@Test public void testAtMostOnceModeCommitsBeforeEmit() throws Exception { //At-most-once mode must commit tuples before they are emitted to the topology to ensure that a spout crash won't cause replays. KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .build(); KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); //The spout should have emitted the tuple, and must have committed it before emit InOrder inOrder = inOrder(consumerMock, collectorMock); inOrder.verify(consumerMock).poll(anyLong()); inOrder.verify(consumerMock).commitSync(commitCapture.capture()); inOrder.verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList()); CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE); Map<TopicPartition, OffsetAndMetadata> committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(0L)); assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); }
@Test public void testNoGuaranteeModeCommitsPolledTuples() throws Exception { //When using the no guarantee mode, the spout must commit tuples periodically, regardless of whether they've been acked KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE) .setTupleTrackingEnforced(true) .build(); try (SimulatedTime time = new SimulatedTime()) { KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); ArgumentCaptor<KafkaSpoutMessageId> msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture()); assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue())); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + spoutConfig.getOffsetsCommitPeriodMs()); spout.nextTuple(); verify(consumerMock).commitAsync(commitCapture.capture(), isNull()); CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE); Map<TopicPartition, OffsetAndMetadata> committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(1L)); assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); } }
@Test public void verifyNoCoordinatorLookupForManualAssignmentWithSeek() { Time time = new MockTime(); Metadata metadata = createMetadata(); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); PartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, true); consumer.assign(singleton(tp0)); consumer.seekToBeginning(singleton(tp0)); // there shouldn't be any need to lookup the coordinator or fetch committed offsets. // we just lookup the starting position and send the record fetch. client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 50L))); client.prepareResponse(fetchResponse(tp0, 50L, 5)); ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1)); assertEquals(5, records.count()); assertEquals(55L, consumer.position(tp0)); consumer.close(Duration.ofMillis(0)); }
@Test public void testResetToCommittedOffset() { Time time = new MockTime(); Metadata metadata = createMetadata(); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); PartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, OffsetResetStrategy.NONE, true, groupId); consumer.assign(singletonList(tp0)); client.prepareResponseFrom(new FindCoordinatorResponse(Errors.NONE, node), node); Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, 539L), Errors.NONE), coordinator); consumer.poll(Duration.ZERO); assertEquals(539L, consumer.position(tp0)); }
@Test public void testResetUsingAutoResetPolicy() { Time time = new MockTime(); Metadata metadata = createMetadata(); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 1)); Node node = metadata.fetch().nodes().get(0); PartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, OffsetResetStrategy.LATEST, true, groupId); consumer.assign(singletonList(tp0)); client.prepareResponseFrom(new FindCoordinatorResponse(Errors.NONE, node), node); Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, -1L), Errors.NONE), coordinator); client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 50L))); consumer.poll(Duration.ZERO); assertEquals(50L, consumer.position(tp0)); }
assertEquals(0, consumer.position(tp0));
assertEquals(11L, consumer.position(tp0));
assertEquals(11L, consumer.position(tp0));