Refine search
private static Map<TopicPartition, OffsetAndMetadata> getOffsetsAtPath( CuratorFramework curator, ObjectMapper objectMapper, String partitionsRoot) throws Exception { Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>(); if (curator.checkExists().forPath(partitionsRoot) == null) { throw new RuntimeException("No such path " + partitionsRoot); } List<String> partitionPaths = curator.getChildren().forPath(partitionsRoot); for (String partitionPath : partitionPaths) { String absPartitionPath = partitionsRoot + "/" + partitionPath; LOG.info("Reading offset data from path {}", absPartitionPath); byte[] partitionBytes = curator.getData().forPath(absPartitionPath); Map<String, Object> partitionMetadata = objectMapper.readValue(partitionBytes, new TypeReference<Map<String, Object>>() { }); String topic = (String) partitionMetadata.get("topic"); int partition = ((Number) partitionMetadata.get("partition")).intValue(); long offset = ((Number) partitionMetadata.get("offset")).longValue(); offsets.put(new TopicPartition(topic, partition), new OffsetAndMetadata(offset)); } return offsets; }
@Override public Long getCommittedOffset(String topicName, int partition) { OffsetAndMetadata committed = offsetClient.committed(new TopicPartition(topicName, partition)); return (committed != null) ? committed.offset() : null; }
private TxnOffsetCommitHandler txnOffsetCommitHandler(TransactionalRequestResult result, Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId) { for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) { OffsetAndMetadata offsetAndMetadata = entry.getValue(); CommittedOffset committedOffset = new CommittedOffset(offsetAndMetadata.offset(), offsetAndMetadata.metadata(), offsetAndMetadata.leaderEpoch()); pendingTxnOffsetCommits.put(entry.getKey(), committedOffset); } TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(transactionalId, consumerGroupId, producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, pendingTxnOffsetCommits); return new TxnOffsetCommitHandler(result, builder); }
/** * Overrides the fetch offsets that the consumer will use on the next {@link #poll(Duration) poll(timeout)}. If this API * is invoked for the same partition more than once, the latest offset will be used on the next poll(). Note that * you may lose data if this API is arbitrarily used in the middle of consumption, to reset the fetch offsets. This * method allows for setting the leaderEpoch along with the desired offset. * * @throws IllegalArgumentException if the provided offset is negative * @throws IllegalStateException if the provided TopicPartition is not assigned to this consumer */ @Override public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { long offset = offsetAndMetadata.offset(); if (offset < 0) { throw new IllegalArgumentException("seek offset must not be a negative number"); } acquireAndEnsureOpen(); try { if (offsetAndMetadata.leaderEpoch().isPresent()) { log.debug("Seeking to offset {} for partition {} with epoch {}", offset, partition, offsetAndMetadata.leaderEpoch().get()); } else { log.debug("Seeking to offset {} for partition {}", offset, partition); } this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); this.subscriptions.seek(partition, offset); } finally { release(); } }
/** * Refresh the committed offsets for provided partitions. * * @param timer Timer bounding how long this method can block * @return true iff the operation completed within the timeout */ public boolean refreshCommittedOffsetsIfNeeded(Timer timer) { final Set<TopicPartition> missingFetchPositions = subscriptions.missingFetchPositions(); final Map<TopicPartition, OffsetAndMetadata> offsets = fetchCommittedOffsets(missingFetchPositions, timer); if (offsets == null) return false; for (final Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) { final TopicPartition tp = entry.getKey(); final long offset = entry.getValue().offset(); log.debug("Setting offset for partition {} to the committed offset {}", tp, offset); entry.getValue().leaderEpoch().ifPresent(epoch -> this.metadata.updateLastSeenEpochIfNewer(entry.getKey(), epoch)); this.subscriptions.seek(tp, offset); } return true; }
consumer.assign(singleton(tp0)); consumer.seekToBeginning(singleton(tp0)); assertEquals(0, consumer.committed(tp0).offset());
@Override public void setCommittedOffset(String topicName, int partition, long offset) { Map<TopicPartition, OffsetAndMetadata> partitionAndOffset = new HashMap<>(); partitionAndOffset.put(new TopicPartition(topicName, partition), new OffsetAndMetadata(offset)); offsetClient.commitSync(partitionAndOffset); }
ConsumerRecords<String, byte[]> records = consumer.poll(duration); it = records.iterator(); tpAndOffsetMetadata.put(new TopicPartition(message.topic(), message.partition()), new OffsetAndMetadata(message.offset() + 1, batchUUID)); consumer.commitSync(tpAndOffsetMetadata); long commitEndTime = System.nanoTime(); counter.addToKafkaCommitTimer((commitEndTime - commitStartTime) / (1000 * 1000));
spout.ack(failedIdReplayCaptor.getValue()); spout.nextTuple(); verify(getKafkaConsumer()).commitSync(commitCapture.capture()); TopicPartition expectedTp = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0); assertThat("Should have committed to the right topic", capturedCommit, Matchers.hasKey(expectedTp)); assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp).offset(), is((long)messageCountExcludingLast));
private void reconsumeRecords() throws InterruptedException { ConsumerRecords<String, byte[]> reconsumeRecords = reconsumer.poll(1000); int retryTimeLevel = getRetryTimeLevelFromTopicName(partition.topic()); Map<TopicPartition, LinkedList<ConsumerRecord<String, byte[]>>> levelRecords = retryRecords.get(retryTimeLevel); LinkedList<ConsumerRecord<String, byte[]>> partitionRecords = levelRecords.get(partition); reconsumer.commitSync(Collections.singletonMap(partitionRecords.getKey(), new OffsetAndMetadata(lastCommitRecord.offset() + 1))); reconsumer.resume(Arrays.asList(partitionRecords.getKey())); } else { reconsumer.pause(Arrays.asList(partitionRecords.getKey()));
private Map<TopicPartition, OffsetAndMetadata> getZookeeperOffsets( KafkaZkClient zkClient, KafkaConsumer<String, byte[]> consumer) { Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>(); List<PartitionInfo> partitions = consumer.partitionsFor(topicStr); for (PartitionInfo partition : partitions) { TopicPartition topicPartition = new TopicPartition(topicStr, partition.partition()); Option<Object> optionOffset = zkClient.getConsumerOffset(groupId, topicPartition); if (optionOffset.nonEmpty()) { Long offset = (Long) optionOffset.get(); OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(offset); offsets.put(topicPartition, offsetAndMetadata); } } return offsets; }
records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, numRecords)); when(consumerMock.poll(anyLong())) .thenReturn(new ConsumerRecords<>(records)); inOrder.verify(consumerMock).commitSync(commitCapture.capture()); inOrder.verify(consumerMock).poll(anyLong()); assertEquals(lastOffset + 1, ((OffsetAndMetadata) (commitCapture.getValue().get(partition))).offset());
@Override public synchronized OffsetAndMetadata committed(TopicPartition partition) { ensureNotClosed(); if (subscriptions.isAssigned(partition)) { return committed.get(partition); } return new OffsetAndMetadata(0); }
TopicPartition myTopicPartition0 = new TopicPartition("my_topic", 0); TopicPartition myTopicPartition1 = new TopicPartition("my_topic", 1); TopicPartition myTopicPartition2 = new TopicPartition("my_topic", 2); assertEquals(10, partitionToOffsetAndMetadata.get(myTopicPartition0).offset()); assertEquals(0, partitionToOffsetAndMetadata.get(myTopicPartition1).offset()); assertEquals(20, partitionToOffsetAndMetadata.get(myTopicPartition2).offset());
@Test public void testAtMostOnceModeCommitsBeforeEmit() throws Exception { //At-most-once mode must commit tuples before they are emitted to the topology to ensure that a spout crash won't cause replays. KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .build(); KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); //The spout should have emitted the tuple, and must have committed it before emit InOrder inOrder = inOrder(consumerMock, collectorMock); inOrder.verify(consumerMock).poll(anyLong()); inOrder.verify(consumerMock).commitSync(commitCapture.capture()); inOrder.verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList()); CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE); Map<TopicPartition, OffsetAndMetadata> committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(0L)); assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); }
.build(), consumerFactory, assignerMock); String topic = SingleTopicKafkaSpoutConfiguration.TOPIC; TopicPartition assignedPartition = new TopicPartition(topic, 1); TopicPartition newPartition = new TopicPartition(topic, 2); when(consumerMock.committed(assignedPartition)).thenReturn(new OffsetAndMetadata(committedOffset)); when(consumerMock.committed(newPartition)).thenReturn(new OffsetAndMetadata(committedOffset)); verify(consumerMock, never()).seek(eq(assignedPartition), anyLong()); verify(consumerMock).seek(newPartition, committedOffset);
@Test public void testNoGuaranteeModeCommitsPolledTuples() throws Exception { //When using the no guarantee mode, the spout must commit tuples periodically, regardless of whether they've been acked KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE) .setTupleTrackingEnforced(true) .build(); try (SimulatedTime time = new SimulatedTime()) { KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); ArgumentCaptor<KafkaSpoutMessageId> msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture()); assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue())); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + spoutConfig.getOffsetsCommitPeriodMs()); spout.nextTuple(); verify(consumerMock).commitAsync(commitCapture.capture(), isNull()); CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE); Map<TopicPartition, OffsetAndMetadata> committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(1L)); assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); } }
if ( logger.isTraceEnabled() ) { logger.trace("Assignment during take: {}", consumerAndRecords.get().consumer.assignment().toString()); ConsumerRecord<String, byte[]> record = consumerAndRecords.get().recordIterator.next(); e = deserializeValue(record.value(), parseAsFlumeEvent); TopicPartition tp = new TopicPartition(record.topic(), record.partition()); OffsetAndMetadata oam = new OffsetAndMetadata(record.offset() + 1, batchUUID); consumerAndRecords.get().saveOffsets(tp,oam);
@Test public void testCommitsFetchedDuringAssign() { long offset1 = 10000; long offset2 = 20000; Time time = new MockTime(); Metadata metadata = createMetadata(); MockClient client = new MockClient(time, metadata); initMetadata(client, Collections.singletonMap(topic, 2)); Node node = metadata.fetch().nodes().get(0); PartitionAssignor assignor = new RoundRobinAssignor(); KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, true); consumer.assign(singletonList(tp0)); // lookup coordinator client.prepareResponseFrom(new FindCoordinatorResponse(Errors.NONE, node), node); Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port()); // fetch offset for one topic client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, offset1), Errors.NONE), coordinator); assertEquals(offset1, consumer.committed(tp0).offset()); consumer.assign(Arrays.asList(tp0, tp1)); // fetch offset for two topics Map<TopicPartition, Long> offsets = new HashMap<>(); offsets.put(tp0, offset1); client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator); assertEquals(offset1, consumer.committed(tp0).offset()); offsets.remove(tp0); offsets.put(tp1, offset2); client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator); assertEquals(offset2, consumer.committed(tp1).offset()); consumer.close(Duration.ofMillis(0)); }