private MetadataSnapshot(SubscriptionState subscription, Cluster cluster) { Map<String, Integer> partitionsPerTopic = new HashMap<>(); for (String topic : subscription.groupSubscription()) partitionsPerTopic.put(topic, cluster.partitionCountForTopic(topic)); this.partitionsPerTopic = partitionsPerTopic; }
@Override public Map<String, Assignment> assign(Cluster metadata, Map<String, Subscription> subscriptions) { Set<String> allSubscribedTopics = new HashSet<>(); for (Map.Entry<String, Subscription> subscriptionEntry : subscriptions.entrySet()) allSubscribedTopics.addAll(subscriptionEntry.getValue().topics()); Map<String, Integer> partitionsPerTopic = new HashMap<>(); for (String topic : allSubscribedTopics) { Integer numPartitions = metadata.partitionCountForTopic(topic); if (numPartitions != null && numPartitions > 0) partitionsPerTopic.put(topic, numPartitions); else log.debug("Skipping assignment for topic {} since no metadata is available", topic); } Map<String, List<TopicPartition>> rawAssignments = assign(partitionsPerTopic, subscriptions); // this class maintains no user data, so just wrap the results Map<String, Assignment> assignments = new HashMap<>(); for (Map.Entry<String, List<TopicPartition>> assignmentEntry : rawAssignments.entrySet()) assignments.put(assignmentEntry.getKey(), new Assignment(assignmentEntry.getValue())); return assignments; }
public static int totalNumPartitions(Cluster cluster) { int totalNumPartitions = 0; for (String topic : cluster.topics()) { totalNumPartitions += cluster.partitionCountForTopic(topic); } return totalNumPartitions; }
if (!prev.partitionCountForTopic(topic).equals(curr.partitionCountForTopic(topic))) { return true;
Integer partitionsCount = cluster.partitionCountForTopic(topic); throw new InvalidTopicException(topic); remainingWaitMs = maxWaitMs - elapsed; partitionsCount = cluster.partitionCountForTopic(topic); } while (partitionsCount == null || (partition != null && partition >= partitionsCount));
assertEquals(metadata.fetch().partitionCountForTopic("topic-1").longValue(), 5); assertFalse(metadata.partitionInfoIfCurrent(tp).isPresent()); assertEquals(metadata.lastSeenLeaderEpoch(tp).get().longValue(), 101); assertEquals(metadata.fetch().partitionCountForTopic("topic-1").longValue(), 5); assertFalse(metadata.partitionInfoIfCurrent(tp).isPresent()); assertEquals(metadata.lastSeenLeaderEpoch(tp).get().longValue(), 101); metadata.update(metadataResponse, 30L); assertNotNull(metadata.fetch().partition(tp)); assertEquals(metadata.fetch().partitionCountForTopic("topic-1").longValue(), 5); assertTrue(metadata.partitionInfoIfCurrent(tp).isPresent()); assertEquals(metadata.lastSeenLeaderEpoch(tp).get().longValue(), 101); assertNull(metadata.fetch().partitionCountForTopic("topic-1")); assertFalse(metadata.partitionInfoIfCurrent(tp).isPresent());
Assert.assertNotNull(topicMetadata.get(topicName)); Assert.assertEquals(metadata.fetch().partitionCountForTopic(topicName).longValue(), topicMetadata.get(topicName).size());
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { if (key != null) { String stringKey = key.toString(); int offset = stringKey.hashCode(); return Math.abs(offset % cluster.partitionCountForTopic(topic)); } else { return 0; } }
@Override public int partition(String s, Object o, byte[] bytes, Object o1, byte[] bytes1, Cluster cluster) { int totalP = cluster.partitionCountForTopic(s); int partition = 0; String stringKey = (String) o; int offset = stringKey.lastIndexOf('.'); if (offset > 0) { partition = Integer.parseInt(stringKey.substring(offset + 1)) % totalP; } return partition; }
final Integer partitions = metadata.partitionCountForTopic(topic); if (partitions == null) { final String str = String.format("%sTopic not found: %s", logPrefix, topic);
private void sendMessages(String topic, int startIndex, int count) { int partitions = cluster.cluster().partitionCountForTopic(topic); for (int i = 0; i < count; i++) { int key = startIndex + i; int partition = key % partitions; cluster.appendMessage(new ProducerRecord<Integer, String>(topic, partition, key, "Message-" + key)); } }
private void sendMessages(String topic, int startIndex, int count) { int partitions = cluster.cluster().partitionCountForTopic(topic); for (int i = 0; i < count; i++) { int key = startIndex + i; int partition = key % partitions; cluster.appendMessage(new ProducerRecord<Integer, String>(topic, partition, key, "Message-" + key)); } }
numPartitionsCandidate = repartitionTopicMetadata.get(sourceTopicName).numPartitions; } else { numPartitionsCandidate = metadata.partitionCountForTopic(sourceTopicName);
public OutgoingRecords append(String topic, int count) { Integer partitions = cluster.cluster().partitionCountForTopic(topic); boolean fail = partitions == null; for (int i = 0; i < count; i++) { int messageID = nextMessageID.getAndIncrement(); int correlation = senderRecords.size(); TopicPartition partition = new TopicPartition(topic, partitions == null ? 0 : messageID % partitions.intValue()); recordPartitions.put(correlation, partition); senderRecords.add(SenderRecord.create(topic, partition.partition(), null, messageID, "Message-" + messageID, correlation)); List<SenderResult<Integer>> partitionResponses = senderResponses.get(partition); if (partitionResponses == null) { partitionResponses = new ArrayList<>(); senderResponses.put(partition, partitionResponses); } RecordMetadata metadata = null; Exception e = null; if (!fail) metadata = new RecordMetadata(partition, 0, partitionResponses.size(), 0, (Long) 0L, 0, 0); else e = new InvalidTopicException("Topic not found: " + topic); partitionResponses.add(new DefaultKafkaSender.Response<Integer>(metadata, e, correlation)); } return this; }