@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
/** * Compute the partition for the given record. * * @param topic The topic name * @param key The key to partition on (or null if no key) * @param keyBytes serialized key to partition on (or null if no key) * @param value The value to partition on or null * @param valueBytes serialized value to partition on or null * @param cluster The current cluster metadata */ public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { List<PartitionInfo> partitions = cluster.partitionsForTopic(topic); int numPartitions = partitions.size(); if (keyBytes == null) { int nextValue = nextValue(topic); List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(topic); if (availablePartitions.size() > 0) { int part = Utils.toPositive(nextValue) % availablePartitions.size(); return availablePartitions.get(part).partition(); } else { // no partitions are available, give a non-available partition return Utils.toPositive(nextValue) % numPartitions; } } else { // hash the keyBytes to choose a partition return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions; } }
@Ignore // By incorporating available partitions instead of overall partition count, // we were getting non-deterministic partitions for known keys. This is not // what we want for some applications, so this was changed. @Test public void nullKeyRoundRobinThreeAvailablePartitionsTest() { List<PartitionInfo> partitions = new ArrayList<>(); for (int i = 0; i < 3; i++) { partitions.add(new PartitionInfo(null, i, null, null, null)); } when(cluster.availablePartitionsForTopic(anyString())).thenReturn(partitions); List<Integer> results = new ArrayList<>(); for (int i = 0; i < 12; i++) { results.add(partitioner.partition("events", null, null, null, null, cluster)); } List<Integer> shouldBe = of(0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2); assertThat(results).isEqualTo(shouldBe); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
@Override public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { return this.next(cluster.availablePartitionsForTopic(topic).size()); }
private static int getPartition(String topic, Cluster cluster, int hash) { List<PartitionInfo> partitions = cluster.availablePartitionsForTopic(topic); if (partitions.isEmpty()) { LOG.warn("No available partitions for {} therefore using total partition count for calculations.", topic); partitions = cluster.partitionsForTopic(topic); } int index = Math.abs(hash) % partitions.size(); return partitions.get(index).partition(); } }
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { List<PartitionInfo> partitions = cluster.partitionsForTopic(topic); int numPartitions = partitions.size(); int msgCount = normalCounter.incrementAndGet(); if (msgCount % 1000 == 0) { log.info("Sent {} messages", msgCount); } if (key == null) { int nextValue = this.counter.getAndIncrement(); List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(topic); if (availablePartitions.size() > 0) { int part = toPositive(nextValue) % availablePartitions.size(); return availablePartitions.get(part).partition(); } else { return toPositive(nextValue) % numPartitions; } } else { return computePartition(key, numPartitions, maxParallelism); } }
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { List<PartitionInfo> partitions = cluster.availablePartitionsForTopic(topic); int numPartitions = partitions.size(); try { long partitionHash = ((DefaultMessage)value).partitionHash(); //按hash分区 if(partitionHash > 0){ long index = partitionHash % numPartitions; //System.out.println("numPartitions:"+numPartitions+",partitionHash:"+partitionHash + ",index:"+index); return (int)index; } } catch (ClassCastException e) {} if (keyBytes == null) { int nextValue = counter.getAndIncrement(); List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(topic); if (availablePartitions.size() > 0) { int part = DefaultPartitioner.toPositive(nextValue) % availablePartitions.size(); return availablePartitions.get(part).partition(); } else { // no partitions are available, give a non-available partition return DefaultPartitioner.toPositive(nextValue) % numPartitions; } } else { // hash the keyBytes to choose a partition return DefaultPartitioner.toPositive(Utils.murmur2(keyBytes)) % numPartitions; } }
} else if (record.key() == null) { int nextValue = counter.getAndIncrement(); List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(record.topic()); if (availablePartitions.size() > 0) { int part = Utils.abs(nextValue) % availablePartitions.size();
@Override public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) { acquire(); try { this.rebalanceCallback = callback; doUnAssign(); subscription.clear(); subscription.addAll(topics); for (String topic : topics) { List<PartitionInfo> partitions = cluster.cluster().availablePartitionsForTopic(topic); if (partitions != null) { for (PartitionInfo p : partitions) { assignment.add(new TopicPartition(p.topic(), p.partition())); } } } assignmentPending = true; } finally { release(); } }
public RecordMetadata doSend(ProducerRecord<Integer, String> record, Callback callback) { List<PartitionInfo> partitionInfo = cluster.cluster().availablePartitionsForTopic(record.topic()); TopicPartition topicPartition = new TopicPartition(record.topic(), record.partition()); inFlightCount.decrementAndGet(); if (partitionInfo == null || partitionInfo.isEmpty()) { InvalidTopicException e = new InvalidTopicException("Topic not found: " + record.topic()); callback.onCompletion(null, e); throw e; } else if (!cluster.leaderAvailable(topicPartition)) { LeaderNotAvailableException e = new LeaderNotAvailableException("Leader not available for " + topicPartition); callback.onCompletion(null, e); throw e; } else { try { long offset = cluster.appendMessage(record, !senderOptions.isTransactional()); RecordMetadata metadata = new RecordMetadata(topicPartition, 0, offset, System.currentTimeMillis(), (Long) 0L, 4, record.value().length()); callback.onCompletion(metadata, null); return metadata; } catch (Exception e) { callback.onCompletion(null, e); throw e; } } }