public void add(TopicWorkload another) { this.bytesPerSecond += another.getBytesPerSecond(); this.msgsPerSecond += another.getMsgsPerSecond(); this.partitions += another.partitions; }
public TopicWorkload totalWorkload(WorkloadInfoRetriever infoRetriever, ITopicWorkloadWeighter weighter) { TopicWorkload total = new TopicWorkload(0, 0, 0); for (TopicPartition part : _topicPartitionSet) { TopicWorkload tw = infoRetriever.topicWorkload(part.getTopic()); double weight = (weighter == null) ? 1.0 : weighter.partitionWeight(part); total.add(tw.getBytesPerSecondPerPartition() * weight, tw.getMsgsPerSecondPerPartition() * weight); } return total; }
public TopicWorkload topicWorkload(String topic) { LinkedList<TopicWorkload> tws = _topicWorkloadMap.get(topic); if (tws == null || tws.isEmpty()) { return _defaultTopicWorkload; } // return the maximum bytes-in-rate during the valid window TopicWorkload maxTw = null; long current = System.currentTimeMillis(); for (TopicWorkload tw : tws) { if (current - tw.getLastUpdate() > _maxValidTimeMillis) { continue; } if (maxTw == null || maxTw.getBytesPerSecond() < tw.getBytesPerSecond()) { maxTw = tw; } } return (maxTw != null) ? maxTw : _defaultTopicWorkload; }
@Override public int compareTo(TopicWorkload that) { if (this == that) { return 0; } if (that == null) { return 1; } // TODO: decide which metric to compare int cmp = Double.compare(this.getBytesPerSecondPerPartition(), that.getBytesPerSecondPerPartition()); if (cmp != 0) { return cmp; } cmp = Double.compare(this.getMsgsPerSecondPerPartition(), that.getMsgsPerSecondPerPartition()); return cmp; }
List<TopicPartition> overloaded = new ArrayList<>(); WorkloadInfoRetriever retriever = _helixMirrorMakerManager.getWorkloadInfoRetriever(); TopicWorkload totalWorkload = new TopicWorkload(0, 0, 0); boolean noPartitions = true; for (TopicPartition tp : partitionsToBeAssigned) { totalWorkload.add(weight * tw.getBytesPerSecondPerPartition(), weight * tw.getMsgsPerSecondPerPartition()); noPartitions = false; partitionCount++; TopicWorkload tw = retriever.topicWorkload(tp.getTopic()); totalWorkload.add(weight * tw.getBytesPerSecondPerPartition(), weight * tw.getMsgsPerSecondPerPartition()); noPartitions = false; TopicWorkload averageWorkload = new TopicWorkload(totalWorkload.getBytesPerSecond() / orderedInstances.size(), totalWorkload.getMsgsPerSecond() / orderedInstances.size()); double weight = weighter.partitionWeight(tp); TopicWorkload tw = retriever.topicWorkload(tp.getTopic()); if (tw.compareTotal(averageWorkload) > 0) { excludeInstances++; totalWorkload.setMsgsPerSecond( totalWorkload.getMsgsPerSecond() - weight * tw.getMsgsPerSecondPerPartition()); totalWorkload.setBytesPerSecond( totalWorkload.getBytesPerSecond() - weight * tw.getBytesPerSecondPerPartition()); averageWorkload = new TopicWorkload(totalWorkload.getBytesPerSecond() / numInstances, totalWorkload.getMsgsPerSecond() / numInstances);
private void retrieveWorkload(long timeInMs, long windowInMs, Map<String, Integer> topicsPartitions) throws IOException { long current = System.currentTimeMillis(); Map<String, TopicWorkload> topicWorkloads = C3QueryUtils.retrieveTopicInRate(timeInMs, windowInMs, _helixMirrorMakerManager.getControllerConf().getC3Host(), _helixMirrorMakerManager.getControllerConf().getC3Port(), _srcKafkaCluster, new ArrayList<>(topicsPartitions.keySet())); synchronized (_topicWorkloadMap) { for (Map.Entry<String, TopicWorkload> entry : topicWorkloads.entrySet()) { String topic = entry.getKey(); TopicWorkload workload = entry.getValue(); Integer partitions = topicsPartitions.get(topic); if (partitions != null) { workload.setParitions(partitions); LinkedList<TopicWorkload> tws = _topicWorkloadMap.get(topic); if (tws == null) { tws = new LinkedList<>(); _topicWorkloadMap.put(topic, tws); } if (tws.isEmpty() || tws.getLast().getLastUpdate() < workload.getLastUpdate()) { tws.add(workload); } // purge the data points out of the valid window while (!tws.isEmpty() && (current - tws.getFirst().getLastUpdate() > _maxValidTimeMillis)) { tws.removeFirst(); } } } } }
@Override public int compare(InstanceTopicPartitionHolder o1, InstanceTopicPartitionHolder o2) { TopicWorkload workload1 = (o1 == null) ? new TopicWorkload(0, 0) : o1.totalWorkload(infoRetriever, weighter); TopicWorkload workload2 = (o2 == null) ? new TopicWorkload(0, 0) : o2.totalWorkload(infoRetriever, weighter); int cmp = workload1.compareTotal(workload2); if (cmp != 0) { return cmp; } // if workload is the same, compare them based on the number of partitions int size1 = (o1 == null) ? -1 : o1.getNumServingTopicPartitions(); int size2 = (o2 == null) ? -1 : o2.getNumServingTopicPartitions(); if (size1 != size2) { return size1 - size2; } else { return o1.getInstanceName().compareTo(o2.getInstanceName()); } } };
continue; TopicWorkload tw = new TopicWorkload(totalBytes / period, totalCount / period); tw.setLastUpdate(endTimeSec * 1000); workloads.put(topic, tw);
public static Comparator<InstanceTopicPartitionHolder> getTotalWorkloadComparator( final WorkloadInfoRetriever infoRetriever, final ITopicWorkloadWeighter weighter) { return new Comparator<InstanceTopicPartitionHolder>() { @Override public int compare(InstanceTopicPartitionHolder o1, InstanceTopicPartitionHolder o2) { TopicWorkload workload1 = (o1 == null) ? new TopicWorkload(0, 0) : o1.totalWorkload(infoRetriever, weighter); TopicWorkload workload2 = (o2 == null) ? new TopicWorkload(0, 0) : o2.totalWorkload(infoRetriever, weighter); int cmp = workload1.compareTotal(workload2); if (cmp != 0) { return cmp; } // if workload is the same, compare them based on the number of partitions int size1 = (o1 == null) ? -1 : o1.getNumServingTopicPartitions(); int size2 = (o2 == null) ? -1 : o2.getNumServingTopicPartitions(); if (size1 != size2) { return size1 - size2; } else { return o1.getInstanceName().compareTo(o2.getInstanceName()); } } }; }
@Override public int compare(TopicPartition o1, TopicPartition o2) { TopicWorkload workload1 = infoRetriever.topicWorkload(o1.getTopic()); TopicWorkload workload2 = infoRetriever.topicWorkload(o2.getTopic()); if (workload1 == workload2) { return 0; } int cmp = workload1.compareTo(workload2); if (cmp != 0) { return cmp; } // if workload is the same, compare them based on the name and partition cmp = o1.getTopic().compareTo(o2.getTopic()); if (cmp != 0) { return cmp; } return Integer.compare(o1.getPartition(), o2.getPartition()); } };
double tpw = workloadRetriever.topicWorkload(tp.getTopic()).getBytesPerSecondPerPartition(); totalWorkload += tpw; instanceMapJson.getJSONArray(name).add(tp.getTopic() + "." + tp.getPartition() + ":" + Math.round(tpw));
/** * Return the lagging time if the given partition has lag. * * @param tp topic partition * @return the lagging time in seconds if the given partition has lag; otherwise return 0. */ private long getLagTime(TopicPartition tp) { TopicPartitionLag tpl = _helixMirrorMakerManager.getOffsetMonitor().getTopicPartitionOffset(tp); if (tpl == null || tpl.getLatestOffset() <= 0 || tpl.getCommitOffset() <= 0 || System.currentTimeMillis() - tpl.getTimeStamp() > _offsetMaxValidTimeMillis) { return 0; } long lag = tpl.getLatestOffset() - tpl.getCommitOffset(); if (lag <= _minLagOffset) { return 0; } double msgRate = _helixMirrorMakerManager.getWorkloadInfoRetriever().topicWorkload(tp.getTopic()) .getMsgsPerSecondPerPartition(); if (msgRate < 1) { msgRate = 1; } double lagTime = lag / msgRate; if (lagTime > _minLagTimeSec) { return Math.round(lagTime); } return 0; }
/** * Compare workload regardless of partitions. */ public int compareTotal(TopicWorkload that) { if (this == that) { return 0; } if (that == null) { return 1; } // TODO: decide which metric to compare int cmp = Double.compare(this.getBytesPerSecond(), that.getBytesPerSecond()); if (cmp != 0) { return cmp; } cmp = Double.compare(this.getMsgsPerSecond(), that.getMsgsPerSecond()); return cmp; }
Arrays.asList("topic1", "topic2"), workloads); TopicWorkload tw1 = workloads.get("topic1"); Assert.assertEquals(tw1.getBytesPerSecond(), 10, 0.01); Assert.assertEquals(tw1.getMsgsPerSecond(), 1, 0.01); TopicWorkload tw2 = workloads.get("topic2"); Assert.assertEquals(tw2.getBytesPerSecond(), 1000, 0.01); Assert.assertEquals(tw2.getMsgsPerSecond(), 0.5, 0.01); workloads.clear(); Assert.assertNull(tw1); tw2 = workloads.get("topic2"); Assert.assertEquals(tw2.getBytesPerSecond(), 1000, 0.01); Assert.assertEquals(tw2.getMsgsPerSecond(), 0.5, 0.01); workloads.clear(); Assert.assertNull(tw1); tw2 = workloads.get("topic2"); Assert.assertEquals(tw2.getBytesPerSecond(), 1000, 0.01); Assert.assertEquals(tw2.getMsgsPerSecond(), 0.5, 0.01); workloads.clear();