private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
} else { int numPartitionsInMirrorMaker = _helixMirrorMakerManager.getIdealStateForTopic(topic).getNumPartitions(); if (numPartitionsInMirrorMaker != tp.getPartition()) { int mismatchedPartitions = Math.abs(numPartitionsInMirrorMaker - tp.getPartition());
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
numServingTopics++; IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName);
IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); JSONObject resultJson = new JSONObject(); resultJson.put("topic", topicName); IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName); JSONObject resultJson = new JSONObject();
if (_helixMirrorMakerManager.isTopicExisted(topicName)) { IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName);
public void refreshWorkloads() throws IOException { long current = System.currentTimeMillis(); if (_lastRefreshTimeMillis + _minRefreshIntervalMillis > current) { LOGGER.info("Too soon to refresh workload, skip"); return; } LOGGER.info("Refreshing workload for source " + _srcKafkaCluster); _lastRefreshTimeMillis = current; List<String> topics = _helixMirrorMakerManager.getTopicLists(); Map<String, Integer> topicsPartitions = new HashMap<>(); for (String topic : topics) { IdealState idealState = _helixMirrorMakerManager.getIdealStateForTopic(topic); if (idealState != null) { int partitions = idealState.getNumPartitions(); if (partitions > 0) { topicsPartitions.put(topic, partitions); } } } retrieveWorkload(current, DEFAULT_WORKLOAD_WINDOW_MILLIS, topicsPartitions); }
public void initializeWorkloads() throws IOException { List<String> topics = _helixMirrorMakerManager.getTopicLists(); Map<String, Integer> topicsPartitions = new HashMap<>(); for (String topic : topics) { IdealState idealState = _helixMirrorMakerManager.getIdealStateForTopic(topic); if (idealState != null) { int partitions = idealState.getNumPartitions(); if (partitions > 0) { topicsPartitions.put(topic, partitions); } } } // use coarse granularity for the time windows older than 1 hour long current = System.currentTimeMillis(); long fromMs = current - _maxValidTimeMillis; long toCoarseMs = current - TimeUnit.HOURS.toMillis(1); LOGGER.info("Initialize workload for source {} for time range [{}, {}]", _srcKafkaCluster, fromMs, current); _lastRefreshTimeMillis = current; for (long tsInMs = fromMs; tsInMs <= toCoarseMs; tsInMs += DEFAULT_WORKLOAD_COARSE_WINDOW_MILLIS) { retrieveWorkload(tsInMs, DEFAULT_WORKLOAD_COARSE_WINDOW_MILLIS, topicsPartitions); } // use fine granularity for the last hour for (long tsInMs = toCoarseMs + DEFAULT_WORKLOAD_WINDOW_MILLIS; tsInMs <= current; tsInMs += DEFAULT_WORKLOAD_WINDOW_MILLIS) { retrieveWorkload(tsInMs, DEFAULT_WORKLOAD_WINDOW_MILLIS, topicsPartitions); } LOGGER.info("Finished initializing workload for source " + _srcKafkaCluster); }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
} else { int numPartitionsInMirrorMaker = _helixMirrorMakerManager.getIdealStateForTopic(topic).getNumPartitions(); if (numPartitionsInMirrorMaker != tp.getPartition()) { int mismatchedPartitions = Math.abs(numPartitionsInMirrorMaker - tp.getPartition());
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
numServingTopics++; IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName);
IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); JSONObject resultJson = new JSONObject(); resultJson.put("topic", topicName); IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName); JSONObject resultJson = new JSONObject();
if (_helixMirrorMakerManager.isTopicExisted(topicName)) { IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName);