@Override protected Map<String, String> getPartitionStateMap(ExternalView state) { Map<String, String> partitionState = new HashMap<>(); for (String partition : state.getPartitionSet()) { Map<String, String> instanceStateMap = state.getStateMap(partition); if (instanceStateMap.containsKey(_instanceName)) { partitionState.put(partition, instanceStateMap.get(_instanceName)); } } return partitionState; } }
private boolean isLargeCluster(ExternalView externalView) { // Check if the number of replicas is sufficient to treat it as a large cluster final String helixReplicaCount = externalView.getRecord().getSimpleField("REPLICAS"); final int replicaCount; try { replicaCount = Integer.parseInt(helixReplicaCount); } catch (Exception e) { LOGGER.warn("Failed to parse the replica count ({}) from external view of table {}", helixReplicaCount, externalView.getResourceName()); return false; } if (replicaCount < _minReplicaCountForLargeCluster) { return false; } // Check if the server count is high enough to count as a large cluster final Set<String> instanceSet = new HashSet<>(); for (String partition : externalView.getPartitionSet()) { instanceSet.addAll(externalView.getStateMap(partition).keySet()); } return _minServerCountForLargeCluster <= instanceSet.size(); }
private ExternalView generateBrokerResource(String tableName) { ExternalView brokerResource = new ExternalView(BROKER_RESOURCE_INSTANCE); brokerResource.setState(tableName, BROKER_INSTANCE_ID, "ONLINE"); brokerResource.setState(tableName, "broker_instance_2", "OFFLINE"); return brokerResource; }
private void assertTopicExternalViewWithGivenPartitions( HelixMirrorMakerManager helixMirrorMakerManager, String topicName, int partitions, int numInstances) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); Assert.assertEquals(externalViewForTopic.getPartitionSet().size(), partitions); Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } int expectedLowerBound = (int) Math.floor((double) partitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) partitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound - 1); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound + 1); } } }
private void updateScheduledTaskStatus(ExternalView ev, HelixManager manager, IdealState taskQueueIdealState) { HelixDataAccessor accessor = manager.getHelixDataAccessor(); ZNRecord finishedTasks = new ZNRecord(ev.getResourceName()); for (String taskPartitionName : ev.getPartitionSet()) { for (String taskState : ev.getStateMap(taskPartitionName).values()) { if (taskState.equalsIgnoreCase(HelixDefinedState.ERROR.toString()) || taskState .equalsIgnoreCase("COMPLETED")) {
while (System.currentTimeMillis() < externalViewChangeCompletedDeadline) { ExternalView externalView = _helixAdmin.getResourceExternalView(_helixClusterName, tableName); Map<String, String> segmentStatsMap = externalView.getStateMap(segmentName); if (segmentStatsMap != null) { LOGGER.info("Found {} instances for segment '{}' in external view", segmentStatsMap.size(), segmentName);
final HelixManager manager, Set<String> monitoringResources, List<ExternalView> newExtViews) { String resourceName = resource.getResourceName(); ExternalView view = new ExternalView(resource.getResourceName()); view.setBucketSize(resource.getBucketSize()); } else { view.setBucketSize(currentStateOutput.getBucketSize(resourceName)); if (currentStateMap != null && currentStateMap.size() > 0) { for (String instance : currentStateMap.keySet()) { view.setState(partition.getPartitionName(), instance, currentStateMap.get(instance)); cache.getStateModelDef(idealState.getStateModelDefRef()); clusterStatusMonitor .setResourceStatus(view, cache.getIdealState(view.getResourceName()), stateModelDef, totalPendingMessageCount); monitoringResources.add(resourceName); view.getRecord().getSimpleFields().putAll(idealState.getRecord().getSimpleFields()); } else if (curExtView != null) { view.getRecord().getSimpleFields().putAll(curExtView.getRecord().getSimpleFields()); if (curExtView == null || !curExtView.getRecord().equals(view.getRecord())) {
/** * return true if IdealState = ExternalView * @return */ public int isStable(String tableName) { IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName); ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.info( "Mismatch: segment " + segment + " server:" + server + " expected state:" + state + " actual state:" + ( (mapEV == null || mapEV.get(server) == null) ? "null" : mapEV.get(server))); numDiff = numDiff + 1; } } } return numDiff; }
@Override public boolean isRoutingTableValid(Map<String, List<String>> routingTable, ExternalView externalView, List<InstanceConfig> instanceConfigs) { Set<String> unassignedSegments = new HashSet<>(); unassignedSegments.addAll(externalView.getPartitionSet()); for (List<String> segmentsForServer : routingTable.values()) { if (!unassignedSegments.containsAll(segmentsForServer)) { // A segment is already assigned to another server and/or doesn't exist in external view return false; } unassignedSegments.removeAll(segmentsForServer); } return unassignedSegments.isEmpty(); } }, "Routing table should contain all segments exactly once");
return; int currentVersionNumber = currentBrokerResource.getRecord().getVersion(); if (currentVersionNumber == _lastKnownBrokerResourceVersion.get()) { LOGGER.info("No qps quota change: external view for broker resource remains the same."); Map<String, String> stateMap = currentBrokerResource.getStateMap(tableNameWithType); if (stateMap == null) { LOGGER.info("No broker resource for Table {}. Removing its rate limit.", tableNameWithType);
@Test public void testHelixExternalViewBasedRoutingTable() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json"); Assert.assertNotNull(resourceUrl); String fileName = resourceUrl.getFile(); byte[] externalViewBytes = IOUtils.toByteArray(new FileInputStream(fileName)); ExternalView externalView = new ExternalView((ZNRecord) new ZNRecordSerializer().deserialize(externalViewBytes)); String tableName = externalView.getResourceName(); List<InstanceConfig> instanceConfigs = getInstanceConfigs(externalView); int numSegmentsInEV = externalView.getPartitionSet().size(); int numServersInEV = instanceConfigs.size(); HelixExternalViewBasedRouting routing = new HelixExternalViewBasedRouting(null, null, new BaseConfiguration()); routing.markDataResourceOnline(generateTableConfig(tableName), externalView, instanceConfigs); for (int i = 0; i < NUM_ROUNDS; i++) { Map<String, List<String>> routingTable = routing.getRoutingTable(new RoutingTableLookupRequest(tableName)); Assert.assertEquals(routingTable.size(), numServersInEV); int numSegments = 0; for (List<String> segmentsForServer : routingTable.values()) { int numSegmentsForServer = segmentsForServer.size(); Assert.assertTrue( numSegmentsForServer >= MIN_NUM_SEGMENTS_PER_SERVER && numSegmentsForServer <= MAX_NUM_SEGMENTS_PER_SERVER); numSegments += numSegmentsForServer; } Assert.assertEquals(numSegments, numSegmentsInEV); } }
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); ExternalView externalView = new ExternalView(tableName); externalView.setState("myTable_0", "pinot1", "ONLINE"); externalView.setState("myTable_0", "pinot2", "ONLINE"); externalView.setState("myTable_1", "pinot1", "ERROR"); externalView.setState("myTable_1", "pinot2", "ONLINE"); segmentStatusChecker.run(); Assert.assertEquals( controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1); Assert .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS), 1); Assert .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS), 33); Assert.assertEquals( controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
if (!resetResourceNames.contains(extView.getResourceName())) { continue; Map<String, Map<String, String>> stateMap = extView.getRecord().getMapFields(); for (String partitionName : stateMap.keySet()) { Map<String, String> instanceStateMap = stateMap.get(partitionName); resetPartition(clusterName, instanceName, extView.getResourceName(), resetPartitionNames.get(instanceName));
@Override public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) { if (!trackEnabled) { return; } for (ExternalView ev : externalViewList) { IdealState is = _resourceMap.get(ev.getResourceName()); if (is == null) { continue; } int replica = is.getReplicaCount(NUM_NODE); for (String p : is.getPartitionSet()) { Map<String, String> stateMap = ev.getStateMap(p); verifyPartitionCount(is.getResourceName(), p, stateMap, replica, "EV", is.getMinActiveReplicas()); } } }
return; String tableName = externalView.getResourceName(); Set<String> offlineSegmentsServing = externalView.getPartitionSet(); if (offlineSegmentsServing.isEmpty()) { LOGGER.info("Skipping updating time boundary service for table '{}' with no offline segments.", tableName);
/** * Update the cached external view map * @param externalViews */ public void updateExternalViews(List<ExternalView> externalViews) { for (ExternalView externalView : externalViews) { _externalViewMap.put(externalView.getResourceName(), externalView); } }
@Test public void testNoBrokerServiceOnBrokerResource() throws Exception { ExternalView brokerResource = new ExternalView(BROKER_RESOURCE_INSTANCE); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); setQps(tableConfig); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 1); }
@Override public boolean verify() { ExternalView externalView = _accessor.getProperty(_accessor.keyBuilder().externalView(_reference.getResourceName())); return _reference.equals(externalView); }
private void assertTopicExternalViewWithGivenPartitions( HelixMirrorMakerManager helixMirrorMakerManager, String topicName, int partitions, int numInstances) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); Assert.assertEquals(externalViewForTopic.getPartitionSet().size(), partitions); Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } int expectedLowerBound = (int) Math.floor((double) partitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) partitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound - 1); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound + 1); } } }