private boolean isLargeCluster(ExternalView externalView) { // Check if the number of replicas is sufficient to treat it as a large cluster final String helixReplicaCount = externalView.getRecord().getSimpleField("REPLICAS"); final int replicaCount; try { replicaCount = Integer.parseInt(helixReplicaCount); } catch (Exception e) { LOGGER.warn("Failed to parse the replica count ({}) from external view of table {}", helixReplicaCount, externalView.getResourceName()); return false; } if (replicaCount < _minReplicaCountForLargeCluster) { return false; } // Check if the server count is high enough to count as a large cluster final Set<String> instanceSet = new HashSet<>(); for (String partition : externalView.getPartitionSet()) { instanceSet.addAll(externalView.getStateMap(partition).keySet()); } return _minServerCountForLargeCluster <= instanceSet.size(); }
return; String tableName = externalView.getResourceName();
@Test public void testHelixExternalViewBasedRoutingTable() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json"); Assert.assertNotNull(resourceUrl); String fileName = resourceUrl.getFile(); byte[] externalViewBytes = IOUtils.toByteArray(new FileInputStream(fileName)); ExternalView externalView = new ExternalView((ZNRecord) new ZNRecordSerializer().deserialize(externalViewBytes)); String tableName = externalView.getResourceName(); List<InstanceConfig> instanceConfigs = getInstanceConfigs(externalView); int numSegmentsInEV = externalView.getPartitionSet().size(); int numServersInEV = instanceConfigs.size(); HelixExternalViewBasedRouting routing = new HelixExternalViewBasedRouting(null, null, new BaseConfiguration()); routing.markDataResourceOnline(generateTableConfig(tableName), externalView, instanceConfigs); for (int i = 0; i < NUM_ROUNDS; i++) { Map<String, List<String>> routingTable = routing.getRoutingTable(new RoutingTableLookupRequest(tableName)); Assert.assertEquals(routingTable.size(), numServersInEV); int numSegments = 0; for (List<String> segmentsForServer : routingTable.values()) { int numSegmentsForServer = segmentsForServer.size(); Assert.assertTrue( numSegmentsForServer >= MIN_NUM_SEGMENTS_PER_SERVER && numSegmentsForServer <= MAX_NUM_SEGMENTS_PER_SERVER); numSegments += numSegmentsForServer; } Assert.assertEquals(numSegments, numSegmentsInEV); } }
/** * Update the cached external view map * @param externalViews */ public void updateExternalViews(List<ExternalView> externalViews) { for (ExternalView externalView : externalViews) { _externalViewMap.put(externalView.getResourceName(), externalView); } }
/** * Update the cached external view map * @param externalViews */ public void updateExternalViews(List<ExternalView> externalViews) { if (_externalViewMap == null) { _externalViewMap = new HashMap<>(); } for (ExternalView externalView : externalViews) { _externalViewMap.put(externalView.getResourceName(), externalView); } }
@Override public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) { for (ExternalView view : externalViewList) { _seenExternalViews.add(view.getResourceName()); } }
@Override public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) { for (ExternalView view : externalViewList) { if (!_externalViewChanges.containsKey(view.getResourceName())) { _externalViewChanges.put(view.getResourceName(), 1); } else { _externalViewChanges.put(view.getResourceName(), _externalViewChanges.get(view.getResourceName()) + 1); } } } }
Map<String, ExternalView> newExternalViewMap = Maps.newHashMap(); for (ExternalView externalView : updatedMap.values()) { newExternalViewMap.put(externalView.getResourceName(), externalView);
Map<String, ExternalView> newExternalViewMap = Maps.newHashMap(); for (ExternalView externalView : updatedMap.values()) { newExternalViewMap.put(externalView.getResourceName(), externalView);
if (!resetResourceNames.contains(extView.getResourceName())) { continue; resetPartition(clusterName, instanceName, extView.getResourceName(), resetPartitionNames.get(instanceName));
if (!resetResourceNames.contains(extView.getResourceName())) { continue; resetPartition(clusterName, instanceName, extView.getResourceName(), resetPartitionNames.get(instanceName));
@Override public void resetInstance(String clusterName, List<String> instanceNames) { // TODO: not mp-safe logger.info("Reset instances {} in cluster {}.", instanceNames == null ? "NULL" : HelixUtil.serializeByComma(instanceNames), clusterName); HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); List<ExternalView> extViews = accessor.getChildValues(keyBuilder.externalViews()); Set<String> resetInstanceNames = new HashSet<String>(instanceNames); for (String instanceName : resetInstanceNames) { List<String> resetPartitionNames = new ArrayList<String>(); for (ExternalView extView : extViews) { Map<String, Map<String, String>> stateMap = extView.getRecord().getMapFields(); for (String partitionName : stateMap.keySet()) { Map<String, String> instanceStateMap = stateMap.get(partitionName); if (instanceStateMap.containsKey(instanceName) && instanceStateMap.get(instanceName) .equals(HelixDefinedState.ERROR.toString())) { resetPartitionNames.add(partitionName); } } resetPartition(clusterName, instanceName, extView.getResourceName(), resetPartitionNames); } } }
@Override public void resetInstance(String clusterName, List<String> instanceNames) { // TODO: not mp-safe logger.info("Reset instances {} in cluster {}.", instanceNames == null ? "NULL" : HelixUtil.serializeByComma(instanceNames), clusterName); HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); List<ExternalView> extViews = accessor.getChildValues(keyBuilder.externalViews()); Set<String> resetInstanceNames = new HashSet<String>(instanceNames); for (String instanceName : resetInstanceNames) { List<String> resetPartitionNames = new ArrayList<String>(); for (ExternalView extView : extViews) { Map<String, Map<String, String>> stateMap = extView.getRecord().getMapFields(); for (String partitionName : stateMap.keySet()) { Map<String, String> instanceStateMap = stateMap.get(partitionName); if (instanceStateMap.containsKey(instanceName) && instanceStateMap.get(instanceName) .equals(HelixDefinedState.ERROR.toString())) { resetPartitionNames.add(partitionName); } } resetPartition(clusterName, instanceName, extView.getResourceName(), resetPartitionNames); } } }
private void updateScheduledTaskStatus(ExternalView ev, HelixManager manager, IdealState taskQueueIdealState) { HelixDataAccessor accessor = manager.getHelixDataAccessor(); ZNRecord finishedTasks = new ZNRecord(ev.getResourceName());
@Override public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) { if (!_startListen) { return; } for (ExternalView view : externalViewList) { IdealState is = _gSetupTool.getClusterManagementTool() .getResourceIdealState(CLUSTER_NAME, view.getResourceName()); validateNoZeroReplica(is, view); } }
for(Iterator<ExternalView> it = newExtViews.iterator(); it.hasNext(); ) { ExternalView view = it.next(); String resourceName = view.getResourceName(); IdealState idealState = cache.getIdealState(resourceName); if (idealState != null && idealState.isExternalViewDisabled()) {
@Override public boolean verify() { ExternalView externalView = _accessor.getProperty(_accessor.keyBuilder().externalView(_reference.getResourceName())); return _reference.equals(externalView); }
String resourceName = extView.getResourceName(); Set<String> partitionSet = extView.getPartitionSet(); for (String partition : partitionSet) {
@Override public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) { if (!trackEnabled) { return; } for (ExternalView ev : externalViewList) { IdealState is = _resourceMap.get(ev.getResourceName()); if (is == null) { continue; } int replica = is.getReplicaCount(NUM_NODE); for (String p : is.getPartitionSet()) { Map<String, String> stateMap = ev.getStateMap(p); verifyPartitionCount(is.getResourceName(), p, stateMap, replica, "EV", is.getMinActiveReplicas()); } } }
cache.getStateModelDef(idealState.getStateModelDefRef()); clusterStatusMonitor .setResourceStatus(view, cache.getIdealState(view.getResourceName()), stateModelDef, totalPendingMessageCount); monitoringResources.add(resourceName);