@Override public IdealState apply(IdealState idealState) { if (idealState.getPartitionSet().contains(resourceTag)) { idealState.getPartitionSet().remove(resourceTag); return idealState; } else { return null; } } };
@Override public IdealState apply(IdealState idealState) { if (idealState == null) { return idealState; } // partitionSet is never null but let's be defensive anyway Set<String> partitionSet = idealState.getPartitionSet(); if (partitionSet != null) { partitionSet.remove(segmentName); } return idealState; } };
@Nullable @Override public IdealState apply(@Nullable IdealState idealState) { if (idealState == null) { return idealState; } // partitionSet is never null but let's be defensive anyway Set<String> partitionSet = idealState.getPartitionSet(); if (partitionSet != null) { partitionSet.removeAll(segments); } return idealState; } };
/** * Returns all instances for the given resource. * * @param idealState IdealState of the resource for which to return the instances of. * @return Returns a Set of strings containing the instance names for the given cluster. */ public static Set<String> getAllInstancesForResource(IdealState idealState) { final Set<String> instances = new HashSet<String>(); for (final String partition : idealState.getPartitionSet()) { for (final String instance : idealState.getInstanceSet(partition)) { instances.add(instance); } } return instances; }
private Set<String> getAllInstancesForTable(String tableName) { Set<String> instanceSet = new HashSet<String>(); IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); for (String partition : tableIdealState.getPartitionSet()) { instanceSet.addAll(tableIdealState.getInstanceSet(partition)); } return instanceSet; }
private IdealState updateIdealState(IdealState idealState, int newNumReplicas) { idealState.setReplicas(Integer.toString(newNumReplicas)); Set<String> segmentIds = idealState.getPartitionSet(); for (String segmentId : segmentIds) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentId); if (instanceStateMap.size() > newNumReplicas) { Set<String> keys = instanceStateMap.keySet(); while (instanceStateMap.size() > newNumReplicas) { instanceStateMap.remove(keys.iterator().next()); } } else if (instanceStateMap.size() < newNumReplicas) { throw new RuntimeException( "Segment " + segmentId + " has " + instanceStateMap.size() + " replicas but want changed to " + newNumReplicas); } } return idealState; }
/** * Returns a map from server instance to list of segments it serves for the given table. */ public Map<String, List<String>> getServerToSegmentsMap(String tableNameWithType) { Map<String, List<String>> serverToSegmentsMap = new HashMap<>(); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); if (idealState == null) { throw new IllegalStateException("Ideal state does not exist for table: " + tableNameWithType); } for (String segment : idealState.getPartitionSet()) { for (String server : idealState.getInstanceStateMap(segment).keySet()) { serverToSegmentsMap.computeIfAbsent(server, key -> new ArrayList<>()).add(segment); } } return serverToSegmentsMap; }
@Override public IdealState apply(IdealState idealState) { Set<String> partitions = idealState.getPartitionSet(); if (partitions.contains(segmentName)) { LOGGER.warn("Segment already exists in the ideal state for segment: {} of table: {}, do not update", segmentName, tableNameWithType); } else { if (assignedInstances.isEmpty()) { LOGGER.warn("No instance assigned for segment: {} of table: {}", segmentName, tableNameWithType); } else { int numPartitions = partitions.size() + 1; for (String instance : assignedInstances) { idealState.setPartitionState(segmentName, instance, ONLINE); } idealState.setNumPartitions(numPartitions); } } return idealState; } };
public void cleanupLLC(final String realtimeTableName) { // If there are any completions in the pipeline we let them commit. Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); IdealState idealState = HelixHelper.getTableIdealState(_helixManager, realtimeTableName); final List<String> segmentsToRemove = new ArrayList<String>(); Set<String> allSegments = idealState.getPartitionSet(); int removeCount = 0; for (String segmentName : allSegments) { if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { segmentsToRemove.add(segmentName); removeCount++; } } LOGGER.info("Attempting to remove {} LLC segments of table {}", removeCount, realtimeTableName); _helixResourceManager.deleteSegments(realtimeTableName, segmentsToRemove); }
public boolean isBrokerTenantDeletable(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); Set<String> taggedInstances = new HashSet<>(HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag)); String brokerName = CommonConstants.Helix.BROKER_RESOURCE_INSTANCE; IdealState brokerIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, brokerName); for (String partition : brokerIdealState.getPartitionSet()) { for (String instance : brokerIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } return true; }
private boolean allSegmentsPushedToIdealState(String tableName, int segmentNum) { IdealState idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE.tableNameWithType(tableName)); return idealState != null && idealState.getPartitionSet() != null && idealState.getPartitionSet().size() == segmentNum; } }
private Map<String, Map<String, String>> getPrevAssignment(IdealState idealState) { Map<String, Map<String, String>> prevAssignment = new HashMap<>(1); for (String segment : idealState.getPartitionSet()) { Map<String, String> instanceMap = new HashMap<>(1); instanceMap.putAll(idealState.getInstanceStateMap(segment)); prevAssignment.put(segment, instanceMap); } return prevAssignment; }
@Nullable @Override public IdealState apply(@Nullable IdealState input) { List<String> segmentNames = Lists.newArrayList(input.getPartitionSet()); Collections.sort(segmentNames); Map<String, String> instanceStateMap1 = input.getInstanceStateMap(segmentNames.get(0)); for (String instance : instanceStateMap1.keySet()) { instanceStateMap1.put(instance, "OFFLINE"); break; } return input; } }, RetryPolicies.fixedDelayRetryPolicy(2, 10));
private void verifySegmentAssignment(IdealState updatedIdealState, IdealState prevIdealState, List<String> completedInstanceList, int nReplicas, Map<String, Integer> segmentNameToNumCompletedInstances) { Assert.assertEquals(updatedIdealState.getPartitionSet().size(), prevIdealState.getPartitionSet().size()); Assert.assertTrue(prevIdealState.getPartitionSet().containsAll(updatedIdealState.getPartitionSet())); for (String segmentName : updatedIdealState.getPartitionSet()) { Map<String, String> newInstanceStateMap = updatedIdealState.getInstanceStateMap(segmentName); int onCompleted = 0; int notOnCompleted = 0; for (String instance : newInstanceStateMap.keySet()) { if (completedInstanceList.contains(instance)) { onCompleted++; } else { notOnCompleted++; } } int expectedOnCompletedServers = segmentNameToNumCompletedInstances.get(segmentName).intValue(); Assert.assertEquals(onCompleted, expectedOnCompletedServers); Assert.assertEquals(notOnCompleted, nReplicas - expectedOnCompletedServers); } }
private void validateIdealStateRealtime(IdealState rebalancedIdealState, int nSegmentsCompleted, int nSegmentsConsuming, int targetNumReplicas, List<String> instancesCompleted, List<String> instancesConsuming, Configuration rebalanceUserConfig) { Assert.assertEquals(rebalancedIdealState.getPartitionSet().size(), nSegmentsCompleted + nSegmentsConsuming); for (String segment : rebalancedIdealState.getPartitionSet()) { Map<String, String> instanceStateMap = rebalancedIdealState.getInstanceStateMap(segment); Assert.assertEquals(instanceStateMap.size(), targetNumReplicas); boolean rebalanceConsuming = rebalanceUserConfig.getBoolean(RebalanceUserConfigConstants.INCLUDE_CONSUMING); if (segment.contains("consuming")) { if (rebalanceConsuming) { Assert.assertTrue(instancesConsuming.containsAll(instanceStateMap.keySet())); } } else { Assert.assertTrue(instancesCompleted.containsAll(instanceStateMap.keySet())); } } }
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
private boolean allSegmentsPushedToIdealState(String tableName, int segmentNum) { IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), TableNameBuilder.OFFLINE.tableNameWithType(tableName)); return idealState != null && idealState.getPartitionSet() != null && idealState.getPartitionSet().size() == segmentNum; }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
private Map<String, Integer> getSegmentsPerInstance(String tableName) { Map<String, Integer> segmentsPerInstance = new HashMap<String, Integer>(); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableName + "_OFFLINE"); for (String partitionName : idealState.getPartitionSet()) { for (String instanceName : idealState.getInstanceSet(partitionName)) { if (!segmentsPerInstance.containsKey(instanceName)) { segmentsPerInstance.put(instanceName, 1); } else { segmentsPerInstance.put(instanceName, segmentsPerInstance.get(instanceName) + 1); } } } return segmentsPerInstance; }
public boolean isServerTenantDeletable(String tenantName) { Set<String> taggedInstances = new HashSet<>( HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getOfflineTagForTenant(tenantName))); taggedInstances .addAll(HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getRealtimeTagForTenant(tenantName))); for (String resourceName : getAllResources()) { if (!TableNameBuilder.isTableResource(resourceName)) { continue; } IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, resourceName); for (String partition : tableIdealState.getPartitionSet()) { for (String instance : tableIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } } return true; }