/** * Gets the ideal state of the table * @param tableNameWithType Table name with suffix * @return IdealState of tableNameWithType */ public IdealState getTableIdealState(String tableNameWithType) { return _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); }
protected IdealState getResourceIdealState(String resourceName) { return _helixAdmin.getResourceIdealState(_clusterName, resourceName); } }
public static IdealState getBrokerIdealStates(HelixAdmin admin, String clusterName) { return admin.getResourceIdealState(clusterName, BROKER_RESOURCE); }
private Set<String> getAllInstancesForTable(String tableName) { Set<String> instanceSet = new HashSet<String>(); IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); for (String partition : tableIdealState.getPartitionSet()) { instanceSet.addAll(tableIdealState.getInstanceSet(partition)); } return instanceSet; }
/** * Returns a map from server instance to list of segments it serves for the given table. */ public Map<String, List<String>> getServerToSegmentsMap(String tableNameWithType) { Map<String, List<String>> serverToSegmentsMap = new HashMap<>(); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); if (idealState == null) { throw new IllegalStateException("Ideal state does not exist for table: " + tableNameWithType); } for (String segment : idealState.getPartitionSet()) { for (String server : idealState.getInstanceStateMap(segment).keySet()) { serverToSegmentsMap.computeIfAbsent(server, key -> new ArrayList<>()).add(segment); } } return serverToSegmentsMap; }
@Nullable public Map<String, Map<String, String>> getIdealState(@Nonnull String tableNameOptType, @Nullable CommonConstants.Helix.TableType tableType) { String tableNameWithType = getTableNameWithType(tableNameOptType, tableType); IdealState resourceIdealState = _pinotHelixResourceManager.getHelixAdmin() .getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), tableNameWithType); return resourceIdealState == null ? null : resourceIdealState.getRecord().getMapFields(); }
/** * Check if IdealState = ExternalView. If its not equal, return the number of differing segments. */ public int isStable(String tableName) { IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); ExternalView externalView = _helixAdmin.getResourceExternalView(_helixClusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.debug("Mismatch: segment" + segment + " server:" + server + " state:" + state); numDiff = numDiff + 1; } } } return numDiff; }
public boolean isBrokerTenantDeletable(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); Set<String> taggedInstances = new HashSet<>(HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag)); String brokerName = CommonConstants.Helix.BROKER_RESOURCE_INSTANCE; IdealState brokerIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, brokerName); for (String partition : brokerIdealState.getPartitionSet()) { for (String instance : brokerIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } return true; }
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
HelixAdmin makeHelixAdmin() { HelixAdmin admin = mock(HelixAdmin.class); ExternalView ev = mock(ExternalView.class); IdealState is = mock(IdealState.class); when(admin.getResourceExternalView(clusterName, tableName)).thenReturn(ev); when(admin.getResourceIdealState(clusterName, tableName)).thenReturn(is); List<String> segmentsInIs = segmentsInIdealStateOrExtView(); Map<String, String> dummy = new HashMap<>(1); dummy.put("someHost", "ONLINE"); for (String segment : segmentsInIs) { when(is.getInstanceStateMap(segment)).thenReturn(dummy); } when(ev.getStateMap(anyString())).thenReturn(null); return admin; }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
private Map<String, Integer> getSegmentsPerInstance(String tableName) { Map<String, Integer> segmentsPerInstance = new HashMap<String, Integer>(); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableName + "_OFFLINE"); for (String partitionName : idealState.getPartitionSet()) { for (String instanceName : idealState.getInstanceSet(partitionName)) { if (!segmentsPerInstance.containsKey(instanceName)) { segmentsPerInstance.put(instanceName, 1); } else { segmentsPerInstance.put(instanceName, segmentsPerInstance.get(instanceName) + 1); } } } return segmentsPerInstance; }
public boolean isServerTenantDeletable(String tenantName) { Set<String> taggedInstances = new HashSet<>( HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getOfflineTagForTenant(tenantName))); taggedInstances .addAll(HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getRealtimeTagForTenant(tenantName))); for (String resourceName : getAllResources()) { if (!TableNameBuilder.isTableResource(resourceName)) { continue; } IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, resourceName); for (String partition : tableIdealState.getPartitionSet()) { for (String instance : tableIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } } return true; }
/** * Rebalances segments and updates the idealstate in Helix * @param tableConfig a table config * @param replicaGroupPartitionAssignment a replica group partition assignment * @return a rebalanced idealstate */ private IdealState rebalanceSegmentsAndUpdateIdealState(final TableConfig tableConfig, final ReplicaGroupPartitionAssignment replicaGroupPartitionAssignment) { final Function<IdealState, IdealState> updaterFunction = new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { return rebalanceSegments(idealState, tableConfig, replicaGroupPartitionAssignment); } }; HelixHelper.updateIdealState(_helixManager, tableConfig.getTableName(), updaterFunction, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000, 2.0f)); return _helixAdmin.getResourceIdealState(_helixClusterName, tableConfig.getTableName()); }
private boolean allSegmentsPushedToIdealState(String tableName, int segmentNum) { IdealState idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE.tableNameWithType(tableName)); return idealState != null && idealState.getPartitionSet() != null && idealState.getPartitionSet().size() == segmentNum; } }
private boolean validateNumSegments(int numSegments) { String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableNameWithType); return idealState.getRecord().getMapFields().keySet().size() == numSegments; }
private boolean allSegmentsPushedToIdealState(String tableName, int segmentNum) { IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), TableNameBuilder.OFFLINE.tableNameWithType(tableName)); return idealState != null && idealState.getPartitionSet() != null && idealState.getPartitionSet().size() == segmentNum; }
private boolean validateTableLevelReplicaGroupRebalance() { TableConfig tableConfig = _helixResourceManager.getTableConfig(TABLE_NAME, CommonConstants.Helix.TableType.OFFLINE); String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); ReplicaGroupPartitionAssignmentGenerator partitionAssignmentGenerator = new ReplicaGroupPartitionAssignmentGenerator(_propertyStore); ReplicaGroupPartitionAssignment replicaGroupMapping = partitionAssignmentGenerator.getReplicaGroupPartitionAssignment(tableNameWithType); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableNameWithType); Map<String, Map<String, String>> segmentAssignment = idealState.getRecord().getMapFields(); Map<Integer, Set<String>> segmentsPerPartition = new HashMap<>(); segmentsPerPartition.put(0, segmentAssignment.keySet()); return ReplicaGroupTestUtils .validateReplicaGroupSegmentAssignment(tableConfig, replicaGroupMapping, segmentAssignment, segmentsPerPartition); }
public void setExistingTableConfig(TableConfig config, String tableNameWithType, TableType type) throws IOException { if (type == TableType.REALTIME) { ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); ensureRealtimeClusterIsSetUp(config, tableNameWithType, config.getIndexingConfig()); } else if (type == TableType.OFFLINE) { // Update replica group partition assignment to the property store if applicable updateReplicaGroupPartitionAssignment(config); ZKMetadataProvider.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); final String configReplication = config.getValidationConfig().getReplication(); if (configReplication != null && !config.getValidationConfig().getReplication() .equals(idealState.getReplicas())) { HelixHelper.updateIdealState(_helixZkManager, tableNameWithType, new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f)); } } }
/** * Regression test for large ideal state updates failing silently */ @Test public void testWriteLargeIdealState() { final int numSegments = 20000; IdealState idealState = new IdealState(RESOURCE_NAME); idealState.setStateModelDefRef("OnlineOffline"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); idealState.setReplicas("0"); String helixClusterName = getHelixClusterName(); _helixAdmin.addResource(helixClusterName, RESOURCE_NAME, idealState); HelixHelper.updateIdealState(_helixManager, RESOURCE_NAME, new Function<IdealState, IdealState>() { @Override public IdealState apply(@Nullable IdealState idealState) { Assert.assertNotNull(idealState); for (int i = 0; i < numSegments; i++) { idealState.setPartitionState("segment_" + i, INSTANCE_NAME, "ONLINE"); } return idealState; } }, RetryPolicies.noDelayRetryPolicy(1)); IdealState resourceIdealState = _helixAdmin.getResourceIdealState(helixClusterName, RESOURCE_NAME); for (int i = 0; i < numSegments; i++) { Assert.assertEquals(resourceIdealState.getInstanceStateMap("segment_" + i).get(INSTANCE_NAME), "ONLINE"); } }