@Override public IdealState apply(@Nullable IdealState idealState) { Preconditions.checkNotNull(idealState); for (String broker : brokersForTenant) { idealState.setPartitionState(tableName, broker, BrokerOnlineOfflineStateModel.ONLINE); } return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
@Override public IdealState apply(IdealState idealState) { Set<String> partitions = idealState.getPartitionSet(); if (partitions.contains(segmentName)) { LOGGER.warn("Segment already exists in the ideal state for segment: {} of table: {}, do not update", segmentName, tableNameWithType); } else { if (assignedInstances.isEmpty()) { LOGGER.warn("No instance assigned for segment: {} of table: {}", segmentName, tableNameWithType); } else { int numPartitions = partitions.size() + 1; for (String instance : assignedInstances) { idealState.setPartitionState(segmentName, instance, ONLINE); } idealState.setNumPartitions(numPartitions); } } return idealState; } };
@Override public IdealState apply(@Nullable IdealState idealState) { Assert.assertNotNull(idealState); for (int i = 0; i < numSegments; i++) { idealState.setPartitionState("segment_" + i, INSTANCE_NAME, "ONLINE"); } return idealState; } }, RetryPolicies.noDelayRetryPolicy(1));
public static IdealState addNewRealtimeSegmentToIdealState(String segmentId, IdealState state, String instanceName) { state.setPartitionState(segmentId, instanceName, ONLINE); state.setNumPartitions(state.getNumPartitions() + 1); return state; }
for (String instance : oldInstances) { idealState .setPartitionState(segment, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE); .setPartitionState(segment, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
idealState.setPartitionState(segmentId, instanceId, targetMap.get(instanceId)); LOGGER.debug("Adding " + instanceId + " to serve segment " + segmentId);
Set<String> currentSegmentInstances = idealState.getInstanceSet(currentSegmentId); for (String instance : currentSegmentInstances) { idealState.setPartitionState(currentSegmentId, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE); .setPartitionState(newSegmentId, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
@BeforeMethod public void setup() { current = new IdealState("rebalance"); current.setPartitionState(segmentId, "host1", "ONLINE"); current.setPartitionState(segmentId, "host2", "ONLINE"); downtime.setProperty(RebalanceUserConfigConstants.DOWNTIME, true); noDowntime.setProperty(RebalanceUserConfigConstants.DOWNTIME, false); }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
/** * An instance is reporting that it has stopped consuming a topic due to some error. * Mark the state of the segment to be OFFLINE in idealstate. * When all replicas of this segment are marked offline, the {@link org.apache.pinot.controller.validation.RealtimeSegmentValidationManager}, * in its next run, will auto-create a new segment with the appropriate offset. */ public void segmentStoppedConsuming(final LLCSegmentName segmentName, final String instance) { String rawTableName = segmentName.getTableName(); String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(rawTableName); final String segmentNameStr = segmentName.getSegmentName(); try { HelixHelper.updateIdealState(_helixManager, realtimeTableName, idealState -> { idealState.setPartitionState(segmentNameStr, instance, CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.OFFLINE); Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentNameStr); LOGGER.info("Attempting to mark {} offline. Current map:{}", segmentNameStr, instanceStateMap.toString()); return idealState; }, RetryPolicies.exponentialBackoffRetryPolicy(10, 500L, 1.2f)); } catch (Exception e) { LOGGER.error("Failed to update idealstate for table {} instance {} segment {}", realtimeTableName, instance, segmentNameStr, e); _controllerMetrics.addMeteredGlobalValue(ControllerMeter.LLC_ZOOKEEPER_UPDATE_FAILURES, 1); throw e; } LOGGER.info("Successfully marked {} offline for instance {} since it stopped consuming", segmentNameStr, instance); }
public PinotResourceManagerResponse rebuildBrokerResource(String tableNameWithType, Set<String> brokerInstances) { IdealState brokerIdealState = HelixHelper.getBrokerIdealStates(_helixAdmin, _helixClusterName); Set<String> brokerInstancesInIdealState = brokerIdealState.getInstanceSet(tableNameWithType); if (brokerInstancesInIdealState.equals(brokerInstances)) { return PinotResourceManagerResponse .success("Broker resource is not rebuilt because ideal state is the same for table: " + tableNameWithType); } // Update ideal state with the new broker instances try { HelixHelper.updateIdealState(getHelixZkManager(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState -> { assert idealState != null; Map<String, String> instanceStateMap = idealState.getInstanceStateMap(tableNameWithType); if (instanceStateMap != null) { instanceStateMap.clear(); } for (String brokerInstance : brokerInstances) { idealState.setPartitionState(tableNameWithType, brokerInstance, BrokerOnlineOfflineStateModel.ONLINE); } return idealState; }, DEFAULT_RETRY_POLICY); LOGGER.info("Successfully rebuilt brokerResource for table: {}", tableNameWithType); return PinotResourceManagerResponse.success("Rebuilt brokerResource for table: " + tableNameWithType); } catch (Exception e) { LOGGER.error("Caught exception while rebuilding broker resource for table: {}", tableNameWithType, e); throw e; } }
allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setPartitionState("myTable_0", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState(segmentName, instance, status);
allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setPartitionState("myTable_0", "pinot1", "OFFLINE"); idealState.setPartitionState("myTable_0", "pinot2", "OFFLINE"); idealState.setPartitionState("myTable_0", "pinot3", "OFFLINE"); idealState.setReplicas("0"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState("myTable_OFFLINE", "pinot1", "OFFLINE"); idealState.setPartitionState("myTable_OFFLINE", "pinot2", "OFFLINE"); idealState.setPartitionState("myTable_OFFLINE", "pinot3", "OFFLINE"); idealState.setReplicas("1"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setPartitionState("myTable_0", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState(segName, serverName, "CONSUMING"); allSegments.add(segmentMetadata); } else if (seq == 1) { idealState.setPartitionState(segName, serverName, "ONLINE"); allSegments.add(segmentMetadata); } else { segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS); idealState.setPartitionState(segName, serverName, "OFFLINE"); allSegments.add(segmentMetadata); if (now - segmentCreationTime > RetentionManager.OLD_LLC_SEGMENTS_RETENTION_IN_MILLIS) {
allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setPartitionState("myTable_0", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setPartitionState("myTable_3", "pinot3", "ONLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
final LLCSegmentName seg3 = new LLCSegmentName(rawTableName, 2, 1, System.currentTimeMillis()); IdealState idealState = new IdealState(tableName); idealState.setPartitionState(seg1.getSegmentName(), "pinot1", "ONLINE"); idealState.setPartitionState(seg1.getSegmentName(), "pinot2", "ONLINE"); idealState.setPartitionState(seg1.getSegmentName(), "pinot3", "ONLINE"); idealState.setPartitionState(seg2.getSegmentName(), "pinot1", "ONLINE"); idealState.setPartitionState(seg2.getSegmentName(), "pinot2", "ONLINE"); idealState.setPartitionState(seg2.getSegmentName(), "pinot3", "ONLINE"); idealState.setPartitionState(seg3.getSegmentName(), "pinot1", "CONSUMING"); idealState.setPartitionState(seg3.getSegmentName(), "pinot2", "CONSUMING"); idealState.setPartitionState(seg3.getSegmentName(), "pinot3", "OFFLINE"); idealState.setReplicas("3"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setPartitionState("myTable_0", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot2", "ONLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);