/** * Sets the idealstate for a given segment to the target mapping */ private void setTargetState(IdealState idealState, String segmentId, Map<String, String> targetMap) { idealState.getInstanceStateMap(segmentId).clear(); idealState.setInstanceStateMap(segmentId, targetMap); }
public IdealStateBuilderUtil addSegment(String segmentName, Map<String, String> instanceStateMap) { _idealState.setInstanceStateMap(segmentName, instanceStateMap); return this; }
private void setInstanceStateMapForIdealStateRealtimeCompleted(IdealState idealState, int nPartitions, int nIterationsCompleted, int nReplicas, List<String> instances, String tableName) { int i = 0; for (int it = 0; it < nIterationsCompleted; it++) { for (int p = 0; p < nPartitions; p++) { Map<String, String> instanceStateMap = new HashMap<>(nReplicas); for (int r = 0; r < nReplicas; r++) { instanceStateMap.put(instances.get(i++), "ONLINE"); if (i == instances.size()) { i = 0; } } idealState.setInstanceStateMap("completed" + tableName + "__" + p + "__" + it + "__1234", instanceStateMap); } } }
private void setInstanceStateMapForIdealStateOffline(IdealState idealState, int nSegments, int nReplicas, List<String> instances, String tableName) { int i = 0; for (int s = 0; s < nSegments; s++) { Map<String, String> instanceStateMap = new HashMap<>(nReplicas); for (int r = 0; r < nReplicas; r++) { instanceStateMap.put(instances.get(i++), "ONLINE"); if (i == instances.size()) { i = 0; } } idealState.setInstanceStateMap(tableName + "__" + s + "__0__1234", instanceStateMap); } }
public IdealStateBuilderUtil addConsumingSegment(int partition, int seqNum, int numReplicas, List<String> instances) { LLCSegmentName llcSegmentName = new LLCSegmentName(_rawTableName, partition, seqNum, System.currentTimeMillis()); Map<String, String> instanceStateMap = new HashMap<>(numReplicas); for (int i = 0; i < numReplicas; i++) { instanceStateMap.put(instances.get(i), CONSUMING); } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), instanceStateMap); return this; }
private void setInstanceStateMapForIdealStateRealtimeConsuming(IdealState idealState, PartitionAssignment partitionAssignment, int nPartitions, int seqNum, int nReplicas, List<String> instances, String tableName) { int i = 0; for (int p = 0; p < nPartitions; p++) { Map<String, String> instanceStateMap = new HashMap<>(nReplicas); for (int r = 0; r < nReplicas; r++) { instanceStateMap.put(instances.get(i++), "CONSUMING"); if (i == instances.size()) { i = 0; } } idealState.setInstanceStateMap("consuming" + tableName + "__" + p + "__" + seqNum + "__1234", instanceStateMap); if (partitionAssignment != null) { partitionAssignment.addPartition(String.valueOf(p), Lists.newArrayList(instanceStateMap.keySet())); } } }
final Map<String, Map<String, String>> newMapping = newZnRecord.getMapFields(); for (Map.Entry<String, Map<String, String>> entry : newMapping.entrySet()) { idealState.setInstanceStateMap(entry.getKey(), entry.getValue()); idealState.setInstanceStateMap(entry.getKey(), entry.getValue());
idealState.setInstanceStateMap(entry.getKey(), entry.getValue());
public IdealStateBuilderUtil addConsumingSegments(int numPartitions, int seqNum, int numReplicas, List<String> instances) { int serverId = 0; for (int p = 0; p < numPartitions; p++) { LLCSegmentName llcSegmentName = new LLCSegmentName(_rawTableName, p, seqNum, System.currentTimeMillis()); Map<String, String> instanceStateMap = new HashMap<>(numReplicas); for (int r = 0; r < numReplicas; r++) { instanceStateMap.put(instances.get(serverId++), CONSUMING); if (serverId == instances.size()) { serverId = 0; } } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), instanceStateMap); } return this; }
instanceStateMap.put(instance, RealtimeSegmentOnlineOfflineStateModel.CONSUMING); idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), instanceStateMap);
/** * Given an ideal state find the segments that need to relocate a replica to completed servers, * and create a new instance state map for those segments * * @param realtimeTagConfig * @param idealState * @param completedServers * @param completedServersQueue * @return */ private void createNewIdealState(final RealtimeTagConfig realtimeTagConfig, IdealState idealState, final List<String> completedServers, MinMaxPriorityQueue<Map.Entry<String, Integer>> completedServersQueue) { // TODO: we are scanning the entire segments list every time. This is unnecessary because only the latest segments will need relocation // Can we do something to avoid this? // 1. Time boundary: scan only last day whereas runFrequency = hourly // 2. For each partition, scan in descending order, and stop when the first segment not needing relocation is found for (String segmentName : idealState.getPartitionSet()) { final Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentName); Map<String, String> newInstanceStateMap = createNewInstanceStateMap(realtimeTagConfig, segmentName, instanceStateMap, completedServers, completedServersQueue); if (MapUtils.isNotEmpty(newInstanceStateMap)) { idealState.setInstanceStateMap(segmentName, newInstanceStateMap); } } }
public IdealStateBuilderUtil moveToServers(int partition, int seqNum, List<String> instances) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); Map<String, String> newInstanceStateMap = new HashMap<>(instanceStateMap.size()); int serverId = 0; for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { newInstanceStateMap.put(instances.get(serverId++), entry.getValue()); } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), newInstanceStateMap); break; } } } return this; }
instanceStateMap0.put(consumingInstanceList.get(1), "CONSUMING"); instanceStateMap0.put(consumingInstanceList.get(2), "CONSUMING"); idealState.setInstanceStateMap("segment0", instanceStateMap0); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); instanceStateMap1.put(consumingInstanceList.get(1), "CONSUMING"); instanceStateMap1.put(consumingInstanceList.get(2), "CONSUMING"); idealState.setInstanceStateMap("segment1", instanceStateMap1); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); instanceStateMap2.put(consumingInstanceList.get(1), "ONLINE"); instanceStateMap2.put(consumingInstanceList.get(2), "ONLINE"); idealState.setInstanceStateMap("segment2", instanceStateMap2); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); instanceStateMap3.put("notAConsumingServer_1", "ONLINE"); instanceStateMap3.put("notAConsumingServer_2", "ONLINE"); idealState.setInstanceStateMap("segment3", instanceStateMap3); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord())));
stateMap.put(_participants[0].getInstanceName(), "SLAVE"); stateMap.put(_participants[1].getInstanceName(), "DROPPED"); idealState.setInstanceStateMap(partitionName, stateMap);
for (Partition partition : assignmentToPersist.keySet()) { Map<String, String> instanceMap = assignmentToPersist.get(partition); idealState.setInstanceStateMap(partition.getPartitionName(), instanceMap);
for (Partition partition : assignmentToPersist.keySet()) { Map<String, String> instanceMap = assignmentToPersist.get(partition); idealState.setInstanceStateMap(partition.getPartitionName(), instanceMap);