public LLCSegmentName(String segmentName) { if (!isLowLevelConsumerSegmentName(segmentName)) { throw new RuntimeException(segmentName + " is not a Low level consumer segment name"); } String[] parts = segmentName.split(SEPARATOR); _segmentName = segmentName; _tableName = parts[0]; _partitionId = Integer.valueOf(parts[1]); _sequenceNumber = Integer.valueOf(parts[2]); _creationTime = parts[3]; }
public int getNumPartitionsFromIdealState(IdealState idealState) { Set<Integer> partitions = new HashSet<>(); Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); partitions.add(llcSegmentName.getPartitionId()); } } return partitions.size(); }
/** * Assigns new segments to instances by referring to the partition assignment * @param newSegments segments to assign * @param partitionAssignment partition assignment for the table to which the segments belong * @return map of segment name to instances list */ public Map<String, List<String>> assign(Collection<String> newSegments, PartitionAssignment partitionAssignment) throws InvalidConfigException { Map<String, List<String>> segmentAssignment = new HashMap<>(newSegments.size()); for (String segmentName : newSegments) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); List<String> instancesListForPartition = partitionAssignment.getInstancesListForPartition(String.valueOf(partitionId)); if (instancesListForPartition == null) { throw new InvalidConfigException( "No partition assignment " + partitionId + " found for segment " + segmentName); } segmentAssignment.put(segmentName, instancesListForPartition); } } return segmentAssignment; } }
/** * Generates a map of partition id to latest llc segment * @param idealState * @return */ @VisibleForTesting public Map<String, LLCSegmentName> getPartitionToLatestSegments(IdealState idealState) { Map<String, LLCSegmentName> partitionIdToLatestSegment = new HashMap<>(); // read all segments Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); // get latest segment in each partition for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); String partitionId = String.valueOf(llcSegmentName.getPartitionId()); LLCSegmentName latestSegment = partitionIdToLatestSegment.get(partitionId); if (latestSegment == null || llcSegmentName.getSequenceNumber() > latestSegment.getSequenceNumber()) { partitionIdToLatestSegment.put(partitionId, llcSegmentName); } } } return partitionIdToLatestSegment; }
public List<String> getInstances(int partition, int seqNum) { List<String> instances = new ArrayList<>(); for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); instances = Lists.newArrayList(instanceStateMap.keySet()); break; } } } return instances; }
public IdealStateBuilderUtil setSegmentState(int partition, int seqNum, String state) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { instanceStateMap.put(entry.getKey(), state); } break; } } } return this; }
public IdealStateBuilderUtil moveToServers(int partition, int seqNum, List<String> instances) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); Map<String, String> newInstanceStateMap = new HashMap<>(instanceStateMap.size()); int serverId = 0; for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { newInstanceStateMap.put(instances.get(serverId++), entry.getValue()); } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), newInstanceStateMap); break; } } } return this; }
difference.removeAll(oldSegmentsSet); for (String addedSegment : difference) { if (LLCSegmentName.isLowLevelConsumerSegmentName(addedSegment)) { LLCSegmentName segmentName = new LLCSegmentName(addedSegment); Set<String> actualInstances = mapFields.get(addedSegment).keySet();
public String getSegment(int partition, int seqNum) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { return segmentName; } } } return null; }