@Override public String getPartitionRange() { return Integer.toString(getPartitionId()); }
/** * Retrieve the partition Id from the segment name of the realtime segment * * @param segmentName the name of the realtime segment * @return partition id of the segment */ private int getPartitionId(String segmentName) { final LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); return llcSegmentName.getPartitionId(); } }
public int getNumPartitionsFromIdealState(IdealState idealState) { Set<Integer> partitions = new HashSet<>(); Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); partitions.add(llcSegmentName.getPartitionId()); } } return partitions.size(); }
/** * Generates a map of partition id to latest llc segment * @param idealState * @return */ @VisibleForTesting public Map<String, LLCSegmentName> getPartitionToLatestSegments(IdealState idealState) { Map<String, LLCSegmentName> partitionIdToLatestSegment = new HashMap<>(); // read all segments Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); // get latest segment in each partition for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); String partitionId = String.valueOf(llcSegmentName.getPartitionId()); LLCSegmentName latestSegment = partitionIdToLatestSegment.get(partitionId); if (latestSegment == null || llcSegmentName.getSequenceNumber() > latestSegment.getSequenceNumber()) { partitionIdToLatestSegment.put(partitionId, llcSegmentName); } } } return partitionIdToLatestSegment; }
/** * Assigns new segments to instances by referring to the partition assignment * @param newSegments segments to assign * @param partitionAssignment partition assignment for the table to which the segments belong * @return map of segment name to instances list */ public Map<String, List<String>> assign(Collection<String> newSegments, PartitionAssignment partitionAssignment) throws InvalidConfigException { Map<String, List<String>> segmentAssignment = new HashMap<>(newSegments.size()); for (String segmentName : newSegments) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); List<String> instancesListForPartition = partitionAssignment.getInstancesListForPartition(String.valueOf(partitionId)); if (instancesListForPartition == null) { throw new InvalidConfigException( "No partition assignment " + partitionId + " found for segment " + segmentName); } segmentAssignment.put(segmentName, instancesListForPartition); } } return segmentAssignment; } }
@Override public int compareTo(Object o) { LLCSegmentName other = (LLCSegmentName) o; if (!this.getTableName().equals(other.getTableName())) { throw new RuntimeException( "Cannot compare segment names " + this.getSegmentName() + " and " + other.getSegmentName()); } if (this.getPartitionId() > other.getPartitionId()) { return 1; } else if (this.getPartitionId() < other.getPartitionId()) { return -1; } else { if (this.getSequenceNumber() > other.getSequenceNumber()) { return 1; } else if (this.getSequenceNumber() < other.getSequenceNumber()) { return -1; } else { if (!this.getCreationTime().equals(other.getCreationTime())) { // If sequence number is the same, time cannot be different. throw new RuntimeException( "Cannot compare segment names " + this.getSegmentName() + " and " + other.getSegmentName()); } return 0; } } }
if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); LLCSegmentName latestSegmentForPartition = partitionIdToLatestSegment.get(partitionId); if (latestSegmentForPartition == null || llcSegmentName.getSequenceNumber() > latestSegmentForPartition int partitionId = llcSegmentName.getPartitionId(); Map<String, String> instanceStateMap = new HashMap<>(); for (String instance : partitionToInstanceMap.get(String.valueOf(partitionId))) {
private void verifyAssignmentIsFromLatest(StreamPartitionAssignmentGenerator streamPartitionAssignmentGenerator, IdealState idealState, Map<String, List<String>> assignment) { Map<String, LLCSegmentName> partitionToLatestSegments = streamPartitionAssignmentGenerator.getPartitionToLatestSegments(idealState); for (Map.Entry<String, List<String>> entry : assignment.entrySet()) { String segmentName = entry.getKey(); List<String> assignedInstances = entry.getValue(); LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); LLCSegmentName latestSegment = partitionToLatestSegments.get(String.valueOf(partitionId)); Set<String> instancesInIdealState = idealState.getInstanceStateMap(latestSegment.getSegmentName()).keySet(); Assert.assertEquals(assignedInstances.size(), instancesInIdealState.size()); Assert.assertTrue(assignedInstances.containsAll(instancesInIdealState)); } }
for (String llcRealtimeSegment : llcRealtimeSegments) { LLCSegmentName segmentName = new LLCSegmentName(llcRealtimeSegment); partitionToLatestSegmentsMap.compute(segmentName.getPartitionId(), (partitionId, segmentNames) -> { if (segmentNames == null) { return new LLCSegmentName[]{segmentName, null};
/** * Verifies that segments in segment assignment matches input list * Verifies that segment assignment is as expected given the partition assignment * @param newSegments * @param partitionAssignment * @param expectException */ private void verifyAssignment(List<String> newSegments, PartitionAssignment partitionAssignment, boolean expectException, List<String> expectedSegmentsInSegmentAssignment) { ConsumingSegmentAssignmentStrategy strategy = new ConsumingSegmentAssignmentStrategy(); try { Map<String, List<String>> segmentAssignment = strategy.assign(newSegments, partitionAssignment); Assert.assertFalse(expectException); Assert.assertEquals(segmentAssignment.keySet().size(), expectedSegmentsInSegmentAssignment.size()); Assert.assertTrue(segmentAssignment.keySet().containsAll(expectedSegmentsInSegmentAssignment)); for (String segmentName : expectedSegmentsInSegmentAssignment) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); List<String> expectedInstances = partitionAssignment.getInstancesListForPartition(String.valueOf(partitionId)); List<String> assignedInstances = segmentAssignment.get(segmentName); Assert.assertEquals(expectedInstances.size(), assignedInstances.size()); Assert.assertTrue(expectedInstances.containsAll(assignedInstances)); } } catch (InvalidConfigException e) { Assert.assertTrue(expectException); } }
private ExternalView buildExternalView(String tableName, FakePropertyStore propertyStore, Map<Integer, List<String>> partitionToServerMapping, List<String> segmentList) throws Exception { // Create External View ExternalView externalView = new ExternalView(tableName); for (String segmentName : segmentList) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); for (String server : partitionToServerMapping.get(partitionId)) { externalView.setState(segmentName, server, "ONLINE"); } } return externalView; }
public List<String> getInstances(int partition, int seqNum) { List<String> instances = new ArrayList<>(); for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); instances = Lists.newArrayList(instanceStateMap.keySet()); break; } } } return instances; }
public IdealStateBuilderUtil setSegmentState(int partition, int seqNum, String state) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { instanceStateMap.put(entry.getKey(), state); } break; } } } return this; }
public IdealStateBuilderUtil moveToServers(int partition, int seqNum, List<String> instances) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); Map<String, String> newInstanceStateMap = new HashMap<>(instanceStateMap.size()); int serverId = 0; for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { newInstanceStateMap.put(instances.get(serverId++), entry.getValue()); } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), newInstanceStateMap); break; } } } return this; }
oldPartitions.add(llcSegmentName.getPartitionId()); String segmentName = entry.getKey(); LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId(); partitionToAllSegmentsMap.get(partitionId).add(segmentName);
public String getSegment(int partition, int seqNum) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { return segmentName; } } } return null; }
private void testCommitSegmentEntries(FakePinotLLCRealtimeSegmentManager segmentManager, LLCRealtimeSegmentZKMetadata committingSegmentMetadata, ZNRecord oldZnRec, ZNRecord newZnRec, Set<String> prevInstances, PartitionAssignment partitionAssignment, int partition) { // Get the old and new segment metadata and make sure that they are correct. LLCRealtimeSegmentZKMetadata oldMetadata = new LLCRealtimeSegmentZKMetadata(oldZnRec); LLCRealtimeSegmentZKMetadata newMetadata = new LLCRealtimeSegmentZKMetadata(newZnRec); LLCSegmentName oldSegmentName = new LLCSegmentName(oldMetadata.getSegmentName()); LLCSegmentName newSegmentName = new LLCSegmentName(newMetadata.getSegmentName()); // Assert on propertystore entries Assert.assertEquals(oldSegmentName.getSegmentName(), committingSegmentMetadata.getSegmentName()); Assert.assertEquals(newSegmentName.getPartitionId(), oldSegmentName.getPartitionId()); Assert.assertEquals(newSegmentName.getSequenceNumber(), oldSegmentName.getSequenceNumber() + 1); Assert.assertEquals(oldMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.DONE); Assert.assertEquals(newMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.IN_PROGRESS); Assert.assertNotNull(oldMetadata.getDownloadUrl()); Assert.assertEquals(Long.valueOf(oldMetadata.getCrc()), Long.valueOf(FakePinotLLCRealtimeSegmentManager.CRC)); Assert.assertEquals(oldMetadata.getStartTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getStartMillis()); Assert.assertEquals(oldMetadata.getEndTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getEndMillis()); Assert.assertEquals(oldMetadata.getTotalRawDocs(), FakePinotLLCRealtimeSegmentManager.NUM_DOCS); Assert.assertEquals(oldMetadata.getIndexVersion(), FakePinotLLCRealtimeSegmentManager.SEGMENT_VERSION); // check ideal state IdealState idealState = segmentManager._tableIdealState; Set<String> currInstances = idealState.getInstanceSet(oldSegmentName.getSegmentName()); Assert.assertEquals(prevInstances.size(), currInstances.size()); Assert.assertTrue(prevInstances.containsAll(currInstances)); Set<String> newSegmentInstances = idealState.getInstanceSet(newSegmentName.getSegmentName()); List<String> expectedNewInstances = partitionAssignment.getInstancesListForPartition(String.valueOf(partition)); Assert.assertEquals(newSegmentInstances.size(), expectedNewInstances.size()); Assert.assertTrue(newSegmentInstances.containsAll(expectedNewInstances)); }
private ZNRecord makeZnRecordForNewSegment(String realtimeTableName, int numReplicas, long startOffset, LLCSegmentName newSegmentName, int numPartitions) { final LLCRealtimeSegmentZKMetadata newSegMetadata = new LLCRealtimeSegmentZKMetadata(); newSegMetadata.setCreationTime(System.currentTimeMillis()); newSegMetadata.setStartOffset(startOffset); newSegMetadata.setEndOffset(END_OFFSET_FOR_CONSUMING_SEGMENTS); newSegMetadata.setNumReplicas(numReplicas); newSegMetadata.setTableName(realtimeTableName); newSegMetadata.setSegmentName(newSegmentName.getSegmentName()); newSegMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS); // Add the partition metadata if available. SegmentPartitionMetadata partitionMetadata = getPartitionMetadataFromTableConfig(realtimeTableName, numPartitions, newSegmentName.getPartitionId()); if (partitionMetadata != null) { newSegMetadata.setPartitionMetadata(partitionMetadata); } return newSegMetadata.toZNRecord(); }
Assert.assertEquals(segName1.getPartitionId(), partitionId); Assert.assertEquals(segName1.getCreationTime(), creationTime); Assert.assertEquals(segName1.getSequenceNumber(), sequenceNumber); Assert.assertEquals(segName2.getPartitionId(), partitionId); Assert.assertEquals(segName2.getCreationTime(), creationTime); Assert.assertEquals(segName2.getSequenceNumber(), sequenceNumber);
int partitionId = newLLCSegmentName.getPartitionId(); int numReplicas = partitionAssignment.getInstancesListForPartition(String.valueOf(partitionId)).size(); ZNRecord newZnRecord =