private LLCSegmentName makeNextLLCSegmentName(LLCSegmentName segmentName, int partition, long now) { final int newSeqNum = segmentName.getSequenceNumber() + 1; LLCSegmentName newLLCSegmentName = new LLCSegmentName(segmentName.getTableName(), partition, newSeqNum, now); return newLLCSegmentName; }
/** * Generates a map of partition id to latest llc segment * @param idealState * @return */ @VisibleForTesting public Map<String, LLCSegmentName> getPartitionToLatestSegments(IdealState idealState) { Map<String, LLCSegmentName> partitionIdToLatestSegment = new HashMap<>(); // read all segments Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); // get latest segment in each partition for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); String partitionId = String.valueOf(llcSegmentName.getPartitionId()); LLCSegmentName latestSegment = partitionIdToLatestSegment.get(partitionId); if (latestSegment == null || llcSegmentName.getSequenceNumber() > latestSegment.getSequenceNumber()) { partitionIdToLatestSegment.put(partitionId, llcSegmentName); } } } return partitionIdToLatestSegment; }
return new LLCSegmentName[]{segmentName, null}; } else { if (segmentName.getSequenceNumber() > segmentNames[0].getSequenceNumber()) { segmentNames[1] = segmentNames[0]; segmentNames[0] = segmentName; } else if (segmentNames[1] == null || segmentName.getSequenceNumber() > segmentNames[1].getSequenceNumber()) { segmentNames[1] = segmentName;
@Override public int compareTo(Object o) { LLCSegmentName other = (LLCSegmentName) o; if (!this.getTableName().equals(other.getTableName())) { throw new RuntimeException( "Cannot compare segment names " + this.getSegmentName() + " and " + other.getSegmentName()); } if (this.getPartitionId() > other.getPartitionId()) { return 1; } else if (this.getPartitionId() < other.getPartitionId()) { return -1; } else { if (this.getSequenceNumber() > other.getSequenceNumber()) { return 1; } else if (this.getSequenceNumber() < other.getSequenceNumber()) { return -1; } else { if (!this.getCreationTime().equals(other.getCreationTime())) { // If sequence number is the same, time cannot be different. throw new RuntimeException( "Cannot compare segment names " + this.getSegmentName() + " and " + other.getSegmentName()); } return 0; } } }
int partitionId = llcSegmentName.getPartitionId(); LLCSegmentName latestSegmentForPartition = partitionIdToLatestSegment.get(partitionId); if (latestSegmentForPartition == null || llcSegmentName.getSequenceNumber() > latestSegmentForPartition .getSequenceNumber()) { partitionIdToLatestSegment.put(partitionId, llcSegmentName);
public List<String> getInstances(int partition, int seqNum) { List<String> instances = new ArrayList<>(); for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); instances = Lists.newArrayList(instanceStateMap.keySet()); break; } } } return instances; }
public IdealStateBuilderUtil setSegmentState(int partition, int seqNum, String state) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { instanceStateMap.put(entry.getKey(), state); } break; } } } return this; }
final LLCSegmentName committingLLCSegmentName = new LLCSegmentName(committingSegmentNameStr); final int partitionId = committingLLCSegmentName.getPartitionId(); final int newSeqNum = committingLLCSegmentName.getSequenceNumber() + 1; final long now = System.currentTimeMillis();
public IdealStateBuilderUtil moveToServers(int partition, int seqNum, List<String> instances) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); Map<String, String> newInstanceStateMap = new HashMap<>(instanceStateMap.size()); int serverId = 0; for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { newInstanceStateMap.put(instances.get(serverId++), entry.getValue()); } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), newInstanceStateMap); break; } } } return this; }
public String getSegment(int partition, int seqNum) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { return segmentName; } } } return null; }
partition); startOffset = getBetterStartOffsetIfNeeded(tableNameWithType, partition, segmentName, startOffset, newLLCSegmentName.getSequenceNumber());
private void testCommitSegmentEntries(FakePinotLLCRealtimeSegmentManager segmentManager, LLCRealtimeSegmentZKMetadata committingSegmentMetadata, ZNRecord oldZnRec, ZNRecord newZnRec, Set<String> prevInstances, PartitionAssignment partitionAssignment, int partition) { // Get the old and new segment metadata and make sure that they are correct. LLCRealtimeSegmentZKMetadata oldMetadata = new LLCRealtimeSegmentZKMetadata(oldZnRec); LLCRealtimeSegmentZKMetadata newMetadata = new LLCRealtimeSegmentZKMetadata(newZnRec); LLCSegmentName oldSegmentName = new LLCSegmentName(oldMetadata.getSegmentName()); LLCSegmentName newSegmentName = new LLCSegmentName(newMetadata.getSegmentName()); // Assert on propertystore entries Assert.assertEquals(oldSegmentName.getSegmentName(), committingSegmentMetadata.getSegmentName()); Assert.assertEquals(newSegmentName.getPartitionId(), oldSegmentName.getPartitionId()); Assert.assertEquals(newSegmentName.getSequenceNumber(), oldSegmentName.getSequenceNumber() + 1); Assert.assertEquals(oldMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.DONE); Assert.assertEquals(newMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.IN_PROGRESS); Assert.assertNotNull(oldMetadata.getDownloadUrl()); Assert.assertEquals(Long.valueOf(oldMetadata.getCrc()), Long.valueOf(FakePinotLLCRealtimeSegmentManager.CRC)); Assert.assertEquals(oldMetadata.getStartTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getStartMillis()); Assert.assertEquals(oldMetadata.getEndTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getEndMillis()); Assert.assertEquals(oldMetadata.getTotalRawDocs(), FakePinotLLCRealtimeSegmentManager.NUM_DOCS); Assert.assertEquals(oldMetadata.getIndexVersion(), FakePinotLLCRealtimeSegmentManager.SEGMENT_VERSION); // check ideal state IdealState idealState = segmentManager._tableIdealState; Set<String> currInstances = idealState.getInstanceSet(oldSegmentName.getSegmentName()); Assert.assertEquals(prevInstances.size(), currInstances.size()); Assert.assertTrue(prevInstances.containsAll(currInstances)); Set<String> newSegmentInstances = idealState.getInstanceSet(newSegmentName.getSegmentName()); List<String> expectedNewInstances = partitionAssignment.getInstancesListForPartition(String.valueOf(partition)); Assert.assertEquals(newSegmentInstances.size(), expectedNewInstances.size()); Assert.assertTrue(newSegmentInstances.containsAll(expectedNewInstances)); }
Assert.assertEquals(segName1.getPartitionId(), partitionId); Assert.assertEquals(segName1.getCreationTime(), creationTime); Assert.assertEquals(segName1.getSequenceNumber(), sequenceNumber); Assert.assertEquals(segName1.getTableName(), tableName); Assert.assertEquals(segName2.getPartitionId(), partitionId); Assert.assertEquals(segName2.getCreationTime(), creationTime); Assert.assertEquals(segName2.getSequenceNumber(), sequenceNumber); Assert.assertEquals(segName2.getTableName(), tableName);
assertEquals(shortNameSegment.getSequenceNumberStr(), "1234567"); assertEquals(shortNameSegment.getSequenceNumber(), 1234567); assertEquals(llcSegment.getSequenceNumber(), 1);
.updateOldSegmentMetadataZNRecord(tableName, latestSegment, latestMetadata.getStartOffset() + 100); LLCSegmentName newLlcSegmentName = new LLCSegmentName(rawTableName, randomlySelectedPartition, latestSegment.getSequenceNumber() + 1, System.currentTimeMillis()); CommittingSegmentDescriptor committingSegmentDescriptor =