/** * Remove all segments from the ideal state, other than LLC ONLINE segments. We only want LLC ONLINE segments for this rebalance * We do not want to rebalance CONSUMING segments as part of this rebalance, it is done separately * * If any segments were OFFLINE instead of ONLINE, we will not rebalance them, * as that state was likely achieved because of some manual operation * @param mapFields * @param removedEntries */ private void filterSegmentsForRealtimeRebalance(Map<String, Map<String, String>> mapFields, Map<String, Map<String, String>> removedEntries) { Iterator<Map.Entry<String, Map<String, String>>> mapFieldsIterator = mapFields.entrySet().iterator(); while (mapFieldsIterator.hasNext()) { // Only keep LLC segments in ONLINE state for rebalance Map.Entry<String, Map<String, String>> entry = mapFieldsIterator.next(); final String segmentName = entry.getKey(); boolean keep = false; if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { Map<String, String> instanceStateMap = entry.getValue(); if (instanceStateMap.values() .contains(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.ONLINE)) { keep = true; } } if (!keep) { removedEntries.put(segmentName, entry.getValue()); mapFieldsIterator.remove(); } } }
@Transition(from = "OFFLINE", to = "CONSUMING") public void onBecomeConsumingFromOffline(Message message, NotificationContext context) { Preconditions.checkState(SegmentName.isLowLevelConsumerSegmentName(message.getPartitionName()), "Tried to go into CONSUMING state on non-low level segment"); _logger.info("SegmentOnlineOfflineStateModel.onBecomeConsumingFromOffline() : " + message); // We do the same processing as usual for going to the consuming state, which adds the segment to the table data // manager and starts Kafka consumption onBecomeOnlineFromOffline(message, context); }
public void cleanupLLC(final String realtimeTableName) { // If there are any completions in the pipeline we let them commit. Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); IdealState idealState = HelixHelper.getTableIdealState(_helixManager, realtimeTableName); final List<String> segmentsToRemove = new ArrayList<String>(); Set<String> allSegments = idealState.getPartitionSet(); int removeCount = 0; for (String segmentName : allSegments) { if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { segmentsToRemove.add(segmentName); removeCount++; } } LOGGER.info("Attempting to remove {} LLC segments of table {}", removeCount, realtimeTableName); _helixResourceManager.deleteSegments(realtimeTableName, segmentsToRemove); }
/** * Compute the table of a sorted list of segments grouped by Kafka partition. * * @param segmentSet is the set of segment names that need to be sorted. * @return map of Stream partition to sorted set of segment names */ public static Map<String, SortedSet<SegmentName>> sortSegmentsByStreamPartition(Set<String> segmentSet) { Map<String, SortedSet<SegmentName>> sortedSegmentsByStreamPartition = new HashMap<>(); for (String segment : segmentSet) { // Ignore segments that are not low level consumer segments if (!SegmentName.isLowLevelConsumerSegmentName(segment)) { continue; } final LLCSegmentName segmentName = new LLCSegmentName(segment); String streamPartitionId = segmentName.getPartitionRange(); SortedSet<SegmentName> segmentsForPartition = sortedSegmentsByStreamPartition.get(streamPartitionId); // Create sorted set if necessary if (segmentsForPartition == null) { segmentsForPartition = new TreeSet<>(); sortedSegmentsByStreamPartition.put(streamPartitionId, segmentsForPartition); } segmentsForPartition.add(segmentName); } return sortedSegmentsByStreamPartition; } }
/** * Returns the LLC realtime segments for the given table. * * @param propertyStore Helix property store * @param realtimeTableName Realtime table name * @return List of LLC realtime segment names */ public static List<String> getLLCRealtimeSegments(ZkHelixPropertyStore<ZNRecord> propertyStore, String realtimeTableName) { List<String> llcRealtimeSegments = new ArrayList<>(); String segmentsPath = constructPropertyStorePathForResource(realtimeTableName); if (propertyStore.exists(segmentsPath, AccessOption.PERSISTENT)) { List<String> segments = propertyStore.getChildNames(segmentsPath, AccessOption.PERSISTENT); for (String segment : segments) { if (SegmentName.isLowLevelConsumerSegmentName(segment)) { llcRealtimeSegments.add(segment); } } } return llcRealtimeSegments; }
public static RealtimeSegmentType getSegmentType(String segmentName) { if (isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName segName = new HLCSegmentName(segmentName); if (segName.isOldStyleNaming()) { return RealtimeSegmentType.HLC_LONG; } else { return RealtimeSegmentType.HLC_SHORT; } } if (isLowLevelConsumerSegmentName(segmentName)) { return RealtimeSegmentType.LLC; } return RealtimeSegmentType.UNSUPPORTED; }
if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId();
private void manageRetentionForRealtimeTable(String realtimeTableName, RetentionStrategy retentionStrategy) { List<String> segmentsToDelete = new ArrayList<>(); IdealState idealState = _pinotHelixResourceManager.getHelixAdmin() .getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), realtimeTableName); for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : _pinotHelixResourceManager .getRealtimeSegmentMetadata(realtimeTableName)) { String segmentName = realtimeSegmentZKMetadata.getSegmentName(); if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) { // In progress segment, only check LLC segment if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { // Delete old LLC segment that hangs around. Do not delete segment that are current since there may be a race // with RealtimeSegmentValidationManager trying to auto-create the LLC segment if (shouldDeleteInProgressLLCSegment(segmentName, idealState, realtimeSegmentZKMetadata)) { segmentsToDelete.add(segmentName); } } } else { // Sealed segment if (retentionStrategy.isPurgeable(realtimeSegmentZKMetadata)) { segmentsToDelete.add(segmentName); } } } if (!segmentsToDelete.isEmpty()) { LOGGER.info("Deleting segments: {} from table: {}", segmentsToDelete, realtimeTableName); _pinotHelixResourceManager.deleteSegments(realtimeTableName, segmentsToDelete); } }
@Override public void computeOnExternalViewChange(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigs) { Set<String> segmentSet = externalView.getPartitionSet(); for (String segmentName : segmentSet) { if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { _hasHLC = true; } if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { _hasLLC = true; } } if (_hasHLC) { _realtimeHLCRoutingTableBuilder.computeOnExternalViewChange(tableName, externalView, instanceConfigs); } if (_hasLLC) { _realtimeLLCRoutingTableBuilder.computeOnExternalViewChange(tableName, externalView, instanceConfigs); } }
protected RealtimeIndexOffHeapMemoryManager(ServerMetrics serverMetrics, String segmentName) { _serverMetrics = serverMetrics; _segmentName = segmentName; if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); _tableName = llcSegmentName.getTableName(); } else if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); _tableName = hlcSegmentName.getTableName(); } else { // For testing only _tableName = "NoSuchTable"; } }
/** * NOTE: this method is very expensive, use {@link #getLLCRealtimeSegments(ZkHelixPropertyStore, String)} instead if * only segment names are needed. */ public static List<LLCRealtimeSegmentZKMetadata> getLLCRealtimeSegmentZKMetadataListForTable( ZkHelixPropertyStore<ZNRecord> propertyStore, String resourceName) { List<LLCRealtimeSegmentZKMetadata> resultList = new ArrayList<>(); if (propertyStore == null) { return resultList; } String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(resourceName); if (propertyStore.exists(constructPropertyStorePathForResource(realtimeTableName), AccessOption.PERSISTENT)) { List<ZNRecord> znRecordList = propertyStore .getChildren(constructPropertyStorePathForResource(realtimeTableName), null, AccessOption.PERSISTENT); if (znRecordList != null) { for (ZNRecord record : znRecordList) { RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = new RealtimeSegmentZKMetadata(record); if (SegmentName.isLowLevelConsumerSegmentName(realtimeSegmentZKMetadata.getSegmentName())) { resultList.add(new LLCRealtimeSegmentZKMetadata(record)); } } } } return resultList; }