@Nullable public static RealtimeSegmentZKMetadata getRealtimeSegmentZKMetadata( @Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName, @Nonnull String segmentName) { String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName); ZNRecord znRecord = propertyStore .get(constructPropertyStorePathForSegment(realtimeTableName, segmentName), null, AccessOption.PERSISTENT); // It is possible that the segment metadata has just been deleted due to retention. if (znRecord == null) { return null; } if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { return new RealtimeSegmentZKMetadata(znRecord); } else { return new LLCRealtimeSegmentZKMetadata(znRecord); } }
@Override protected Map<String, List<String>> computeSegmentToServersMapFromExternalView(ExternalView externalView, List<InstanceConfig> instanceConfigs) { Map<String, List<String>> segmentToServersMap = new HashMap<>(); RoutingTableInstancePruner instancePruner = new RoutingTableInstancePruner(instanceConfigs); for (String segmentName : externalView.getPartitionSet()) { List<String> servers = new ArrayList<>(); for (Map.Entry<String, String> entry : externalView.getStateMap(segmentName).entrySet()) { String serverName = entry.getKey(); if (entry.getValue().equals(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.ONLINE) && !instancePruner.isInactive(serverName) && SegmentName.isHighLevelConsumerSegmentName(segmentName)) { servers.add(serverName); } } if (servers.size() != 0) { segmentToServersMap.put(segmentName, servers); } else { handleNoServingHost(segmentName); } } return segmentToServersMap; }
public static RealtimeSegmentType getSegmentType(String segmentName) { if (isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName segName = new HLCSegmentName(segmentName); if (segName.isOldStyleNaming()) { return RealtimeSegmentType.HLC_LONG; } else { return RealtimeSegmentType.HLC_SHORT; } } if (isLowLevelConsumerSegmentName(segmentName)) { return RealtimeSegmentType.LLC; } return RealtimeSegmentType.UNSUPPORTED; }
deletedSegmentDestURI.toString()); } else { if (!SegmentName.isHighLevelConsumerSegmentName(segmentId)) { LOGGER.warn("Not found local segment file for segment {}" + fileToMoveURI.toString());
@VisibleForTesting static long computeRealtimeTotalDocumentInSegments(List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadataList, boolean countHLCSegments) { long numTotalDocs = 0; String groupId = ""; for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadataList) { String segmentName = realtimeSegmentZKMetadata.getSegmentName(); if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { if (countHLCSegments) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); String segmentGroupIdName = hlcSegmentName.getGroupId(); if (groupId.isEmpty()) { groupId = segmentGroupIdName; } // Discard all segments with different groupids as they are replicas if (groupId.equals(segmentGroupIdName) && realtimeSegmentZKMetadata.getTotalRawDocs() >= 0) { numTotalDocs += realtimeSegmentZKMetadata.getTotalRawDocs(); } } } else { // Low level segments if (!countHLCSegments) { numTotalDocs += realtimeSegmentZKMetadata.getTotalRawDocs(); } } } return numTotalDocs; }
@Override public void computeOnExternalViewChange(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigs) { Set<String> segmentSet = externalView.getPartitionSet(); for (String segmentName : segmentSet) { if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { _hasHLC = true; } if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { _hasLLC = true; } } if (_hasHLC) { _realtimeHLCRoutingTableBuilder.computeOnExternalViewChange(tableName, externalView, instanceConfigs); } if (_hasLLC) { _realtimeLLCRoutingTableBuilder.computeOnExternalViewChange(tableName, externalView, instanceConfigs); } }
if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, this, _indexDir.getAbsolutePath(), indexLoadingConfig, schema, _serverMetrics);
protected RealtimeIndexOffHeapMemoryManager(ServerMetrics serverMetrics, String segmentName) { _serverMetrics = serverMetrics; _segmentName = segmentName; if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); _tableName = llcSegmentName.getTableName(); } else if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); _tableName = hlcSegmentName.getTableName(); } else { // For testing only _tableName = "NoSuchTable"; } }
if (SegmentName.isHighLevelConsumerSegmentName(partition)) { HLCSegmentName segName = new HLCSegmentName(partition); RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider
if (!SegmentName.isHighLevelConsumerSegmentName(segmentName)) { continue;
/** * * @param tableConfig * @param emptyIdealState may contain HLC segments if both HLC and LLC are configured */ public void setupNewTable(TableConfig tableConfig, IdealState emptyIdealState) throws InvalidConfigException { final StreamConfig streamConfig = new StreamConfig(tableConfig.getIndexingConfig().getStreamConfigs()); int partitionCount = getPartitionCount(streamConfig); List<String> currentSegments = getExistingSegments(tableConfig.getTableName()); // Make sure that there are no low-level segments existing. if (currentSegments != null) { for (String segment : currentSegments) { if (!SegmentName.isHighLevelConsumerSegmentName(segment)) { // For now, we don't support re-creating the low-level realtime segments throw new RuntimeException("Low-level segments already exist for table " + tableConfig.getTableType()); } } } _flushThresholdUpdateManager.clearFlushThresholdUpdater(tableConfig); if (!isConnected()) { throw new RuntimeException( "Lost zk connection while setting up new table " + tableConfig.getTableName() + " isConnected=" + isConnected()); } IdealState idealState = setupTable(tableConfig, emptyIdealState, partitionCount); setTableIdealState(tableConfig.getTableName(), idealState); }