@Override protected Map<String, String> getPartitionStateMap(CurrentState state) { return state.getPartitionStateMap(); } }
private void logSegmentsLoadingInfo() { InstanceDataManager instanceDataManager = _serverInstance.getInstanceDataManager(); if (instanceDataManager == null) { return; } HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor(); Builder keyBuilder = helixDataAccessor.keyBuilder(); LiveInstance liveInstance = helixDataAccessor.getProperty(keyBuilder.liveInstance(_instanceId)); String sessionId = liveInstance.getSessionId(); List<String> tableNames = _helixAdmin.getResourcesInCluster(_helixClusterName); for (String tableName : tableNames) { PropertyKey currentStateKey = keyBuilder.currentState(_instanceId, sessionId, tableName); CurrentState currentState = helixDataAccessor.getProperty(currentStateKey); int numSegmentsLoaded = instanceDataManager.getAllSegmentsMetadata(tableName).size(); if (currentState != null && currentState.isValid()) { int numSegmentsToLoad = currentState.getPartitionStateMap().size(); LOGGER.info( "Segments are not fully loaded during server bootstrap, current progress: table: {}, segments loading progress [ {} / {} ]", tableName, numSegmentsLoaded, numSegmentsToLoad); } } }
@Override public ZNRecord update(ZNRecord currentData) { CurrentState curState = null; if (currentData == null) { curState = new CurrentState(_lastCurState.getId()); // copy all simple fields settings and overwrite session-id to current session curState.getRecord().setSimpleFields(_lastCurState.getRecord().getSimpleFields()); curState.setSessionId(_curSessionId); } else { curState = new CurrentState(currentData); } for (String partitionName : _lastCurState.getPartitionStateMap().keySet()) { // carry-over only when current-state not exist if (curState.getState(partitionName) == null) { curState.setState(partitionName, _initState); } } return curState.getRecord(); }
if (!instanceSessionId.equals(currentState.getSessionId())) { continue; String resourceName = currentState.getResourceName(); String stateModelDefName = currentState.getStateModelDefRef(); Resource resource = resourceMap.get(resourceName); if (resource == null) { currentStateOutput.setBucketSize(resourceName, currentState.getBucketSize()); Map<String, String> partitionStateMap = currentState.getPartitionStateMap(); for (String partitionName : partitionStateMap.keySet()) { Partition partition = resource.getPartition(partitionName); if (partition != null) { currentStateOutput.setCurrentState(resourceName, partition, instanceName, currentState.getState(partitionName)); currentStateOutput.setRequestedState(resourceName, partition, instanceName, currentState.getRequestedState(partitionName)); currentStateOutput .setInfo(resourceName, partition, instanceName, currentState.getInfo(partitionName)); currentStateOutput.setEndTime(resourceName, partition, instanceName, currentState.getEndTime(partitionName));
LOG.info("Carrying over old session: " + session + ", resource: " + lastCurState.getId() + " to current session: " + _sessionId); String stateModelDefRef = lastCurState.getStateModelDefRef(); if (stateModelDefRef == null) { LOG.error("skip carry-over because previous current state doesn't have a state model definition. previous current-state: " _keyBuilder.currentState(_instanceName, _sessionId, lastCurState.getResourceName()) .getPath(); if (lastCurState.getBucketSize() > 0) { ZNRecord metaRecord = new ZNRecord(lastCurState.getId()); metaRecord.setSimpleFields(lastCurState.getRecord().getSimpleFields()); DataUpdater<ZNRecord> metaRecordUpdater = new CurStateCarryOverUpdater(_sessionId, initState, new CurrentState(metaRecord)); boolean success = baseAccessor.update(curStatePath, metaRecordUpdater, AccessOption.PERSISTENT); if (success) { ZNRecordBucketizer bucketizer = new ZNRecordBucketizer(lastCurState.getBucketSize()); Map<String, ZNRecord> map = bucketizer.bucketize(lastCurState.getRecord()); List<String> paths = new ArrayList<String>(); List<DataUpdater<ZNRecord>> updaters = new ArrayList<DataUpdater<ZNRecord>>(); for (String bucketName : map.keySet()) { paths.add(curStatePath + "/" + bucketName); updaters.add(new CurStateCarryOverUpdater(_sessionId, initState, new CurrentState(map .get(bucketName))));
String resourceName = currentState.getResourceName(); Map<String, String> resourceStateMap = currentState.getPartitionStateMap(); addResource(resourceName, resourceMap); Resource resource = resourceMap.get(resourceName); resource.setStateModelDefRef(currentState.getStateModelDefRef()); resource.setStateModelFactoryName(currentState.getStateModelFactoryName()); resource.setBucketSize(currentState.getBucketSize()); resource.setBatchMessageMode(currentState.getBatchMessageMode()); if (resource.getStateModelDefRef() == null && !cache.isTaskCache() || resource.getStateModelDefRef() != null && ( if (currentState.getStateModelDefRef() == null) { LogUtil.logError(LOG, _eventId, "state model def is null." + "resource:" + currentState.getResourceName() + ", partitions: " + currentState.getPartitionStateMap().keySet() + ", states: " + currentState.getPartitionStateMap().values()); throw new StageException("State model def is null for resource:" + currentState.getResourceName());
private Map<String, CurrentState> generateCurrentStateMap( Map<String, CurrentStateInfo> currentStateRawData) { Map<String, CurrentState> currentStateMap = new HashMap<String, CurrentState>(); for (String instanceName : currentStateRawData.keySet()) { CurrentStateInfo info = currentStateRawData.get(instanceName); CurrentState currentState = new CurrentState(TEST_RESOURCE); currentState.setSessionId(SESSION_PREFIX + instanceName.split("_")[1]); currentState.setState(PARTITION, info.currentState); currentState.setPreviousState(PARTITION, info.previousState); currentState.setStartTime(PARTITION, info.startTime); currentState.setEndTime(PARTITION, info.endTime); currentStateMap.put(instanceName, currentState); } return currentStateMap; }
_currentStateDelta.setInfo(partitionKey, taskResult.getInfo()); _currentStateDelta.setEndTime(partitionKey, taskResult.getCompleteTime()); _currentStateDelta.setPreviousState(partitionKey, _message.getFromState()); _currentStateDelta.setTriggerHost(partitionKey, _message.getRelaySrcHost()); } else { _currentStateDelta.setTriggerHost(partitionKey, _message.getMsgSrc()); _currentStateDelta.setState(partitionKey, toState); ZNRecord rec = new ZNRecord(_currentStateDelta.getId()); rec.getMapFields().put(partitionKey, null); ZNRecordDelta delta = new ZNRecordDelta(rec, MergeOperation.SUBTRACT); _currentStateDelta.setDeltaList(deltaList); _stateModelFactory.removeStateModel(resource, partitionKey); } else { + partitionKey + ", currentState: " + _stateModel.getCurrentState() + ", message: " + _message); _currentStateDelta.setState(partitionKey, _stateModel.getCurrentState()); } else { StateTransitionError error = _currentStateDelta.setState(partitionKey, HelixDefinedState.ERROR.toString()); _stateModel.updateState(HelixDefinedState.ERROR.toString());
private void setCurrentState(String clusterName, String instance, String resourceGroupName, String resourceKey, String sessionId, String state, boolean updateTimestamp) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); CurrentState curState = new CurrentState(resourceGroupName); curState.setState(resourceKey, state); curState.setSessionId(sessionId); curState.setStateModelDefRef("MasterSlave"); if (updateTimestamp) { curState.setEndTime(resourceKey, System.currentTimeMillis()); } accessor.setProperty(keyBuilder.currentState(instance, sessionId, resourceGroupName), curState); }
CurrentState currentStateDelta = new CurrentState(resourceName); currentStateDelta.setSessionId(sessionId); currentStateDelta.setStateModelDefRef(stateModelName); currentStateDelta.setStateModelFactoryName(factoryName); currentStateDelta.setBucketSize(bucketSize); currentStateDelta.setState(partitionKey, (stateModel.getCurrentState() == null) ? initState : stateModel.getCurrentState());
CurrentState currentState = new CurrentState(oldResource); currentState.setState("testResourceOld_0", "OFFLINE"); currentState.setState("testResourceOld_1", "SLAVE"); currentState.setState("testResourceOld_2", "MASTER"); currentState.setStateModelDefRef("MasterSlave"); accessor.setProperty(keyBuilder.currentState(instanceName, sessionId, oldResource), currentState); AssertJUnit.assertEquals(resourceMap.get(oldResource).getResourceName(), oldResource); AssertJUnit.assertEquals(resourceMap.get(oldResource).getStateModelDefRef(), currentState.getStateModelDefRef()); AssertJUnit.assertEquals(resourceMap.get(oldResource).getPartitions().size(), currentState .getPartitionStateMap().size()); AssertJUnit.assertNotNull(resourceMap.get(oldResource).getPartition("testResourceOld_0")); AssertJUnit.assertNotNull(resourceMap.get(oldResource).getPartition("testResourceOld_1"));
createCurStateKeys.add(keyBuilder.currentState(instanceName, sessionId, resourceName)); CurrentState metaCurState = new CurrentState(resourceName); metaCurState.setBucketSize(message.getBucketSize()); metaCurState.setStateModelDefRef(message.getStateModelDef()); metaCurState.setSessionId(sessionId); metaCurState.setBatchMessageMode(message.getBatchMessageMode()); String ftyName = message.getStateModelFactoryName(); if (ftyName != null) { metaCurState.setStateModelFactoryName(ftyName); } else { metaCurState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);
private void verifyP2PEnabled(long startTime) { ClusterDataCache dataCache = new ClusterDataCache(CLUSTER_NAME); dataCache.refresh(_accessor); Map<String, LiveInstance> liveInstanceMap = dataCache.getLiveInstances(); for (LiveInstance instance : liveInstanceMap.values()) { Map<String, CurrentState> currentStateMap = dataCache.getCurrentState(instance.getInstanceName(), instance.getSessionId()); Assert.assertNotNull(currentStateMap); for (CurrentState currentState : currentStateMap.values()) { for (String partition : currentState.getPartitionStateMap().keySet()) { String state = currentState.getState(partition); long start = currentState.getStartTime(partition); if (state.equalsIgnoreCase("MASTER") && start > startTime) { String triggerHost = currentState.getTriggerHost(partition); if (!triggerHost.equals(_controllerName)) { p2pTrigged ++; } total ++; } } } } }
CurrentState currentState = currentStateMap.get(dbName); Assert.assertNotNull(currentState); Assert.assertEquals(currentState.getPartitionStateMap().size(), PARTITION_NUMBER); for (String partition : currentState.getPartitionStateMap().keySet()) { String state = currentState.getState(partition); Assert.assertEquals(state, expectedState, dbName + " Partition " + partition + "'s state is different as expected!"); String triggerHost = currentState.getTriggerHost(partition); if (triggerHost.equals(expectedTriggerHost)) { expectedHost ++;
String resourceName = currentState.getResourceName(); Map<String, String> stateMap = currentState.getPartitionStateMap();
: _currentStateDelta.getState(partitionName); _currentStateDelta.setStartTime(_message.getPartitionName(), System.currentTimeMillis()); List<ZNRecordDelta> deltaList = new ArrayList<ZNRecordDelta>(); deltaList.add(delta); CurrentState currStateUpdate = new CurrentState(resource); currStateUpdate.setDeltaList(deltaList);
for (String partition : curState.getPartitionStateMap().keySet()) { long newEndTime = curState.getEndTime(partition); if (prevState == null || prevState.getEndTime(partition) < newEndTime) { if (partitionUpdateEndTimes == null) { partitionUpdateEndTimes = new HashMap<>();
CurrentState currentStateDelta = new CurrentState(resourceName); currentStateDelta.setState(partition, HelixDefinedState.ERROR.toString()); _stateModel.updateState(HelixDefinedState.ERROR.toString());
@Override public boolean isValid() { if (getStateModelDefRef() == null) { LOG.error("Current state does not contain state model ref. id:" + getResourceName()); return false; } if (getSessionId() == null) { LOG.error("CurrentState does not contain session id, id : " + getResourceName()); return false; } return true; }
&& currentState.getPreviousState(partition.getPartitionName()) != null && currentState .getPreviousState(partition.getPartitionName()).equalsIgnoreCase(topState)) { long fromTopStateStartTime = currentState.getStartTime(partition.getPartitionName()); if (fromTopStateStartTime > startTime) { startTime = fromTopStateStartTime; fromTopStateUserLatency = currentState.getEndTime(partition.getPartitionName()) - startTime; startTime = Math.max(startTime, currentState.getStartTime(partition.getPartitionName()));