public static IdealState getTableIdealState(HelixManager manager, String resourceName) { final HelixDataAccessor accessor = manager.getHelixDataAccessor(); final Builder builder = accessor.keyBuilder(); return accessor.getProperty(builder.idealStates(resourceName)); }
PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder(); helixDataAccessor.getBaseDataAccessor().getStats(externalViewPaths, AccessOption.PERSISTENT); long statEndTime = System.currentTimeMillis(); List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs()); long icFetchEnd = System.currentTimeMillis(); icFetchTime = icFetchEnd - icFetchStart; ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableThatChanged));
public List<String> getOnlineInstanceList() { return _helixDataAccessor.getChildNames(_keyBuilder.liveInstances()); }
/** * Returns the config for all the instances in the cluster. */ public static List<InstanceConfig> getInstanceConfigs(HelixManager helixManager) { HelixDataAccessor helixDataAccessor = helixManager.getHelixDataAccessor(); return helixDataAccessor.getChildValues(helixDataAccessor.keyBuilder().instanceConfigs()); }
HelixDataAccessor accessor = new ZKHelixDataAccessor(helixClusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient)); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.stateModelDef(segmentStateModelName), newStateModelDef); LOGGER.info("Completed updating statemodel {}", segmentStateModelName); zkClient.close();
private void updateHistory(HelixManager manager) { HelixDataAccessor accessor = manager.getHelixDataAccessor(); Builder keyBuilder = accessor.keyBuilder(); LeaderHistory history = accessor.getProperty(keyBuilder.controllerLeaderHistory()); if (history == null) { history = new LeaderHistory(PropertyType.HISTORY.toString()); } history.updateHistory(manager.getClusterName(), manager.getInstanceName(), manager.getVersion()); if(!accessor.setProperty(keyBuilder.controllerLeaderHistory(), history)) { LOG.error("Failed to persist leader history to ZK!"); } } }
public static List<String> liveInstances(HelixManager helixManager) { HelixDataAccessor helixDataAccessor = helixManager.getHelixDataAccessor(); PropertyKey liveInstancesKey = helixDataAccessor.keyBuilder().liveInstances(); return ImmutableList.copyOf(helixDataAccessor.getChildNames(liveInstancesKey)); }
@GET @Path("{instanceName}/resources/{resourceName}") public Response getResourceOnInstance(@PathParam("clusterId") String clusterId, @PathParam("instanceName") String instanceName, @PathParam("resourceName") String resourceName) throws IOException { HelixDataAccessor accessor = getDataAccssor(clusterId); List<String> sessionIds = accessor.getChildNames(accessor.keyBuilder().sessions(instanceName)); if (sessionIds == null || sessionIds.size() == 0) { return notFound(); } // Only get resource list from current session id String currentSessionId = sessionIds.get(0); CurrentState resourceCurrentState = accessor .getProperty(accessor.keyBuilder().currentState(instanceName, currentSessionId, resourceName)); if (resourceCurrentState != null) { return JSONRepresentation(resourceCurrentState.getRecord()); } return notFound(); }
private void updateIdealState() { HelixDataAccessor accessor = _manager.getHelixDataAccessor(); Builder keyBuilder = accessor.keyBuilder(); List<String> idealStates = accessor.getChildNames(keyBuilder.idealStates()); IdealState idealState = accessor.getProperty(keyBuilder.idealStates(idealStates.get(0))); idealState.setNumPartitions((int)(System.currentTimeMillis()%50L)); accessor.setProperty(keyBuilder.idealStates(idealState.getId()), idealState); } }
PropertyKey.Builder keyBuilder = accessor.keyBuilder(); properties = accessor.getChildValues(keyBuilder.liveInstances()); Set<String> liveParticipants = accessor.getChildValuesMap(keyBuilder.liveInstances()).keySet(); List<ZNRecordRow> result = Lists.newArrayList(); for (ZNRecordRow row : allRows) {
@Override public Boolean call() { HelixDataAccessor dataAccessor = helixManager.getHelixDataAccessor(); PropertyKey idealStateKey = dataAccessor.keyBuilder().idealStates(resourceName); IdealState idealState = dataAccessor.getProperty(idealStateKey); if (dataAccessor.getBaseDataAccessor() .set(idealStateKey.getPath(), updatedIdealState.getRecord(), idealState.getRecord().getVersion(), AccessOption.PERSISTENT)) {
/** * Create Helix cluster if needed, and then start a Pinot controller instance. */ public synchronized void start() { _helixZkManager = HelixSetupUtils .setup(_helixClusterName, _helixZkURL, _instanceId, _isUpdateStateModel, _enableBatchMessageMode); Preconditions.checkNotNull(_helixZkManager); _helixAdmin = _helixZkManager.getClusterManagmentTool(); _propertyStore = _helixZkManager.getHelixPropertyStore(); _helixDataAccessor = _helixZkManager.getHelixDataAccessor(); // Cache instance zk paths. BaseDataAccessor<ZNRecord> baseDataAccessor = _helixDataAccessor.getBaseDataAccessor(); String instanceConfigs = PropertyPathBuilder.instanceConfig(_helixClusterName); _cacheInstanceConfigsDataAccessor = new ZkCacheBaseDataAccessor<>((ZkBaseDataAccessor<ZNRecord>) baseDataAccessor, instanceConfigs, null, Collections.singletonList(instanceConfigs)); _keyBuilder = _helixDataAccessor.keyBuilder(); _segmentDeletionManager = new SegmentDeletionManager(_dataDir, _helixAdmin, _helixClusterName, _propertyStore); ZKMetadataProvider.setClusterTenantIsolationEnabled(_propertyStore, _isSingleTenantCluster); _tableRebalancer = new TableRebalancer(_helixZkManager, _helixAdmin, _helixClusterName); }
@Test public void testHelixDataAccessorReadData() { accessor.getProperty(new ArrayList<>(propertyKeys), false); try { accessor.getProperty(new ArrayList<>(propertyKeys), true); Assert.fail(); } catch (HelixMetaDataAccessException ex) { } PropertyKey idealStates = accessor.keyBuilder().idealStates(); accessor.getChildValues(idealStates, false); try { accessor.getChildValues(idealStates, true); Assert.fail(); } catch (HelixMetaDataAccessException ex) { } accessor.getChildValuesMap(idealStates, false); try { accessor.getChildValuesMap(idealStates, true); Assert.fail(); } catch (HelixMetaDataAccessException ex) { } }
/** * Batch get the configurations of all workflows in this cluster. * * @return */ public Map<String, WorkflowConfig> getWorkflows() { Map<String, WorkflowConfig> workflowConfigMap = new HashMap<String, WorkflowConfig>(); Map<String, ResourceConfig> resourceConfigMap = _accessor.getChildValuesMap(_accessor.keyBuilder().resourceConfigs()); for (Map.Entry<String, ResourceConfig> resource : resourceConfigMap.entrySet()) { try { WorkflowConfig config = WorkflowConfig.fromHelixProperty(resource.getValue()); workflowConfigMap.put(resource.getKey(), config); } catch (IllegalArgumentException ex) { // ignore if it is not a workflow config. } } return workflowConfigMap; }
toggleSucceed = true; PropertyKey liveInstanceKey = _keyBuilder.liveInstance(instanceName); LiveInstance liveInstance = _helixDataAccessor.getProperty(liveInstanceKey); if (liveInstance == null) { return toggle ? PinotResourceManagerResponse.FAILURE : PinotResourceManagerResponse.SUCCESS; List<CurrentState> instanceCurrentStates = _helixDataAccessor.getChildValues(instanceCurrentStatesKey); if (instanceCurrentStates == null) { return PinotResourceManagerResponse.SUCCESS;
/** * Locate the controller leader so that we can send LLC segment completion requests to it. * Checks the {@link ControllerLeaderLocator::_cachedControllerLeaderInvalid} flag and fetches the leader from helix if cached value is invalid * * @return The host:port string of the current controller leader. */ public synchronized Pair<String, Integer> getControllerLeader() { if (!_cachedControllerLeaderInvalid) { return _controllerLeaderHostPort; } BaseDataAccessor<ZNRecord> dataAccessor = _helixManager.getHelixDataAccessor().getBaseDataAccessor(); Stat stat = new Stat(); try { ZNRecord znRecord = dataAccessor.get("/" + _clusterName + "/CONTROLLER/LEADER", stat, AccessOption.THROW_EXCEPTION_IFNOTEXIST); String leader = znRecord.getId(); int index = leader.lastIndexOf('_'); String leaderHost = leader.substring(0, index); int leaderPort = Integer.valueOf(leader.substring(index + 1)); _controllerLeaderHostPort = new Pair<>(leaderHost, leaderPort); _cachedControllerLeaderInvalid = false; LOGGER.info("Setting controller leader to be {}:{} as per znode version {}, mtime {}", leaderHost, leaderPort, stat.getVersion(), stat.getMtime()); return _controllerLeaderHostPort; } catch (Exception e) { LOGGER.warn("Could not locate controller leader, exception", e); _cachedControllerLeaderInvalid = true; return null; } }
public List<LiveInstance> getCurrentLiveInstances() { HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor(); PropertyKey liveInstancePropertyKey = new Builder(_helixClusterName).liveInstances(); List<LiveInstance> liveInstances = helixDataAccessor.getChildValues(liveInstancePropertyKey); return liveInstances; }
if (_helixDataAccessor.getProperty(_keyBuilder.liveInstance(instanceName)) != null) { return PinotResourceManagerResponse.failure("Instance " + instanceName + " is still live"); DEFAULT_RETRY_POLICY.attempt(() -> _helixDataAccessor.removeProperty(_keyBuilder.instance(instanceName))); } catch (Exception e) { return PinotResourceManagerResponse.failure("Failed to remove /INSTANCES/" + instanceName); DEFAULT_RETRY_POLICY.attempt(() -> _helixDataAccessor.removeProperty(_keyBuilder.instanceConfig(instanceName))); } catch (Exception e) { return PinotResourceManagerResponse.failure(
private List<String> getTagsForInstance(String instanceName) { InstanceConfig config = _helixDataAccessor.getProperty(_keyBuilder.instanceConfig(instanceName)); return config.getTags(); }
@Override public boolean verify() throws Exception { ClusterDataCache cache = new ClusterDataCache(); cache.refresh(_accessor); List<String> liveInstances = new ArrayList<String>(); liveInstances.addAll(cache.getLiveInstances().keySet()); boolean success = verifyLiveNodes(liveInstances); if (!success) { LOG.info("liveNodes not match, expect: " + _expectSortedLiveNodes + ", actual: " + liveInstances); return false; } BestPossibleStateOutput bestPossbileStates = calculateBestPossibleState(cache); Map<String, ExternalView> externalViews = _accessor.getChildValuesMap(_keyBuilder.externalViews()); // TODO all ideal-states should be included in external-views for (String resourceName : externalViews.keySet()) { ExternalView externalView = externalViews.get(resourceName); Map<Partition, Map<String, String>> bestPossbileState = bestPossbileStates.getResourceMap(resourceName); success = verifyExternalView(externalView, bestPossbileState); if (!success) { LOG.info("external-view for resource: " + resourceName + " not match"); return false; } } return true; }