public ReplicaGroupRebalanceSegmentStrategy(HelixManager helixManager) { _helixManager = helixManager; _helixAdmin = helixManager.getClusterManagmentTool(); _helixClusterName = helixManager.getClusterName(); _propertyStore = helixManager.getHelixPropertyStore(); }
public DefaultRebalanceSegmentStrategy(HelixManager helixManager) { _helixManager = helixManager; _helixAdmin = helixManager.getClusterManagmentTool(); _helixClusterName = helixManager.getClusterName(); _propertyStore = helixManager.getHelixPropertyStore(); }
/** * Get QuotaConfig from property store. * @param rawTableName table name without table type. * @param tableType table type: offline or real-time. * @return QuotaConfig, which could be null. */ private QuotaConfig getQuotaConfigFromPropertyStore(String rawTableName, CommonConstants.Helix.TableType tableType) { ZkHelixPropertyStore<ZNRecord> propertyStore = _helixManager.getHelixPropertyStore(); String tableNameWithType = TableNameBuilder.forType(tableType).tableNameWithType(rawTableName); TableConfig tableConfig = ZKMetadataProvider.getTableConfig(propertyStore, tableNameWithType); if (tableConfig == null) { return null; } return tableConfig.getQuotaConfig(); }
private void addInstanceTagIfNeeded(String clusterName, String instanceName) { InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(clusterName, instanceName); List<String> instanceTags = instanceConfig.getTags(); if (instanceTags == null || instanceTags.size() == 0) { if (ZKMetadataProvider.getClusterTenantIsolationEnabled(_helixManager.getHelixPropertyStore())) { _helixAdmin.addInstanceTag(clusterName, instanceName, TableNameBuilder.OFFLINE.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); _helixAdmin.addInstanceTag(clusterName, instanceName, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); } else { _helixAdmin.addInstanceTag(clusterName, instanceName, CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE); } } }
/** * Create Helix cluster if needed, and then start a Pinot controller instance. */ public synchronized void start() { _helixZkManager = HelixSetupUtils .setup(_helixClusterName, _helixZkURL, _instanceId, _isUpdateStateModel, _enableBatchMessageMode); Preconditions.checkNotNull(_helixZkManager); _helixAdmin = _helixZkManager.getClusterManagmentTool(); _propertyStore = _helixZkManager.getHelixPropertyStore(); _helixDataAccessor = _helixZkManager.getHelixDataAccessor(); // Cache instance zk paths. BaseDataAccessor<ZNRecord> baseDataAccessor = _helixDataAccessor.getBaseDataAccessor(); String instanceConfigs = PropertyPathBuilder.instanceConfig(_helixClusterName); _cacheInstanceConfigsDataAccessor = new ZkCacheBaseDataAccessor<>((ZkBaseDataAccessor<ZNRecord>) baseDataAccessor, instanceConfigs, null, Collections.singletonList(instanceConfigs)); _keyBuilder = _helixDataAccessor.keyBuilder(); _segmentDeletionManager = new SegmentDeletionManager(_dataDir, _helixAdmin, _helixClusterName, _propertyStore); ZKMetadataProvider.setClusterTenantIsolationEnabled(_propertyStore, _isSingleTenantCluster); _tableRebalancer = new TableRebalancer(_helixZkManager, _helixAdmin, _helixClusterName); }
@BeforeTest public void beforeTest() { _zookeeperInstance = ZkStarter.startLocalZkServer(); String helixClusterName = "TestTableQueryQuotaManagerService"; _helixManager = initHelixManager(helixClusterName); _testPropertyStore = _helixManager.getHelixPropertyStore(); _tableQueryQuotaManager = new TableQueryQuotaManager(_helixManager); }
_helixAdmin = _helixZkManager.getClusterManagmentTool(); _partitionAssignmentGenerator = new ReplicaGroupPartitionAssignmentGenerator(_helixZkManager.getHelixPropertyStore());
_helixAdmin = _helixManager.getClusterManagmentTool(); addInstanceTagIfNeeded(helixClusterName, _instanceId); ZkHelixPropertyStore<ZNRecord> propertyStore = _helixManager.getHelixPropertyStore();
_spectatorHelixManager.connect(); _helixAdmin = _spectatorHelixManager.getClusterManagmentTool(); _propertyStore = _spectatorHelixManager.getHelixPropertyStore(); _helixExternalViewBasedRouting = new HelixExternalViewBasedRouting(_propertyStore, _spectatorHelixManager, pinotHelixProperties.subset(ROUTING_TABLE_PARAMS_SUBSET_KEY));
/** * Get the runtime context of a single job. * This method is internal API, please use TaskDriver.getJobContext(); * @param manager a connection to Helix * @param jobResource the name of the job * @return the {@link JobContext}, or null if none is available */ protected static JobContext getJobContext(HelixManager manager, String jobResource) { return getJobContext(manager.getHelixPropertyStore(), jobResource); }
/** * Remove the runtime context of a single job. * This method is internal API. * @param manager A connection to Helix * @param jobResource The name of the job * @return True if remove success, otherwise false */ protected static boolean removeJobContext(HelixManager manager, String jobResource) { return removeJobContext(manager.getHelixPropertyStore(), jobResource); }
/** * Get the runtime context of a single job. * This method is internal API, please use TaskDriver.getJobContext(); * @param manager a connection to Helix * @param jobResource the name of the job * @return the {@link JobContext}, or null if none is available */ protected static JobContext getJobContext(HelixManager manager, String jobResource) { return getJobContext(manager.getHelixPropertyStore(), jobResource); }
/** * Remove the runtime context of a single job. * This method is internal API. * @param manager A connection to Helix * @param jobResource The name of the job * @return True if remove success, otherwise false */ protected static boolean removeJobContext(HelixManager manager, String jobResource) { return removeJobContext(manager.getHelixPropertyStore(), jobResource); }
/** * Set the last task assignment for a given job * @param resourceName the name of the job * @param ra {@link ResourceAssignment} containing the task assignment */ private void setPrevResourceAssignment(String resourceName, ResourceAssignment ra) { _manager.getHelixPropertyStore().set( Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, resourceName, TaskConstants.PREV_RA_NODE), ra.getRecord(), AccessOption.PERSISTENT); }
/** * Set the last task assignment for a given job * @param manager a connection to Helix * @param resourceName the name of the job * @param ra {@link ResourceAssignment} containing the task assignment */ public void setPrevResourceAssignment(HelixManager manager, String resourceName, ResourceAssignment ra) { manager.getHelixPropertyStore().set( Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, resourceName, PREV_RA_NODE), ra.getRecord(), AccessOption.PERSISTENT); }
/** * Set the runtime context of a single workflow * @param manager a connection to Helix * @param workflow the name of the workflow * @param workflowContext the up-to-date {@link WorkflowContext} for the workflow */ protected static void setWorkflowContext(HelixManager manager, String workflow, WorkflowContext workflowContext) { manager.getHelixPropertyStore().set( Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, workflow, CONTEXT_NODE), workflowContext.getRecord(), AccessOption.PERSISTENT); }
/** * Get the last task assignment for a given job * @param resourceName the name of the job * @return {@link ResourceAssignment} instance, or null if no assignment is available */ private ResourceAssignment getPrevResourceAssignment(String resourceName) { ZNRecord r = _manager.getHelixPropertyStore().get( Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, resourceName, PREV_RA_NODE), null, AccessOption.PERSISTENT); return r != null ? new ResourceAssignment(r) : null; }
/** * Set the last task assignment for a given job * * @param resourceName the name of the job * @param ra {@link ResourceAssignment} containing the task assignment */ private void setPrevResourceAssignment(String resourceName, ResourceAssignment ra) { _manager.getHelixPropertyStore() .set(Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, resourceName, PREV_RA_NODE), ra.getRecord(), AccessOption.PERSISTENT); }
/** * Get the last task assignment for a given job * @param resourceName the name of the job * @return {@link ResourceAssignment} instance, or null if no assignment is available */ private ResourceAssignment getPrevResourceAssignment(String resourceName) { ZNRecord r = _manager.getHelixPropertyStore().get( Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, resourceName, TaskConstants.PREV_RA_NODE), null, AccessOption.PERSISTENT); return r != null ? new ResourceAssignment(r) : null; }
/** * Set the runtime context of a single workflow * @param manager a connection to Helix * @param workflow the name of the workflow * @param workflowContext the up-to-date {@link WorkflowContext} for the workflow */ protected static void setWorkflowContext(HelixManager manager, String workflow, WorkflowContext workflowContext) { manager.getHelixPropertyStore().set( Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, workflow, CONTEXT_NODE), workflowContext.getRecord(), AccessOption.PERSISTENT); }