private void createHelixEntriesForHighLevelConsumer(TableConfig config, String realtimeTableName, IdealState idealState) { if (idealState == null) { idealState = PinotTableIdealStateBuilder .buildInitialHighLevelRealtimeIdealStateFor(realtimeTableName, config, _helixZkManager, _propertyStore, _enableBatchMessageMode); LOGGER.info("Adding helix resource with empty HLC IdealState for {}", realtimeTableName); _helixAdmin.addResource(_helixClusterName, realtimeTableName, idealState); } else { // TODO jfim: We get in this block if we're trying to add a HLC or it already exists. If it doesn't already exist, we need to set instance configs properly (which is done in buildInitialHighLevelRealtimeIdealState, surprisingly enough). For now, do nothing. LOGGER.info("Not reconfiguring HLC for table {}", realtimeTableName); } LOGGER.info("Successfully created empty ideal state for high level consumer for {} ", realtimeTableName); // Finally, create the propertystore entry that will trigger watchers to create segments String tablePropertyStorePath = ZKMetadataProvider.constructPropertyStorePathForResource(realtimeTableName); if (!_propertyStore.exists(tablePropertyStorePath, AccessOption.PERSISTENT)) { _propertyStore.create(tablePropertyStorePath, new ZNRecord(realtimeTableName), AccessOption.PERSISTENT); } }
/** * Regression test for large ideal state updates failing silently */ @Test public void testWriteLargeIdealState() { final int numSegments = 20000; IdealState idealState = new IdealState(RESOURCE_NAME); idealState.setStateModelDefRef("OnlineOffline"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); idealState.setReplicas("0"); String helixClusterName = getHelixClusterName(); _helixAdmin.addResource(helixClusterName, RESOURCE_NAME, idealState); HelixHelper.updateIdealState(_helixManager, RESOURCE_NAME, new Function<IdealState, IdealState>() { @Override public IdealState apply(@Nullable IdealState idealState) { Assert.assertNotNull(idealState); for (int i = 0; i < numSegments; i++) { idealState.setPartitionState("segment_" + i, INSTANCE_NAME, "ONLINE"); } return idealState; } }, RetryPolicies.noDelayRetryPolicy(1)); IdealState resourceIdealState = _helixAdmin.getResourceIdealState(helixClusterName, RESOURCE_NAME); for (int i = 0; i < numSegments; i++) { Assert.assertEquals(resourceIdealState.getInstanceStateMap("segment_" + i).get(INSTANCE_NAME), "ONLINE"); } }
_enableBatchMessageMode); LOGGER.info("adding table via the admin"); _helixAdmin.addResource(_helixClusterName, tableNameWithType, offlineIdealState); LOGGER.info("successfully added the table : " + tableNameWithType + " to the cluster");
public synchronized void addTopicToMirrorMaker(String topicName, int numTopicPartitions) { setEmptyResourceConfig(topicName); updateCurrentServingInstance(); synchronized (_currentServingInstance) { _helixAdmin.addResource(_helixClusterName, topicName, IdealStateBuilder.buildCustomIdealStateFor(topicName, numTopicPartitions, _currentServingInstance)); } }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode, String rebalanceStrategy) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode, rebalanceStrategy); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode, String rebalanceStrategy) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode, rebalanceStrategy); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode, int bucketSize) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode, bucketSize); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode, int bucketSize, int maxPartitionsPerInstance) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode, bucketSize, maxPartitionsPerInstance); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode, int bucketSize) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode, bucketSize); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode, int bucketSize, int maxPartitionsPerInstance) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode, bucketSize, maxPartitionsPerInstance); }
public void addResourceToCluster(String clusterName, String resourceName, int numPartitions, String stateModelRef, String rebalancerMode) { _admin.addResource(clusterName, resourceName, numPartitions, stateModelRef, rebalancerMode); }
public void addResourceToCluster(String clusterName, String resourceName, IdealState idealState) { _admin.addResource(clusterName, resourceName, idealState); }
public void addResourceToCluster(String clusterName, String resourceName, IdealState idealState) { _admin.addResource(clusterName, resourceName, idealState); }
public synchronized void addTopicToMirrorMaker(String topicName, int numTopicPartitions) { setEmptyResourceConfig(topicName); updateCurrentServingInstance(); synchronized (_currentServingInstance) { _helixAdmin.addResource(_helixClusterName, topicName, IdealStateBuilder.buildCustomIdealStateFor(topicName, numTopicPartitions, _currentServingInstance)); } }
/** Posts new workflow resource to cluster */ private void addWorkflowResource(String workflow) { // Add workflow resource _admin.addResource(_clusterName, workflow, 1, TaskConstants.STATE_MODEL_NAME); IdealState is = buildWorkflowIdealState(workflow); TaskUtil .createUserContent(_propertyStore, workflow, new ZNRecord(TaskUtil.USER_CONTENT_NODE)); _admin.setResourceIdealState(_clusterName, workflow, is); }
/** Posts new workflow resource to cluster */ private void addWorkflowResource(String workflow) { // Add workflow resource _admin.addResource(_clusterName, workflow, 1, TaskConstants.STATE_MODEL_NAME); IdealState is = buildWorkflowIdealState(workflow); TaskUtil .createUserContent(_propertyStore, workflow, new ZNRecord(TaskUtil.USER_CONTENT_NODE)); _admin.setResourceIdealState(_clusterName, workflow, is); }
@Test (dependsOnMethods = "testMaintenanceModeAddNewInstance") public void testMaintenanceModeAddNewResource() throws InterruptedException { _gSetupTool.getClusterManagementTool() .addResource(CLUSTER_NAME, newResourceAddedDuringMaintenanceMode, 7, "MasterSlave", IdealState.RebalanceMode.FULL_AUTO.name()); _gSetupTool.getClusterManagementTool() .rebalance(CLUSTER_NAME, newResourceAddedDuringMaintenanceMode, 3); Assert.assertTrue(_clusterVerifier.verifyByPolling()); ExternalView externalView = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, newResourceAddedDuringMaintenanceMode); Assert.assertNull(externalView); }
public static void setup() { admin = new ZKHelixAdmin(ZK_ADDRESS); // create cluster echo("Creating cluster: " + CLUSTER_NAME); admin.addCluster(CLUSTER_NAME, true); // Add nodes to the cluster echo("Adding " + NUM_NODES + " participants to the cluster"); for (int i = 0; i < NUM_NODES; i++) { admin.addInstance(CLUSTER_NAME, INSTANCE_CONFIG_LIST.get(i)); echo("\t Added participant: " + INSTANCE_CONFIG_LIST.get(i).getInstanceName()); } // Add a state model StateModelDefinition myStateModel = defineStateModel(); echo("Configuring StateModel: " + "MyStateModel with 1 Master and 1 Slave"); admin.addStateModelDef(CLUSTER_NAME, STATE_MODEL_NAME, myStateModel); // Add a resource with 6 partitions and 2 replicas echo("Adding a resource MyResource: " + "with 6 partitions and 2 replicas"); admin.addResource(CLUSTER_NAME, RESOURCE_NAME, NUM_PARTITIONS, STATE_MODEL_NAME, "AUTO"); // this will set up the ideal state, it calculates the preference list for // each partition similar to consistent hashing admin.rebalance(CLUSTER_NAME, RESOURCE_NAME, NUM_REPLICAS); }
private void setupResource(String dbName) throws InterruptedException { IdealState idealState = new FullAutoModeISBuilder(dbName).setStateModel("OnlineOffline") .setStateModelFactoryName("TestFactory").setNumPartitions(10).setNumReplica(1).build(); _gSetupTool.getClusterManagementTool().addResource(CLUSTER_NAME, dbName, idealState); }