@VisibleForTesting void connectHelixManager() { try { this.helixManager.connect(); } catch (Exception e) { LOGGER.error("HelixManager failed to connect", e); throw Throwables.propagate(e); } }
@VisibleForTesting void connectHelixManager() { try { this.helixManager.connect(); } catch (Exception e) { LOGGER.error("HelixManager failed to connect", e); throw Throwables.propagate(e); } }
@VisibleForTesting void connectHelixManager() { try { if (this.helixManager.isPresent()) { this.helixManager.get().connect(); this.helixManager.get() .getMessagingService() .registerMessageHandlerFactory(Message.MessageType.USER_DEFINE_MSG.toString(), new ControllerUserDefinedMessageHandlerFactory(flowCatalogLocalCommit, scheduler, resourceHandler, serviceName)); } } catch (Exception e) { LOGGER.error("HelixManager failed to connect", e); throw Throwables.propagate(e); } }
try { this.isLeader = false; this.managerClusterHelixManager.connect(); if (this.dedicatedManagerCluster) { if (jobClusterController.isPresent()) { this.jobClusterController.get().connect(); this.taskDriverClusterController.get().connect(); this.jobClusterHelixManager.connect(); if (this.taskDriverHelixManager.isPresent()) { this.taskDriverHelixManager.get().connect();
@VisibleForTesting void connectHelixManager() { try { this.jobHelixManager.connect(); this.jobHelixManager.getMessagingService() .registerMessageHandlerFactory(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new ParticipantShutdownMessageHandlerFactory()); this.jobHelixManager.getMessagingService() .registerMessageHandlerFactory(Message.MessageType.USER_DEFINE_MSG.toString(), getUserDefinedMessageHandlerFactory()); if (this.taskDriverHelixManager.isPresent()) { this.taskDriverHelixManager.get().connect(); } } catch (Exception e) { logger.error("HelixManager failed to connect", e); throw Throwables.propagate(e); } }
HelixManagerFactory.getZKHelixManager(_helixClusterName, _instanceId, InstanceType.SPECTATOR, _zkServers); try { spectatorManager.connect();
public PinotZKChanger(String zkAddress, String clusterName) { this.clusterName = clusterName; helixAdmin = new ZKHelixAdmin(zkAddress); helixManager = HelixManagerFactory .getZKHelixManager(clusterName, "PinotNumReplicaChanger", InstanceType.ADMINISTRATOR, zkAddress); try { helixManager.connect(); } catch (Exception e) { throw new RuntimeException(e); } ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName); propertyStore = new ZkHelixPropertyStore<>(zkAddress, serializer, path); }
_helixManager.getStateMachineEngine().registerStateModelFactory("Task", new TaskStateModelFactory(_helixManager, new TaskFactoryRegistry(_taskExecutorFactoryRegistry, _eventObserverFactoryRegistry).getTaskFactoryRegistry())); _helixManager.connect(); _helixAdmin = _helixManager.getClusterManagmentTool(); addInstanceTagIfNeeded();
@Test void testJobShouldGetCancelled() throws Exception { this.suite =new IntegrationJobCancelSuite(); Config helixConfig = this.suite.getManagerConfig(); String clusterName = helixConfig.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); String instanceName = ConfigUtils.getString(helixConfig, GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, GobblinClusterManager.class.getSimpleName()); String zkConnectString = helixConfig.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixManager helixManager = HelixManagerFactory.getZKHelixManager(clusterName, instanceName, InstanceType.CONTROLLER, zkConnectString); suite.startCluster(); helixManager.connect(); TaskDriver taskDriver = new TaskDriver(helixManager); while (TaskDriver.getWorkflowContext(helixManager, IntegrationJobCancelSuite.JOB_ID) == null) { log.warn("Waiting for the job to start..."); Thread.sleep(1000L); } // Give the job some time to reach writer, where it sleeps Thread.sleep(2000L); log.info("Stopping the job"); taskDriver.stop(IntegrationJobCancelSuite.JOB_ID); suite.shutdownCluster(); suite.waitForAndVerifyOutputFiles(); }
@Test(dependsOnMethods = "testCreateHelixCluster") public void testSendShutdownRequest() throws Exception { this.helixManager.connect(); this.helixManager.getMessagingService().registerMessageHandlerFactory(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new TestShutdownMessageHandlerFactory(this)); this.gobblinYarnAppLauncher.connectHelixManager(); this.gobblinYarnAppLauncher.sendShutdownRequest(); Assert.assertEquals(this.curatorFramework.checkExists() .forPath(String.format("/%s/CONTROLLER/MESSAGES", GobblinYarnAppLauncherTest.class.getSimpleName())) .getVersion(), 0); YarnSecurityManagerTest.GetControllerMessageNumFunc getCtrlMessageNum = new YarnSecurityManagerTest.GetControllerMessageNumFunc(GobblinYarnAppLauncherTest.class.getSimpleName(), this.curatorFramework); AssertWithBackoff assertWithBackoff = AssertWithBackoff.create().logger(LoggerFactory.getLogger("testSendShutdownRequest")).timeoutMs(20000); assertWithBackoff.assertEquals(getCtrlMessageNum, 1, "1 controller message queued"); // Give Helix sometime to handle the message assertWithBackoff.assertEquals(getCtrlMessageNum, 0, "all controller messages processed"); }
this.helixManager.connect();
public static void addFakeDataInstanceToAutoJoinHelixCluster(String helixClusterName, String zkServer, String instanceId, boolean isSingleTenant, int adminPort) throws Exception { final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, instanceId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptySegmentOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptySegmentOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.OFFLINE.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, UNTAGGED_SERVER_INSTANCE); } HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.PARTICIPANT, helixClusterName) .forParticipant(instanceId).build(); Map<String, String> props = new HashMap<>(); props.put(CommonConstants.Helix.Instance.ADMIN_PORT_KEY, String.valueOf(adminPort)); helixZkManager.getClusterManagmentTool().setConfig(scope, props); }
@BeforeClass public void setUp() throws Exception { // Use a random ZK port this.testingZKServer = new TestingServer(-1); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); URL url = GobblinClusterManagerTest.class.getClassLoader().getResource( GobblinClusterManager.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); Config config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .resolve(); String zkConnectionString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); HelixUtils.createGobblinHelixCluster(zkConnectionString, config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY)); this.helixManager = HelixManagerFactory .getZKHelixManager(config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.PARTICIPANT, zkConnectionString); this.helixManager.connect(); this.helixManager.getMessagingService().registerMessageHandlerFactory(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new TestShutdownMessageHandlerFactory(this)); this.gobblinClusterManager = new GobblinClusterManager(TestHelper.TEST_APPLICATION_NAME, TestHelper.TEST_APPLICATION_ID, config, Optional.<Path>absent()); this.gobblinClusterManager.getEventBus().register(this.gobblinClusterManager); this.gobblinClusterManager.connectHelixManager(); }
this.helixManager.connect();
@Test(dependsOnMethods = "testGetReconnectableApplicationId") public void testSendShutdownRequest() throws Exception { // Connect to Helix as Controller and register a shutdown request handler this.helixManager.connect(); this.helixManager.getMessagingService().registerMessageHandlerFactory(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new TestShutdownMessageHandlerFactory(this)); // Make Gobblin AWS Cluster launcher start a shutdown this.gobblinAwsClusterLauncher.connectHelixManager(); this.gobblinAwsClusterLauncher.sendShutdownRequest(); Assert.assertEquals(this.curatorFramework.checkExists() .forPath(String.format("/%s/CONTROLLER/MESSAGES", this.helixClusterName)).getVersion(), 0); GetControllerMessageNumFunc getCtrlMessageNum = new GetControllerMessageNumFunc(this.helixClusterName, this.curatorFramework); // Assert to check if shutdown message was issued AssertWithBackoff assertWithBackoff = AssertWithBackoff.create().logger(LoggerFactory.getLogger("testSendShutdownRequest")).timeoutMs(20000); assertWithBackoff.assertEquals(getCtrlMessageNum, 1, "1 controller message queued"); // Assert to check if shutdown message was processed // Give Helix sometime to handle the message assertWithBackoff.assertEquals(getCtrlMessageNum, 0, "all controller messages processed"); }
public static void addFakeBrokerInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant) throws Exception { for (int i = 0; i < numInstances; ++i) { final String brokerId = "Broker_localhost_" + i; final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptyBrokerOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptyBrokerOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, UNTAGGED_BROKER_INSTANCE); } } }
_spectatorHelixManager.connect(); _helixAdmin = _spectatorHelixManager.getClusterManagmentTool(); _propertyStore = _spectatorHelixManager.getHelixPropertyStore(); stateMachineEngine .registerStateModelFactory(BrokerResourceOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); _helixManager.connect(); _tbiMessageHandler = new TimeboundaryRefreshMessageHandlerFactory(_helixExternalViewBasedRouting, _pinotHelixProperties.getLong(CommonConstants.Broker.CONFIG_OF_BROKER_REFRESH_TIMEBOUNDARY_INFO_SLEEP_INTERVAL,
HelixManagerFactory.getZKHelixManager(helixClusterName, _instanceId, InstanceType.PARTICIPANT, _zkServers); final StateMachineEngine stateMachineEngine = _helixManager.getStateMachineEngine(); _helixManager.connect(); _helixAdmin = _helixManager.getClusterManagmentTool(); addInstanceTagIfNeeded(helixClusterName, _instanceId);
/** * Helper method to start a fake server that only implements Helix part. * * @throws Exception */ private void startFakeServer() throws Exception { _serverInstance = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + NetUtil.getHostAddress() + "_" + CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT; // Create server instance with the fake server state model _serverHelixManager = HelixManagerFactory .getZKHelixManager(_clusterName, _serverInstance, InstanceType.PARTICIPANT, ZkStarter.DEFAULT_ZK_STR); _serverHelixManager.getStateMachineEngine() .registerStateModelFactory(SegmentOnlineOfflineStateModelFactory.getStateModelName(), new FakeServerSegmentStateModelFactory()); _serverHelixManager.connect(); // Add Helix tag to the server _serverHelixManager.getClusterManagmentTool().addInstanceTag(_clusterName, _serverInstance, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); // Initialize controller leader locator ControllerLeaderLocator.create(_serverHelixManager); }
public void start() throws Exception { _helixZkManager = HelixManagerFactory.getZKHelixManager(_helixClusterName, _instanceId, InstanceType.PARTICIPANT, _zkServer); StateMachineEngine stateMachineEngine = _helixZkManager.getStateMachineEngine(); // create a stateModelFactory that returns a statemodel object for each partition. TestOnlineOfflineStateModelFactory stateModelFactory = new TestOnlineOfflineStateModelFactory(_instanceId, 0); stateMachineEngine.registerStateModelFactory("OnlineOffline", stateModelFactory); _helixZkManager.connect(); }