@Override public boolean execute() throws Exception { LOGGER.info("Connecting to Zookeeper at address: {}", _zkAddress); ZkClient zkClient = new ZkClient(_zkAddress, 5000); String helixClusterName = "/" + _clusterName; LOGGER.info("Executing command: " + toString()); if (!zkClient.exists(helixClusterName)) { LOGGER.error("Cluster {} does not exist.", _clusterName); return false; } zkClient.deleteRecursive(helixClusterName); return true; } }
public void start(ControllerMetrics controllerMetrics) { _controllerMetrics = controllerMetrics; LOGGER.info("Starting realtime segments manager, adding a listener on the property store table configs path."); String zkUrl = _pinotHelixResourceManager.getHelixZkURL(); _zkClient = new ZkClient(zkUrl, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT); _zkClient.setZkSerializer(new ZNRecordSerializer()); _zkClient.waitUntilConnected(CommonConstants.Helix.ZkClient.DEFAULT_CONNECT_TIMEOUT_SEC, TimeUnit.SECONDS); // Subscribe to any data/child changes to property _zkClient.subscribeChildChanges(_tableConfigPath, this); _zkClient.subscribeDataChanges(_tableConfigPath, this); // Subscribe to leadership changes ControllerLeadershipManager.getInstance().subscribe(PinotLLCRealtimeSegmentManager.class.getName(), this); // Setup change listeners for already existing tables, if any. processPropertyStoreChange(_tableConfigPath); }
StateModelDefinition newStateModelDef = PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition(); ZkClient zkClient = new ZkClient(zkPath); zkClient.waitUntilConnected(CommonConstants.Helix.ZkClient.DEFAULT_CONNECT_TIMEOUT_SEC, TimeUnit.SECONDS); zkClient.setZkSerializer(new ZNRecordSerializer());
List<String> tables = zkHelixAdmin.getResourcesInCluster(_clusterName); ZkClient zkClient = new ZkClient(_zkAddress); zkClient.setZkSerializer(new ZNRecordStreamingSerializer()); LOGGER.info("Connecting to Zookeeper at: {}", _zkAddress);
@BeforeClass public void setUp() throws Exception { _zookeeperInstance = ZkStarter.startLocalZkServer(); _zkClient = new ZkClient(ZkStarter.DEFAULT_ZK_STR); final String instanceId = "localhost_helixController"; _pinotHelixResourceManager = new PinotHelixResourceManager(ZkStarter.DEFAULT_ZK_STR, HELIX_CLUSTER_NAME, instanceId, null, 10000L, true, /*isUpdateStateModel=*/ false, true); _pinotHelixResourceManager.start(); _helixAdmin = _pinotHelixResourceManager.getHelixAdmin(); ControllerRequestBuilderUtil .addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 1, true); ControllerRequestBuilderUtil .addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, 1, true); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 1); Assert .assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_OFFLINE").size(), 1); Assert .assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_REALTIME").size(), 1); // Adding table TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TABLE_NAME).build(); _pinotHelixResourceManager.addTable(tableConfig); }
FakeHelixManager(String clusterName, String instanceName, InstanceType instanceType, String zkAddress) { super(clusterName, instanceName, instanceType, zkAddress); super._zkclient = new ZkClient(StringUtil.join("/", StringUtils.chomp(ZkStarter.DEFAULT_ZK_STR, "/")), ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); _zkclient.deleteRecursive("/" + clusterName + "/PROPERTYSTORE"); _zkclient.createPersistent("/" + clusterName + "/PROPERTYSTORE", true); setPropertyStore(clusterName); }
protected void startController(ControllerConf config) { Assert.assertNotNull(config); Assert.assertNull(_controllerStarter); _controllerPort = Integer.valueOf(config.getControllerPort()); _controllerBaseApiUrl = "http://localhost:" + _controllerPort; _controllerRequestURLBuilder = ControllerRequestURLBuilder.baseUrl(_controllerBaseApiUrl); _controllerDataDir = config.getDataDir(); String helixClusterName = getHelixClusterName(); config.setHelixClusterName(helixClusterName); String zkStr = config.getZkStr(); _zkClient = new ZkClient(zkStr); if (_zkClient.exists("/" + helixClusterName)) { _zkClient.deleteRecursive("/" + helixClusterName); } startControllerStarter(config); _helixManager = _helixResourceManager.getHelixZkManager(); _helixAdmin = _helixResourceManager.getHelixAdmin(); _propertyStore = _helixResourceManager.getPropertyStore(); }
@BeforeClass public void setUp() throws Exception { _zookeeperInstance = ZkStarter.startLocalZkServer(); _zkClient = new ZkClient(ZK_STR); Thread.sleep(1000); _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_STR, HELIX_CLUSTER_NAME, CONTROLLER_INSTANCE_NAME, null, 1000L, true, /*isUpdateStateModel=*/ false, true); _pinotHelixResourceManager.start(); ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true); ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true); _offlineTableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TEST_TABLE_NAME).setNumReplicas(2) .build(); final String instanceId = "localhost_helixController"; _helixManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, ZK_STR, instanceId, /*isUpdateStateModel=*/false, true); _pinotHelixResourceManager.addTable(_offlineTableConfig); }
@BeforeTest public void beforeTest() { _zookeeperInstance = ZkStarter.startLocalZkServer(); _zkClient = new ZkClient(StringUtil.join("/", StringUtils.chomp(ZkStarter.DEFAULT_ZK_STR, "/")), ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); String helixClusterName = "TestTimeBoundaryService"; _zkClient.deleteRecursive("/" + helixClusterName + "/PROPERTYSTORE"); _zkClient.createPersistent("/" + helixClusterName + "/PROPERTYSTORE", true); _propertyStore = new ZkHelixPropertyStore<>(new ZkBaseDataAccessor<ZNRecord>(_zkClient), "/" + helixClusterName + "/PROPERTYSTORE", null); }
public ZKDumper(String zkAddress) { client = new ZkClient(zkAddress, ZkClient.DEFAULT_CONNECTION_TIMEOUT); ZkSerializer zkSerializer = new ByteArraySerializer(); client.setZkSerializer(zkSerializer); filter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { return !name.startsWith("."); } }; }
private static Map<TestCommand, Boolean> executeTestHelper(List<TestCommand> commandList, String zkAddr, CountDownLatch countDown) { final Map<TestCommand, Boolean> testResults = new ConcurrentHashMap<TestCommand, Boolean>(); ZkClient zkClient = null; zkClient = new ZkClient(zkAddr, ZkClient.DEFAULT_CONNECTION_TIMEOUT); zkClient.setZkSerializer(new ZNRecordSerializer()); // sort on trigger's start time, stable sort Collections.sort(commandList, new Comparator<TestCommand>() { @Override public int compare(TestCommand o1, TestCommand o2) { return (int) (o1._trigger._startTime - o2._trigger._startTime); } }); for (TestCommand command : commandList) { testResults.put(command, new Boolean(false)); TestTrigger trigger = command._trigger; command._startTimestamp = System.currentTimeMillis() + trigger._startTime; new Thread(new ExecuteCommand(command._startTimestamp, command, countDown, zkClient, testResults)).start(); } return testResults; }
public ZKHelixAdmin(String zkAddress) { int timeOutInSec = Integer.parseInt(System.getProperty(CONNECTION_TIMEOUT, "30")); _zkClient = new ZkClient(zkAddress, timeOutInSec * 1000); _zkClient.setZkSerializer(new ZNRecordSerializer()); _zkClient.waitUntilConnected(timeOutInSec, TimeUnit.SECONDS); _configAccessor = new ConfigAccessor(_zkClient); }
public static ZkClient getZkClient(String zkServer) { // happy path that we cache the zkclient and it's still connected if (_zkClientMap.containsKey(zkServer)) { ZkClient zkClient = _zkClientMap.get(zkServer); if (zkClient.getConnection().getZookeeperState() == States.CONNECTED) { return zkClient; } } synchronized (_zkClientMap) { // if we cache a stale zkclient, purge it if (_zkClientMap.containsKey(zkServer)) { ZkClient zkClient = _zkClientMap.get(zkServer); if (zkClient.getConnection().getZookeeperState() != States.CONNECTED) { _zkClientMap.remove(zkServer); } } // get a new zkclient if (!_zkClientMap.containsKey(zkServer)) { ZkClient zkClient = new ZkClient(zkServer, DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); _zkClientMap.put(zkServer, zkClient); } return _zkClientMap.get(zkServer); } }
@BeforeClass public void beforeClass() { _zkClient = new ZkClient(ZK_ADDR); _zkClient.setZkSerializer(new ZNRecordSerializer()); }
public ZkClient build() { if (_connection == null) { if (_zkServer == null) { throw new HelixException( "Failed to build ZkClient since no connection or ZK server address is specified."); } else { _connection = new ZkConnection(_zkServer, _sessionTimeout); } } if (_zkSerializer == null) { _zkSerializer = new BasicZkSerializer(new SerializableSerializer()); } return new ZkClient(_connection, _connectionTimeout, _operationRetryTimeout, _zkSerializer, _monitorType, _monitorKey, _monitorInstanceName, _monitorRootPathOnly); } }
public ZkClient build() { if (_connection == null) { if (_zkServer == null) { throw new HelixException( "Failed to build ZkClient since no connection or ZK server address is specified."); } else { _connection = new ZkConnection(_zkServer, _sessionTimeout); } } if (_zkSerializer == null) { _zkSerializer = new BasicZkSerializer(new SerializableSerializer()); } return new ZkClient(_connection, _connectionTimeout, _operationRetryTimeout, _zkSerializer, _monitorType, _monitorKey, _monitorInstanceName, _monitorRootPathOnly); } }
@Test public void testCloseZkClient() { String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); ZkClient client = new ZkClient(ZK_ADDR, HelixZkClient.DEFAULT_SESSION_TIMEOUT, HelixZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); String path = String.format("/%s", clusterName); client.createEphemeral(path); client.close(); Assert.assertFalse(_gZkClient.exists(path), "Ephemeral node: " + path + " should be removed after ZkClient#close()"); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); }
@Test public void testZkSessionExpiry() throws Exception { String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); ZkClient client = new ZkClient(ZK_ADDR, HelixZkClient.DEFAULT_SESSION_TIMEOUT, HelixZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); String path = String.format("/%s", clusterName); client.createEphemeral(path); String oldSessionId = ZkTestHelper.getSessionId(client); ZkTestHelper.expireSession(client); String newSessionId = ZkTestHelper.getSessionId(client); Assert.assertNotSame(newSessionId, oldSessionId); Assert.assertFalse(client.exists(path), "Ephemeral znode should be gone after session expiry"); client.close(); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); }
/** * Build a Dedicated ZkClient based on connection config and client config * * @param connectionConfig * @param clientConfig * @return */ @Override public HelixZkClient buildZkClient(HelixZkClient.ZkConnectionConfig connectionConfig, HelixZkClient.ZkClientConfig clientConfig) { return new ZkClient(createZkConnection(connectionConfig), (int) clientConfig.getConnectInitTimeout(), clientConfig.getOperationRetryTimeout(), clientConfig.getZkSerializer(), clientConfig.getMonitorType(), clientConfig.getMonitorKey(), clientConfig.getMonitorInstanceName(), clientConfig.isMonitorRootPathOnly()); } }
public void post(String zkServer, Message message, String clusterName, String instanceName) { ZkClient client = new ZkClient(zkServer); client.setZkSerializer(new ZNRecordSerializer()); String path = PropertyPathBuilder.instanceMessage(clusterName, instanceName, message.getId()); client.delete(path); ZNRecord record = client.readData(PropertyPathBuilder.liveInstance(clusterName, instanceName)); message.setTgtSessionId(record.getSimpleField(LiveInstanceProperty.SESSION_ID.toString())); message.setTgtName(record.getId()); // System.out.println(message); client.createPersistent(path, message.getRecord()); }