public AutoAddInvertedIndex(@Nonnull String zkAddress, @Nonnull String clusterName, @Nonnull String controllerAddress, @Nonnull String brokerAddress, @Nonnull Strategy strategy, @Nonnull Mode mode) { _clusterName = clusterName; _controllerAddress = controllerAddress; _brokerAddress = brokerAddress; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); _strategy = strategy; _mode = mode; }
private void init() { LOGGER.info("Trying to connect to " + _zkAddress + " cluster " + _clusterName); _helixAdmin = new ZKHelixAdmin(_zkAddress); ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, _clusterName); _propertyStore = new ZkHelixPropertyStore<>(_zkAddress, serializer, path); }
public TableRetentionValidator(@Nonnull String zkAddress, @Nonnull String clusterName) { _clusterName = clusterName; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); }
@Override public boolean execute() throws Exception { ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress); List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
public PinotZKChanger(String zkAddress, String clusterName) { this.clusterName = clusterName; helixAdmin = new ZKHelixAdmin(zkAddress); helixManager = HelixManagerFactory .getZKHelixManager(clusterName, "PinotNumReplicaChanger", InstanceType.ADMINISTRATOR, zkAddress); try { helixManager.connect(); } catch (Exception e) { throw new RuntimeException(e); } ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName); propertyStore = new ZkHelixPropertyStore<>(zkAddress, serializer, path); }
public static void waitForExternalViewUpdate(String zkAddress, final String clusterName, long timeoutInMilliseconds) { final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress); List<String> allResourcesInCluster = helixAdmin.getResourcesInCluster(clusterName); Set<String> tableAndBrokerResources = new HashSet<>(); for (String resourceName : allResourcesInCluster) { // Only check table resources and broker resource if (TableNameBuilder.isTableResource(resourceName) || resourceName .equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) { tableAndBrokerResources.add(resourceName); } } StrictMatchExternalViewVerifier verifier = new StrictMatchExternalViewVerifier.Builder(clusterName).setZkAddr(zkAddress) .setResources(tableAndBrokerResources).build(); boolean success = verifier.verify(timeoutInMilliseconds); if (success) { LOGGER.info("Cluster is ready to serve queries"); } }
public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath, boolean isUpdateStateModel, boolean enableBatchMessageMode) { final HelixAdmin admin = new ZKHelixAdmin(zkPath); final String segmentStateModelName = PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL;
clusterInfo.clusterName = _clusterName; ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(_zkAddress); if (!zkHelixAdmin.getClusters().contains(_clusterName)) { LOGGER.error("Cluster {} not found in {}.", _clusterName, _zkAddress);
public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath) { final HelixAdmin admin = new ZKHelixAdmin(zkPath); if (admin.getClusters().contains(helixClusterName)) { LOGGER.info( "cluster already exist, skipping it.. ********************************************* "); return; } LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName + " was not found ********************************************* "); admin.addCluster(helixClusterName, false); LOGGER.info("Enable mirror maker machines auto join."); final HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER) .forCluster(helixClusterName).build(); final Map<String, String> props = new HashMap<String, String>(); props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true)); props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(100)); admin.setConfig(scope, props); LOGGER.info("Adding state model definition named : OnlineOffline generated using : " + OnlineOfflineStateModel.class.toString() + " ********************************************** "); // add state model definition admin.addStateModelDef(helixClusterName, "OnlineOffline", OnlineOfflineStateModel.build()); LOGGER.info("New Cluster setup completed... ********************************************** "); }
public ClusterSetup(HelixZkClient zkClient) { _zkServerAddress = zkClient.getServers(); _zkClient = zkClient; _admin = new ZKHelixAdmin(_zkClient); }
public HelixAdmin getHelixAdmin() { if (_zkHelixAdmin == null) { _zkHelixAdmin = new ZKHelixAdmin(getHelixZkClient()); } return _zkHelixAdmin; }
public ClusterSetup(String zkServerAddress) { _zkServerAddress = zkServerAddress; _zkClient = ZKClientPool.getZkClient(_zkServerAddress); _admin = new ZKHelixAdmin(_zkClient); }
@Override public synchronized HelixAdmin getClusterManagmentTool() { checkConnected(_waitForConnectedTimeout); if (_zkclient != null && !_zkclient.isClosed()) { return new ZKHelixAdmin(_zkclient); } LOG.error("Couldn't get ZKClusterManagementTool because zkclient is null"); return null; }
@Override public synchronized HelixAdmin getClusterManagmentTool() { checkConnected(_waitForConnectedTimeout); if (_zkclient != null && !_zkclient.isClosed()) { return new ZKHelixAdmin(_zkclient); } LOG.error("Couldn't get ZKClusterManagementTool because zkclient is null"); return null; }
public TaskDriver(ZkClient client, ZkBaseDataAccessor<ZNRecord> baseAccessor, String clusterName) { this(new ZKHelixAdmin(client), new ZKHelixDataAccessor(clusterName, baseAccessor), new ZkHelixPropertyStore<ZNRecord>(baseAccessor, PropertyPathBuilder.propertyStore(clusterName), null), clusterName); }
public TaskDriver(HelixZkClient client, ZkBaseDataAccessor<ZNRecord> baseAccessor, String clusterName) { this(new ZKHelixAdmin(client), new ZKHelixDataAccessor(clusterName, baseAccessor), new ZkHelixPropertyStore<>(baseAccessor, PropertyPathBuilder.propertyStore(clusterName), null), clusterName); }
public GobblinHelixTaskDriver(ZkClient client, ZkBaseDataAccessor<ZNRecord> baseAccessor, String clusterName) { this(new ZKHelixAdmin(client), new ZKHelixDataAccessor(clusterName, baseAccessor), new ConfigAccessor(client), new ZkHelixPropertyStore<ZNRecord>(baseAccessor, PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName), null), clusterName); }
@BeforeClass public void beforeClass() throws Exception { _numDbs = 1; _numNodes = 2; _numPartitions = 3; _numReplicas = 2; _partitionVary = false; _admin = new ZKHelixAdmin(_gZkClient); _configAccessor = new ConfigAccessor(_gZkClient); super.beforeClass(); }
public ClusterSetup(String zkServerAddress) { _zkServerAddress = zkServerAddress; _zkClient = SharedZkClientFactory.getInstance() .buildZkClient(new HelixZkClient.ZkConnectionConfig(_zkServerAddress)); _zkClient.setZkSerializer(new ZNRecordSerializer()); _admin = new ZKHelixAdmin(_zkClient); }
protected void setupInstances(String clusterName, int[] instances) { HelixAdmin admin = new ZKHelixAdmin(_gZkClient); for (int i = 0; i < instances.length; i++) { String instance = "localhost_" + instances[i]; InstanceConfig instanceConfig = new InstanceConfig(instance); instanceConfig.setHostName("localhost"); instanceConfig.setPort("" + instances[i]); instanceConfig.setInstanceEnabled(true); admin.addInstance(clusterName, instanceConfig); } }