public AutoAddInvertedIndex(@Nonnull String zkAddress, @Nonnull String clusterName, @Nonnull String controllerAddress, @Nonnull String brokerAddress, @Nonnull Strategy strategy, @Nonnull Mode mode) { _clusterName = clusterName; _controllerAddress = controllerAddress; _brokerAddress = brokerAddress; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); _strategy = strategy; _mode = mode; }
public TableRetentionValidator(@Nonnull String zkAddress, @Nonnull String clusterName) { _clusterName = clusterName; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); }
private void init() { LOGGER.info("Trying to connect to " + _zkAddress + " cluster " + _clusterName); _helixAdmin = new ZKHelixAdmin(_zkAddress); ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, _clusterName); _propertyStore = new ZkHelixPropertyStore<>(_zkAddress, serializer, path); }
@Override public boolean execute() throws Exception { if (!_validateTableConfig && !_validateSchema) { throw new RuntimeException("Need to specify at least one of -schema and -tableConfig"); } LOGGER.info("Connecting to Zookeeper: {}, cluster: ", _zkAddress, _clusterName); ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, _clusterName); _helixPropertyStore = new ZkHelixPropertyStore<>(_zkAddress, serializer, path); LOGGER.info("\n\n-------------------- Starting Validation --------------------"); if (_validateTableConfig) { validateTableConfig(); } if (_validateSchema) { validateSchema(); } return true; }
private TableConfig getTableConfig(String tableName) throws IOException { ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, zkPath); ZkHelixPropertyStore<ZNRecord> propertyStore = new ZkHelixPropertyStore<>(zkHost, serializer, path); ZNRecord tcZnRecord = propertyStore.get("/CONFIGS/TABLE/" + tableName, null, 0); TableConfig tableConfig = TableConfig.fromZnRecord(tcZnRecord); LOGGER.debug("Loaded table config"); return tableConfig; }
public PinotZKChanger(String zkAddress, String clusterName) { this.clusterName = clusterName; helixAdmin = new ZKHelixAdmin(zkAddress); helixManager = HelixManagerFactory .getZKHelixManager(clusterName, "PinotNumReplicaChanger", InstanceType.ADMINISTRATOR, zkAddress); try { helixManager.connect(); } catch (Exception e) { throw new RuntimeException(e); } ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName); propertyStore = new ZkHelixPropertyStore<>(zkAddress, serializer, path); }
public void start(ControllerMetrics controllerMetrics) { _controllerMetrics = controllerMetrics; LOGGER.info("Starting realtime segments manager, adding a listener on the property store table configs path."); String zkUrl = _pinotHelixResourceManager.getHelixZkURL(); _zkClient = new ZkClient(zkUrl, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT); _zkClient.setZkSerializer(new ZNRecordSerializer()); _zkClient.waitUntilConnected(CommonConstants.Helix.ZkClient.DEFAULT_CONNECT_TIMEOUT_SEC, TimeUnit.SECONDS); // Subscribe to any data/child changes to property _zkClient.subscribeChildChanges(_tableConfigPath, this); _zkClient.subscribeDataChanges(_tableConfigPath, this); // Subscribe to leadership changes ControllerLeadershipManager.getInstance().subscribe(PinotLLCRealtimeSegmentManager.class.getName(), this); // Setup change listeners for already existing tables, if any. processPropertyStoreChange(_tableConfigPath); }
ZkClient client = new ZkClient("localhost:2191", 10000, 10000, new ZNRecordSerializer());
private static void initPropertyStorePath(String helixClusterName, String zkPath) { String propertyStorePath = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, helixClusterName); ZkHelixPropertyStore<ZNRecord> propertyStore = new ZkHelixPropertyStore<ZNRecord>(zkPath, new ZNRecordSerializer(), propertyStorePath); propertyStore.create("/CONFIGS", new ZNRecord(""), AccessOption.PERSISTENT); propertyStore.create("/CONFIGS/CLUSTER", new ZNRecord(""), AccessOption.PERSISTENT); propertyStore.create("/CONFIGS/TABLE", new ZNRecord(""), AccessOption.PERSISTENT); propertyStore.create("/CONFIGS/INSTANCE", new ZNRecord(""), AccessOption.PERSISTENT); propertyStore.create("/SCHEMAS", new ZNRecord(""), AccessOption.PERSISTENT); propertyStore.create("/SEGMENTS", new ZNRecord(""), AccessOption.PERSISTENT); }
FakeHelixManager(String clusterName, String instanceName, InstanceType instanceType, String zkAddress) { super(clusterName, instanceName, instanceType, zkAddress); super._zkclient = new ZkClient(StringUtil.join("/", StringUtils.chomp(ZkStarter.DEFAULT_ZK_STR, "/")), ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); _zkclient.deleteRecursive("/" + clusterName + "/PROPERTYSTORE"); _zkclient.createPersistent("/" + clusterName + "/PROPERTYSTORE", true); setPropertyStore(clusterName); }
ZkClient zkClient = new ZkClient(zkPath); zkClient.waitUntilConnected(CommonConstants.Helix.ZkClient.DEFAULT_CONNECT_TIMEOUT_SEC, TimeUnit.SECONDS); zkClient.setZkSerializer(new ZNRecordSerializer()); HelixDataAccessor accessor = new ZKHelixDataAccessor(helixClusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
@Test public void testHelixExternalViewBasedRoutingTable() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json"); Assert.assertNotNull(resourceUrl); String fileName = resourceUrl.getFile(); byte[] externalViewBytes = IOUtils.toByteArray(new FileInputStream(fileName)); ExternalView externalView = new ExternalView((ZNRecord) new ZNRecordSerializer().deserialize(externalViewBytes)); String tableName = externalView.getResourceName(); List<InstanceConfig> instanceConfigs = getInstanceConfigs(externalView); int numSegmentsInEV = externalView.getPartitionSet().size(); int numServersInEV = instanceConfigs.size(); HelixExternalViewBasedRouting routing = new HelixExternalViewBasedRouting(null, null, new BaseConfiguration()); routing.markDataResourceOnline(generateTableConfig(tableName), externalView, instanceConfigs); for (int i = 0; i < NUM_ROUNDS; i++) { Map<String, List<String>> routingTable = routing.getRoutingTable(new RoutingTableLookupRequest(tableName)); Assert.assertEquals(routingTable.size(), numServersInEV); int numSegments = 0; for (List<String> segmentsForServer : routingTable.values()) { int numSegmentsForServer = segmentsForServer.size(); Assert.assertTrue( numSegmentsForServer >= MIN_NUM_SEGMENTS_PER_SERVER && numSegmentsForServer <= MAX_NUM_SEGMENTS_PER_SERVER); numSegments += numSegmentsForServer; } Assert.assertEquals(numSegments, numSegmentsInEV); } }
@Test public void testGetInstanceConfigs() throws Exception { Set<String> servers = _helixResourceManager.getAllInstancesForServerTenant(SERVER_TENANT_NAME); for (String server : servers) { InstanceConfig cachedInstanceConfig = _helixResourceManager.getHelixInstanceConfig(server); InstanceConfig realInstanceConfig = _helixAdmin.getInstanceConfig(_helixClusterName, server); Assert.assertEquals(cachedInstanceConfig, realInstanceConfig); } ZkClient zkClient = new ZkClient(_helixResourceManager.getHelixZkURL(), CONNECTION_TIMEOUT_IN_MILLISECOND, CONNECTION_TIMEOUT_IN_MILLISECOND, new ZNRecordSerializer()); modifyExistingInstanceConfig(zkClient); addAndRemoveNewInstanceConfig(zkClient); zkClient.close(); }
private void validateLLCPartitionsIncrease(FakePinotLLCRealtimeSegmentManager segmentManager, IdealState idealState, TableConfig tableConfig, int nPartitions, int nReplicas, List<String> instances, boolean skipNewPartitions) { ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); IdealState idealStateCopy = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord())));
@BeforeTest public void beforeTest() { _zookeeperInstance = ZkStarter.startLocalZkServer(); _zkClient = new ZkClient(StringUtil.join("/", StringUtils.chomp(ZkStarter.DEFAULT_ZK_STR, "/")), ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); String helixClusterName = "TestTimeBoundaryService"; _zkClient.deleteRecursive("/" + helixClusterName + "/PROPERTYSTORE"); _zkClient.createPersistent("/" + helixClusterName + "/PROPERTYSTORE", true); _propertyStore = new ZkHelixPropertyStore<>(new ZkBaseDataAccessor<ZNRecord>(_zkClient), "/" + helixClusterName + "/PROPERTYSTORE", null); }
ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
int nReplicas = 2; IdealStateBuilderUtil idealStateBuilder = new IdealStateBuilderUtil(tableName); ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
/** * @param zkAddr * @param clusterName * @param dimensionName Identify of the capacity attribute. For example memory, CPU. * Need to match resource weight dimension. */ public ZkBasedPartitionWeightProvider(String zkAddr, String clusterName, String dimensionName) { this(new ZkHelixPropertyStore<ZNRecord>(zkAddr, new ZNRecordSerializer(), PropertyPathBuilder.propertyStore(clusterName)), dimensionName); }
public ZkHelixClusterVerifier(String zkAddr, String clusterName) { if (zkAddr == null || clusterName == null) { throw new IllegalArgumentException("requires zkAddr|clusterName"); } _zkClient = DedicatedZkClientFactory.getInstance() .buildZkClient(new HelixZkClient.ZkConnectionConfig(zkAddr)); _zkClient.setZkSerializer(new ZNRecordSerializer()); _clusterName = clusterName; _accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); _keyBuilder = _accessor.keyBuilder(); }
public static String getInstancePropertiesAsString(ZkClient zkClient, String clusterName, PropertyKey propertyKey, MediaType mediaType) throws JsonGenerationException, JsonMappingException, IOException { zkClient.setZkSerializer(new ZNRecordSerializer()); ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient)); List<ZNRecord> records = HelixProperty.convertToList(accessor.getChildValues(propertyKey)); return ObjectToJson(records); }