public AutoAddInvertedIndex(@Nonnull String zkAddress, @Nonnull String clusterName, @Nonnull String controllerAddress, @Nonnull String brokerAddress, @Nonnull Strategy strategy, @Nonnull Mode mode) { _clusterName = clusterName; _controllerAddress = controllerAddress; _brokerAddress = brokerAddress; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); _strategy = strategy; _mode = mode; }
private void validateLLCPartitionsIncrease(FakePinotLLCRealtimeSegmentManager segmentManager, IdealState idealState, TableConfig tableConfig, int nPartitions, int nReplicas, List<String> instances, boolean skipNewPartitions) { ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); IdealState idealStateCopy = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); Map<String, LLCRealtimeSegmentZKMetadata> oldMetadataMap = new HashMap<>(segmentManager._metadataMap.size());
public static IdealState cloneIdealState(IdealState idealState) { return new IdealState( (ZNRecord) ZN_RECORD_SERIALIZER.deserialize(ZN_RECORD_SERIALIZER.serialize(idealState.getRecord()))); }
@Test public void testHelixExternalViewBasedRoutingTable() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json"); Assert.assertNotNull(resourceUrl); String fileName = resourceUrl.getFile(); byte[] externalViewBytes = IOUtils.toByteArray(new FileInputStream(fileName)); ExternalView externalView = new ExternalView((ZNRecord) new ZNRecordSerializer().deserialize(externalViewBytes)); String tableName = externalView.getResourceName(); List<InstanceConfig> instanceConfigs = getInstanceConfigs(externalView); int numSegmentsInEV = externalView.getPartitionSet().size(); int numServersInEV = instanceConfigs.size(); HelixExternalViewBasedRouting routing = new HelixExternalViewBasedRouting(null, null, new BaseConfiguration()); routing.markDataResourceOnline(generateTableConfig(tableName), externalView, instanceConfigs); for (int i = 0; i < NUM_ROUNDS; i++) { Map<String, List<String>> routingTable = routing.getRoutingTable(new RoutingTableLookupRequest(tableName)); Assert.assertEquals(routingTable.size(), numServersInEV); int numSegments = 0; for (List<String> segmentsForServer : routingTable.values()) { int numSegmentsForServer = segmentsForServer.size(); Assert.assertTrue( numSegmentsForServer >= MIN_NUM_SEGMENTS_PER_SERVER && numSegmentsForServer <= MAX_NUM_SEGMENTS_PER_SERVER); numSegments += numSegmentsForServer; } Assert.assertEquals(numSegments, numSegmentsInEV); } }
public static void main(String[] args) { ZNRecordSerializer serializer = new ZNRecordSerializer(); System.out.println(new String(serializer.serialize(generateConfigForMasterSlave()))); }
@Override public ZNRecord deserialize(byte[] bytes) throws PropertyStoreException { return (ZNRecord) _serializer.deserialize(bytes); }
@Override public byte[] serialize(ZNRecord data) throws PropertyStoreException { return _serializer.serialize(data); }
int max = getListFieldBound(record); if (max < Integer.MAX_VALUE) { Map<String, List<String>> listMap = record.getListFields();
@Override public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { logger.info("Add IdealState for resource {} to cluster {} by file name {}.", resourceName, clusterName, idealStateFile); ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
public static void main(String[] args) { ZNRecordSerializer serializer = new ZNRecordSerializer(); System.out.println(new String(serializer.serialize(generateConfigForMasterSlave()))); }
@Override public ZNRecord deserialize(byte[] bytes) throws PropertyStoreException { return (ZNRecord) _serializer.deserialize(bytes); }
@Override public byte[] serialize(ZNRecord data) throws PropertyStoreException { return _serializer.serialize(data); }
int max = getListFieldBound(record); if (max < Integer.MAX_VALUE) { Map<String, List<String>> listMap = record.getListFields();
public TableRetentionValidator(@Nonnull String zkAddress, @Nonnull String clusterName) { _clusterName = clusterName; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); }
ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); _realtimeSegmentRelocator.setTagToInstance(serverTenantCompleted, completedInstanceList); IdealState prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); idealState.setInstanceStateMap("segment0", instanceStateMap0); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertNotSame(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertNotSame(idealState, prevIdealState);
@Override public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { logger.info("Add IdealState for resource {} to cluster {} by file name {}.", resourceName, clusterName, idealStateFile); ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
/** * get configs * @param type config-scope-type, e.g. CLUSTER, RESOURCE, etc. * @param scopeArgsCsv csv-formatted scope-args, e.g myCluster,testDB * @param keysCsv csv-formatted keys. e.g. k1,k2 * @return json-formated key-value pairs, e.g. {k1=v1,k2=v2} */ public String getConfig(ConfigScopeProperty type, String scopeArgsCsv, String keysCsv) { // ConfigScope scope = new ConfigScopeBuilder().build(scopesStr); String[] scopeArgs = scopeArgsCsv.split("[\\s,]"); HelixConfigScope scope = new HelixConfigScopeBuilder(type, scopeArgs).build(); String[] keys = keysCsv.split("[\\s,]"); // parse keys // String[] keys = keysStr.split("[\\s,]"); // Set<String> keysSet = new HashSet<String>(Arrays.asList(keys)); Map<String, String> keyValueMap = _admin.getConfig(scope, Arrays.asList(keys)); ZNRecord record = new ZNRecord(type.toString()); // record.setMapField(scopesStr, propertiesMap); record.getSimpleFields().putAll(keyValueMap); ZNRecordSerializer serializer = new ZNRecordSerializer(); return new String(serializer.serialize(record)); }
String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes()); String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes()); String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes()); formatter(liBw, timestamp, record.getId(), record.getSimpleField("SESSION_ID"), "ADD"); String zkSessionId = getAttributeValue(inputLine, "session:"); String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes()); String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes()); String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes()); String timestamp = getAttributeValue(inputLine, "time:"); ZNRecord record = (ZNRecord) _deserializer.deserialize(inputLine.substring(pos + 5).getBytes());
public void readZNode(String path) { ZNRecord record = _zkclient.readData(path, true); if (record == null) { System.out.println("null"); } else { System.out.println(new String(_serializer.serialize(record))); } }
private void init() { LOGGER.info("Trying to connect to " + _zkAddress + " cluster " + _clusterName); _helixAdmin = new ZKHelixAdmin(_zkAddress); ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, _clusterName); _propertyStore = new ZkHelixPropertyStore<>(_zkAddress, serializer, path); }