public static IdealState cloneIdealState(IdealState idealState) { return new IdealState( (ZNRecord) ZN_RECORD_SERIALIZER.deserialize(ZN_RECORD_SERIALIZER.serialize(idealState.getRecord()))); }
@Test public void testHelixExternalViewBasedRoutingTable() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json"); Assert.assertNotNull(resourceUrl); String fileName = resourceUrl.getFile(); byte[] externalViewBytes = IOUtils.toByteArray(new FileInputStream(fileName)); ExternalView externalView = new ExternalView((ZNRecord) new ZNRecordSerializer().deserialize(externalViewBytes)); String tableName = externalView.getResourceName(); List<InstanceConfig> instanceConfigs = getInstanceConfigs(externalView); int numSegmentsInEV = externalView.getPartitionSet().size(); int numServersInEV = instanceConfigs.size(); HelixExternalViewBasedRouting routing = new HelixExternalViewBasedRouting(null, null, new BaseConfiguration()); routing.markDataResourceOnline(generateTableConfig(tableName), externalView, instanceConfigs); for (int i = 0; i < NUM_ROUNDS; i++) { Map<String, List<String>> routingTable = routing.getRoutingTable(new RoutingTableLookupRequest(tableName)); Assert.assertEquals(routingTable.size(), numServersInEV); int numSegments = 0; for (List<String> segmentsForServer : routingTable.values()) { int numSegmentsForServer = segmentsForServer.size(); Assert.assertTrue( numSegmentsForServer >= MIN_NUM_SEGMENTS_PER_SERVER && numSegmentsForServer <= MAX_NUM_SEGMENTS_PER_SERVER); numSegments += numSegmentsForServer; } Assert.assertEquals(numSegments, numSegmentsInEV); } }
ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); IdealState idealStateCopy = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); Map<String, LLCRealtimeSegmentZKMetadata> oldMetadataMap = new HashMap<>(segmentManager._metadataMap.size());
_realtimeSegmentRelocator.setTagToInstance(serverTenantCompleted, completedInstanceList); IdealState prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); idealState.setInstanceStateMap("segment0", instanceStateMap0); prevIdealState = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertEquals(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertNotSame(idealState, prevIdealState); new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); _realtimeSegmentRelocator.relocateSegments(realtimeTagConfig, idealState); Assert.assertNotSame(idealState, prevIdealState); _realtimeSegmentRelocator.setTagToInstance(serverTenantCompleted, completedInstanceList);
nPartitions = expectedPartitionAssignment.getNumPartitions(); IdealState idealStateCopy = new IdealState( (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); nPartitions = expectedPartitionAssignment.getNumPartitions(); IdealState idealStateCopy = new IdealState( (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields(); (ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord()))); Map<String, Map<String, String>> oldMapFields = idealStateCopy.getRecord().getMapFields();
@Override public ZNRecord deserialize(byte[] bytes) throws PropertyStoreException { return (ZNRecord) _serializer.deserialize(bytes); }
@Override public ZNRecord deserialize(byte[] bytes) throws PropertyStoreException { return (ZNRecord) _serializer.deserialize(bytes); }
@Override public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { logger.info("Add IdealState for resource {} to cluster {} by file name {}.", resourceName, clusterName, idealStateFile); ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
@Override public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { logger.info("Add IdealState for resource {} to cluster {} by file name {}.", resourceName, clusterName, idealStateFile); ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
@Override public void addStateModelDef(String clusterName, String stateModelDefName, String stateModelDefFile) throws IOException { ZNRecord record = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(stateModelDefFile))); if (record == null || record.getId() == null || !record.getId().equals(stateModelDefName)) { throw new IllegalArgumentException( "state model definition must have same id as state model def name"); } addStateModelDef(clusterName, stateModelDefName, new StateModelDefinition(record), false); }
public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } _admin.setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException { ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile))); if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) { throw new IllegalArgumentException("ideal state must have same id as resource name"); } _admin.setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord)); }
@Override public void addStateModelDef(String clusterName, String stateModelDefName, String stateModelDefFile) throws IOException { ZNRecord record = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(stateModelDefFile))); if (record == null || record.getId() == null || !record.getId().equals(stateModelDefName)) { throw new IllegalArgumentException( "state model definition must have same id as state model def name"); } addStateModelDef(clusterName, stateModelDefName, new StateModelDefinition(record), false); }
/** * Test that simple, list, and map fields are initialized as empty even when not in json */ @Test public void fieldAutoInitTest() { StringBuilder jsonString = new StringBuilder("{\n").append("\"id\": \"myId\"\n").append("}"); ZNRecordSerializer serializer = new ZNRecordSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(jsonString.toString().getBytes()); Assert.assertNotNull(result); Assert.assertEquals(result.getId(), "myId"); Assert.assertNotNull(result.getSimpleFields()); Assert.assertTrue(result.getSimpleFields().isEmpty()); Assert.assertNotNull(result.getListFields()); Assert.assertTrue(result.getListFields().isEmpty()); Assert.assertNotNull(result.getMapFields()); Assert.assertTrue(result.getMapFields().isEmpty()); }
@Test public void testCompression() { int runId = 1; while (runId < 20) { int numPartitions = runId * 1000; int replicas = 3; int numNodes = 100; Random random = new Random(); ZNRecord record = new ZNRecord("testId"); System.out.println("Partitions:" + numPartitions); for (int p = 0; p < numPartitions; p++) { Map<String, String> map = new HashMap<String, String>(); for (int r = 0; r < replicas; r++) { map.put("host_" + random.nextInt(numNodes), "ONLINE"); } record.setMapField("TestResource_" + p, map); } ZNRecordSerializer serializer = new ZNRecordSerializer(); byte[] serializedBytes; record.setSimpleField("enableCompression", "true"); serializedBytes = serializer.serialize(record); int compressedSize = serializedBytes.length; System.out.println("compressed serialized data length = " + compressedSize); ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes); Assert.assertEquals(result, record); runId = runId + 1; } } }
@Test (enabled = false) public void testPerformance() { ZNRecord record = createZnRecord(); ZNRecordSerializer serializer1 = new ZNRecordSerializer(); ZNRecordStreamingSerializer serializer2 = new ZNRecordStreamingSerializer(); int loop = 100000; long start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { serializer1.serialize(record); } System.out.println("ZNRecordSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); byte[] data = serializer1.serialize(record); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { serializer1.deserialize(data); } System.out.println("ZNRecordSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { data = serializer2.serialize(record); } System.out.println("ZNRecordStreamingSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { ZNRecord result = (ZNRecord) serializer2.deserialize(data); } System.out.println("ZNRecordStreamingSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); }
@Test public void testBasicCompression() { ZNRecord record = new ZNRecord("testId"); int numPartitions = 1024; int replicas = 3; int numNodes = 100; Random random = new Random(); for (int p = 0; p < numPartitions; p++) { Map<String, String> map = new HashMap<String, String>(); for (int r = 0; r < replicas; r++) { map.put("host_" + random.nextInt(numNodes), "ONLINE"); } record.setMapField("TestResource_" + p, map); } ZNRecordSerializer serializer = new ZNRecordSerializer(); byte[] serializedBytes; serializedBytes = serializer.serialize(record); int uncompressedSize = serializedBytes.length; System.out.println("raw serialized data length = " + serializedBytes.length); record.setSimpleField("enableCompression", "true"); serializedBytes = serializer.serialize(record); int compressedSize = serializedBytes.length; System.out.println("compressed serialized data length = " + serializedBytes.length); System.out.printf("compression ratio: %.2f \n", (uncompressedSize * 1.0 / compressedSize)); ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes); Assert.assertEquals(result, record); }
/** * Test the normal case of serialize/deserialize where ZNRecord is well-formed */ @Test public void basicTest() { ZNRecord record = new ZNRecord("testId"); record.setMapField("k1", ImmutableMap.of("a", "b", "c", "d")); record.setMapField("k2", ImmutableMap.of("e", "f", "g", "h")); record.setListField("k3", ImmutableList.of("a", "b", "c", "d")); record.setListField("k4", ImmutableList.of("d", "e", "f", "g")); record.setSimpleField("k5", "a"); record.setSimpleField("k5", "b"); ZNRecordSerializer serializer = new ZNRecordSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(serializer.serialize(record)); Assert.assertEquals(result, record); }
@Test public void testNullFields() { ZNRecord record = new ZNRecord("testId"); record.setMapField("K1", null); record.setListField("k2", null); record.setSimpleField("k3", null); ZNRecordSerializer serializer = new ZNRecordSerializer(); byte [] data = serializer.serialize(record); ZNRecord result = (ZNRecord) serializer.deserialize(data); Assert.assertEquals(result, record); Assert.assertNull(result.getMapField("K1")); Assert.assertNull(result.getListField("K2")); Assert.assertNull(result.getSimpleField("K3")); Assert.assertNull(result.getListField("K4")); }
/** * Test that the payload can be deserialized after serializing and deserializing the ZNRecord * that encloses it. This uses ZNRecordSerializer. */ @Test public void testFullZNRecordSerializeDeserialize() { final String RECORD_ID = "testFullZNRecordSerializeDeserialize"; SampleDeserialized sample = getSample(); ZNRecord znRecord = new ZNRecord(RECORD_ID); znRecord.setPayloadSerializer(new JacksonPayloadSerializer()); znRecord.setPayload(sample); ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); byte[] serialized = znRecordSerializer.serialize(znRecord); ZNRecord deserialized = (ZNRecord) znRecordSerializer.deserialize(serialized); deserialized.setPayloadSerializer(new JacksonPayloadSerializer()); SampleDeserialized duplicate = deserialized.getPayload(SampleDeserialized.class); Assert.assertEquals(duplicate, sample); }