zkClient.setZkSerializer(new ZNRecordStreamingSerializer()); LOGGER.info("Connecting to Zookeeper at: {}", _zkAddress); zkClient.waitUntilConnected(CommonConstants.Helix.ZkClient.DEFAULT_CONNECT_TIMEOUT_SEC, TimeUnit.SECONDS);
ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); byte[] bytes = serializer.serialize(record); System.out.println(new String(bytes)); ZNRecord record2 = (ZNRecord) serializer.deserialize(bytes); System.out.println(record2); bytes = serializer.serialize(record); record2 = (ZNRecord) serializer.deserialize(bytes);
/** * Check that the ZNRecord is not constructed if there is no id in the json */ @Test public void noIdTest() { StringBuilder jsonString = new StringBuilder("{\n").append(" \"simpleFields\": {},\n") .append(" \"listFields\": {},\n").append(" \"mapFields\": {}\n").append("}\n"); ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(jsonString.toString().getBytes()); Assert.assertNull(result); }
/** * Test that the payload is not included whenever it is not null. This is mainly to maintain * backward * compatibility. */ @Test public void testRawPayloadMissingIfUnspecified() { final String RECORD_ID = "testRawPayloadMissingIfUnspecified"; ZNRecord znRecord = new ZNRecord(RECORD_ID); ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer(); byte[] serialized = znRecordSerializer.serialize(znRecord); ZNRecordStreamingSerializer znRecordStreamingSerializer = new ZNRecordStreamingSerializer(); byte[] streamingSerialized = znRecordStreamingSerializer.serialize(znRecord); ObjectMapper mapper = new ObjectMapper(); try { JsonNode jsonNode = mapper.readTree(new String(serialized)); Assert.assertFalse(jsonNode.has("rawPayload")); JsonNode streamingJsonNode = mapper.readTree(new String(streamingSerialized)); Assert.assertFalse(streamingJsonNode.has("rawPayload")); } catch (JsonProcessingException e) { Assert.fail(); } catch (IOException e) { Assert.fail(); } }
int max = getListFieldBound(record); if (max < Integer.MAX_VALUE) { Map<String, List<String>> listMap = record.getListFields();
ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); byte[] bytes = serializer.serialize(record); System.out.println(new String(bytes)); ZNRecord record2 = (ZNRecord) serializer.deserialize(bytes); System.out.println(record2); bytes = serializer.serialize(record); record2 = (ZNRecord) serializer.deserialize(bytes);
/** * Test that the json still deserizalizes correctly if id is not first */ @Test public void idNotFirstTest() { StringBuilder jsonString = new StringBuilder("{\n").append(" \"simpleFields\": {},\n") .append(" \"listFields\": {},\n").append(" \"mapFields\": {},\n") .append("\"id\": \"myId\"\n").append("}"); ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(jsonString.toString().getBytes()); Assert.assertNotNull(result); Assert.assertEquals(result.getId(), "myId"); }
System.currentTimeMillis())); ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); HelixZkClient zkClient = SharedZkClientFactory.getInstance() .buildZkClient(new HelixZkClient.ZkConnectionConfig(ZK_ADDR)); Assert.assertTrue(serializer.serialize(record).length > 900 * 1024); byte[] arr = serializer.serialize(record); byte[] arrNew = serializer.serialize(recordNew); Assert.assertFalse(Arrays.equals(arr, arrNew)); Assert.assertTrue(succeed); record = accessor.getProperty(keyBuilder.idealStates("TestDB_2")).getRecord(); Assert.assertTrue(serializer.serialize(record).length > 900 * 1024); Assert.assertTrue(succeed); recordNew = accessor.getProperty(keyBuilder.idealStates("TestDB_2")).getRecord(); arr = serializer.serialize(record); arrNew = serializer.serialize(recordNew); Assert.assertFalse(Arrays.equals(arr, arrNew)); } catch (HelixException ex) {
int max = getListFieldBound(record); if (max < Integer.MAX_VALUE) { Map<String, List<String>> listMap = record.getListFields();
@Test public void testCompression() { int runId = 1; while (runId < 20) { int numPartitions = runId * 1000; int replicas = 3; int numNodes = 100; Random random = new Random(); ZNRecord record = new ZNRecord("testId"); System.out.println("Partitions:" + numPartitions); for (int p = 0; p < numPartitions; p++) { Map<String, String> map = new HashMap<String, String>(); for (int r = 0; r < replicas; r++) { map.put("host_" + random.nextInt(numNodes), "ONLINE"); } record.setMapField("TestResource_" + p, map); } ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); byte[] serializedBytes; record.setSimpleField("enableCompression", "true"); serializedBytes = serializer.serialize(record); int compressedSize = serializedBytes.length; System.out.println("compressed serialized data length = " + compressedSize); ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes); Assert.assertEquals(result, record); runId = runId + 1; } } }
/** * Test that simple, list, and map fields are initialized as empty even when not in json */ @Test public void fieldAutoInitTest() { StringBuilder jsonString = new StringBuilder("{\n").append("\"id\": \"myId\"\n").append("}"); ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(jsonString.toString().getBytes()); Assert.assertNotNull(result); Assert.assertEquals(result.getId(), "myId"); Assert.assertNotNull(result.getSimpleFields()); Assert.assertTrue(result.getSimpleFields().isEmpty()); Assert.assertNotNull(result.getListFields()); Assert.assertTrue(result.getListFields().isEmpty()); Assert.assertNotNull(result.getMapFields()); Assert.assertTrue(result.getMapFields().isEmpty()); } @Test
void createClient() throws Exception { PathBasedZkSerializer zkSerializer = ChainedPathZkSerializer.builder(new ZNRecordStreamingSerializer()).build();
@Test public void testBasicCompression() { ZNRecord record = new ZNRecord("testId"); int numPartitions = 1024; int replicas = 3; int numNodes = 100; Random random = new Random(); for (int p = 0; p < numPartitions; p++) { Map<String, String> map = new HashMap<String, String>(); for (int r = 0; r < replicas; r++) { map.put("host_" + random.nextInt(numNodes), "ONLINE"); } record.setMapField("TestResource_" + p, map); } ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); byte[] serializedBytes; serializedBytes = serializer.serialize(record); int uncompressedSize = serializedBytes.length; System.out.println("raw serialized data length = " + serializedBytes.length); record.setSimpleField("enableCompression", "true"); serializedBytes = serializer.serialize(record); int compressedSize = serializedBytes.length; System.out.println("compressed serialized data length = " + serializedBytes.length); System.out.printf("compression ratio: %.2f \n", (uncompressedSize * 1.0 / compressedSize)); ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes); Assert.assertEquals(result, record); }
@Test (enabled = false) public void testParallelPerformance() throws ExecutionException, InterruptedException { final ZNRecord record = createZnRecord(); final ZNRecordSerializer serializer1 = new ZNRecordSerializer(); final ZNRecordStreamingSerializer serializer2 = new ZNRecordStreamingSerializer(); int loop = 100000; ExecutorService executorService = Executors.newFixedThreadPool(10000); long start = System.currentTimeMillis(); batchSerialize(serializer1, executorService, loop, record); System.out.println("ZNRecordSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); byte[] data = serializer1.serialize(record); start = System.currentTimeMillis(); batchSerialize(serializer2, executorService, loop, record); System.out.println("ZNRecordSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { data = serializer2.serialize(record); } System.out.println("ZNRecordStreamingSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { ZNRecord result = (ZNRecord) serializer2.deserialize(data); } System.out.println("ZNRecordStreamingSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); }
/** * Test the normal case of serialize/deserialize where ZNRecord is well-formed */ @Test public void basicTest() { ZNRecord record = new ZNRecord("testId"); record.setMapField("k1", ImmutableMap.of("a", "b", "c", "d")); record.setMapField("k2", ImmutableMap.of("e", "f", "g", "h")); record.setListField("k3", ImmutableList.of("a", "b", "c", "d")); record.setListField("k4", ImmutableList.of("d", "e", "f", "g")); record.setSimpleField("k5", "a"); record.setSimpleField("k5", "b"); ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); ZNRecord result = (ZNRecord) serializer.deserialize(serializer.serialize(record)); Assert.assertEquals(result, record); }
@Test (enabled = false) public void testPerformance() { ZNRecord record = createZnRecord(); ZNRecordSerializer serializer1 = new ZNRecordSerializer(); ZNRecordStreamingSerializer serializer2 = new ZNRecordStreamingSerializer(); int loop = 100000; long start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { serializer1.serialize(record); } System.out.println("ZNRecordSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); byte[] data = serializer1.serialize(record); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { serializer1.deserialize(data); } System.out.println("ZNRecordSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { data = serializer2.serialize(record); } System.out.println("ZNRecordStreamingSerializer serialize took " + (System.currentTimeMillis() - start) + " ms"); start = System.currentTimeMillis(); for (int i = 0; i < loop; i++) { ZNRecord result = (ZNRecord) serializer2.deserialize(data); } System.out.println("ZNRecordStreamingSerializer deserialize took " + (System.currentTimeMillis() - start) + " ms"); }
@Test (enabled = false) public void testNullFields() { ZNRecord record = new ZNRecord("testId"); record.setMapField("K1", null); record.setListField("k2", null); record.setSimpleField("k3", null); ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer(); byte [] data = serializer.serialize(record); ZNRecord result = (ZNRecord) serializer.deserialize(data); Assert.assertEquals(result, record); Assert.assertNull(result.getMapField("K1")); Assert.assertNull(result.getListField("K2")); Assert.assertNull(result.getSimpleField("K3")); Assert.assertNull(result.getListField("K4")); }
/** * Test that the payload can be deserialized after serializing and deserializing the ZNRecord * that encloses it. This uses ZNRecordStreamingSerializer. */ @Test public void testFullZNRecordStreamingSerializeDeserialize() { final String RECORD_ID = "testFullZNRecordStreamingSerializeDeserialize"; SampleDeserialized sample = getSample(); ZNRecord znRecord = new ZNRecord(RECORD_ID); znRecord.setPayloadSerializer(new JacksonPayloadSerializer()); znRecord.setPayload(sample); ZNRecordStreamingSerializer znRecordSerializer = new ZNRecordStreamingSerializer(); byte[] serialized = znRecordSerializer.serialize(znRecord); ZNRecord deserialized = (ZNRecord) znRecordSerializer.deserialize(serialized); deserialized.setPayloadSerializer(new JacksonPayloadSerializer()); SampleDeserialized duplicate = deserialized.getPayload(SampleDeserialized.class); Assert.assertEquals(duplicate, sample); }