@Override public void emit(T data, Map<String, Object> partitions) { try { ByteArrayOutputStream output = new ByteArrayOutputStream(); output.write(schemaHash); writer.encode(data, new BinaryEncoder(output)); producerSupplier.get().enqueue(new QueueEntry(Maps.transformValues(partitions, PARTITION_MAP_TRANSFORMER), output.toByteArray())); } catch (IOException e) { throw Throwables.propagate(e); } }
@Override protected void publish(Iterator<MetricValues> metrics) throws Exception { int size = topicPayloads.size(); while (metrics.hasNext()) { encoderOutputStream.reset(); MetricValues metricValues = metrics.next(); // Encode MetricValues into bytes recordWriter.encode(metricValues, encoder); TopicPayload topicPayload = topicPayloads.get(Math.abs(metricValues.getTags().hashCode() % size)); // Calculate the topic number with the hashcode of MetricValues' tags and store the encoded payload in the // corresponding list of the topic number topicPayload.addPayload(encoderOutputStream.toByteArray(), metricValues.getTags(), metricValues.getMetrics().size()); } publishMetric(topicPayloads.values()); }
@Override protected void publish(Iterator<MetricValues> metrics) throws Exception { int size = topicPayloads.size(); while (metrics.hasNext()) { encoderOutputStream.reset(); MetricValues metricValues = metrics.next(); // Encode MetricValues into bytes recordWriter.encode(metricValues, encoder); TopicPayload topicPayload = topicPayloads.get(Math.abs(metricValues.getTags().hashCode() % size)); // Calculate the topic number with the hashcode of MetricValues' tags and store the encoded payload in the // corresponding list of the topic number topicPayload.addPayload(encoderOutputStream.toByteArray(), metricValues.getTags(), metricValues.getMetrics().size()); } publishMetric(topicPayloads.values()); }
@Override public byte[] encode(AccessToken token) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(AccessToken.Schemas.getVersion()); DatumWriter<AccessToken> writer = writerFactory.create(ACCESS_TOKEN_TYPE, AccessToken.Schemas.getCurrentSchema()); writer.encode(token, encoder); return bos.toByteArray(); }
@Override public byte[] encode(KeyIdentifier keyIdentifier) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(KeyIdentifier.Schemas.getVersion()); DatumWriter<KeyIdentifier> writer = writerFactory.create(KEY_IDENTIFIER_TYPE, KeyIdentifier.Schemas.getCurrentSchema()); writer.encode(keyIdentifier, encoder); return bos.toByteArray(); }
@Override public byte[] encode(AccessTokenIdentifier identifier) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(AccessTokenIdentifier.Schemas.getVersion()); DatumWriter<AccessTokenIdentifier> writer = writerFactory.create(ACCESS_TOKEN_IDENTIFIER_TYPE, AccessTokenIdentifier.Schemas.getCurrentSchema()); writer.encode(identifier, encoder); return bos.toByteArray(); }
@Test public void testMap() throws IOException, UnsupportedTypeException { TypeToken<Map<String, List<String>>> type = new TypeToken<Map<String, List<String>>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Map<String, List<String>>> writer = getWriter(type); ImmutableMap<String, List<String>> map = ImmutableMap.<String, List<String>>of("k1", Lists.newArrayList("v1"), "k2", Lists.newArrayList("v2", null)); writer.encode(map, new BinaryEncoder(os)); ReflectionDatumReader<Map<String, List<String>>> reader = new ReflectionDatumReader<>(getSchema(type), type); Assert.assertEquals(map, reader.read(new BinaryDecoder(is), getSchema(type))); }
@Test public void testURI() throws IOException, UnsupportedTypeException { TypeToken<List<URI>> type = new TypeToken<List<URI>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<List<URI>> writer = getWriter(type); List<URI> writeValue = ImmutableList.of(URI.create("http://www.abc.com")); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<URI>> reader = new ReflectionDatumReader<>(getSchema(type), type); Assert.assertEquals(writeValue, reader.read(new BinaryDecoder(is), getSchema(type))); }
@Test public void testRecordContainer() throws IOException, UnsupportedTypeException { TypeToken<List<Record>> type = new TypeToken<List<Record>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<List<Record>> writer = getWriter(type); List<Record> writeValue = ImmutableList.of( new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2)); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<Record>> reader = new ReflectionDatumReader<>(getSchema(type), type); List<Record> value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testUUID() throws UnsupportedTypeException, IOException { TypeToken<UUID> type = new TypeToken<UUID>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<UUID> writer = getWriter(type); UUID uuid = UUID.randomUUID(); writer.encode(uuid, new BinaryEncoder(os)); ReflectionDatumReader<UUID> reader = new ReflectionDatumReader<>(getSchema(type), type); UUID value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(uuid, value); }
@Test public void testList() throws IOException, UnsupportedTypeException { TypeToken<List<Long>> type = new TypeToken<List<Long>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); List<Long> writeValue = ImmutableList.of(1L, 10L, 100L, 1000L); DatumWriter<List<Long>> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<Long>> reader = new ReflectionDatumReader<>(getSchema(type), type); List<Long> value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testRecordArray() throws IOException, UnsupportedTypeException { TypeToken<Record[][]> type = new TypeToken<Record[][]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Record[][]> writer = getWriter(type); Record[][] writeValue = new Record[][] {{ new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2)}}; writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<Record[][]> reader = new ReflectionDatumReader<>(getSchema(type), type); Record[][] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testRecord() throws IOException, UnsupportedTypeException { TypeToken<Record> type = new TypeToken<Record>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Record> writer = getWriter(type); Record writeValue = new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<Record> reader = new ReflectionDatumReader<>(getSchema(type), type); Record value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testInt() throws UnsupportedTypeException, IOException { TypeToken<Integer> type = new TypeToken<Integer>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Integer> writer = getWriter(type); writer.encode(12234234, new BinaryEncoder(os)); ReflectionDatumReader<Integer> reader = new ReflectionDatumReader<>(getSchema(type), type); int value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(12234234, value); }
@Test public void testDouble() throws UnsupportedTypeException, IOException { TypeToken<Double> type = new TypeToken<Double>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Double> writer = getWriter(type); writer.encode(3.14d, new BinaryEncoder(os)); ReflectionDatumReader<Double> reader = new ReflectionDatumReader<>(getSchema(type), type); double value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(3.14d, value, 0.000001d); }
@Test public void testShort() throws UnsupportedTypeException, IOException { TypeToken<Short> type = new TypeToken<Short>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Short> writer = getWriter(type); writer.encode((short) 3000, new BinaryEncoder(os)); ReflectionDatumReader<Short> reader = new ReflectionDatumReader<>(getSchema(type), type); short value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals((short) 3000, value); }
@Test public void testString() throws UnsupportedTypeException, IOException { TypeToken<String> type = new TypeToken<String>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<String> writer = getWriter(type); writer.encode("Testing message", new BinaryEncoder(os)); ReflectionDatumReader<String> reader = new ReflectionDatumReader<>(getSchema(type), type); String value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals("Testing message", value); }
@Test public void testPrimitiveArray() throws IOException, UnsupportedTypeException { TypeToken<int[]> type = new TypeToken<int[]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); int[] writeValue = {1, 2, 3, 4, -5, -6, -7, -8}; DatumWriter<int[]> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<int[]> reader = new ReflectionDatumReader<>(getSchema(type), type); int[] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testReferenceArray() throws IOException, UnsupportedTypeException { TypeToken<String[]> type = new TypeToken<String[]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); String[] writeValue = new String[] {"1", "2", null, "3"}; DatumWriter<String[]> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<String[]> reader = new ReflectionDatumReader<>(getSchema(type), type); String[] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testTree() throws IOException, UnsupportedTypeException { TypeToken<Node> type = new TypeToken<Node>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Node> writer = getWriter(type); Node root = new Node((short) 1, new Node((short) 2, null, new Node((short) 3, null, null)), new Node((short) 4, new Node((short) 5, null, null), null)); writer.encode(root, new BinaryEncoder(os)); ReflectionDatumReader<Node> reader = new ReflectionDatumReader<>(getSchema(type), type); Node value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(root, value); }