@Override public Encoder apply(OutputStream input) { return new BinaryEncoder(input); } };
private static byte[] serializeEmptyHashKeys() { try { // we don't synchronize here: the worst thing that go wrong here is repeated assignment to the same value ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(0); return bos.toByteArray(); } catch (IOException e) { throw new RuntimeException("encoding empty hash keys went wrong - bailing out: " + e.getMessage(), e); } } private static final byte[] SERIALIZED_EMPTY_HASH_KEYS = serializeEmptyHashKeys();
private byte[] encode(T object) { // encode T using schema ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); try { this.datumWriter.encode(object, encoder); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to encode object to be written: " + e.getMessage(), e); } return bos.toByteArray(); }
private byte[] encode(T object) { // encode T using schema ByteArrayOutputStream bos = new ByteArrayOutputStream(); BinaryEncoder encoder = new BinaryEncoder(bos); try { this.datumWriter.encode(object, encoder); } catch (IOException e) { // SHOULD NEVER happen throw new DataSetException("Failed to encode object to be written: " + e.getMessage(), e); } return bos.toByteArray(); }
@Override public void emit(T data, Map<String, Object> partitions) { try { ByteArrayOutputStream output = new ByteArrayOutputStream(); output.write(schemaHash); writer.encode(data, new BinaryEncoder(output)); producerSupplier.get().enqueue(new QueueEntry(Maps.transformValues(partitions, PARTITION_MAP_TRANSFORMER), output.toByteArray())); } catch (IOException e) { throw Throwables.propagate(e); } }
@Override public byte[] encode(AccessToken token) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(AccessToken.Schemas.getVersion()); DatumWriter<AccessToken> writer = writerFactory.create(ACCESS_TOKEN_TYPE, AccessToken.Schemas.getCurrentSchema()); writer.encode(token, encoder); return bos.toByteArray(); }
@Override public byte[] encode(KeyIdentifier keyIdentifier) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(KeyIdentifier.Schemas.getVersion()); DatumWriter<KeyIdentifier> writer = writerFactory.create(KEY_IDENTIFIER_TYPE, KeyIdentifier.Schemas.getCurrentSchema()); writer.encode(keyIdentifier, encoder); return bos.toByteArray(); }
@Override public byte[] encode(AccessTokenIdentifier identifier) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(bos); encoder.writeInt(AccessTokenIdentifier.Schemas.getVersion()); DatumWriter<AccessTokenIdentifier> writer = writerFactory.create(ACCESS_TOKEN_IDENTIFIER_TYPE, AccessTokenIdentifier.Schemas.getCurrentSchema()); writer.encode(identifier, encoder); return bos.toByteArray(); }
@Test public void testURI() throws IOException, UnsupportedTypeException { TypeToken<List<URI>> type = new TypeToken<List<URI>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<List<URI>> writer = getWriter(type); List<URI> writeValue = ImmutableList.of(URI.create("http://www.abc.com")); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<URI>> reader = new ReflectionDatumReader<>(getSchema(type), type); Assert.assertEquals(writeValue, reader.read(new BinaryDecoder(is), getSchema(type))); }
@Test public void testUUID() throws UnsupportedTypeException, IOException { TypeToken<UUID> type = new TypeToken<UUID>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<UUID> writer = getWriter(type); UUID uuid = UUID.randomUUID(); writer.encode(uuid, new BinaryEncoder(os)); ReflectionDatumReader<UUID> reader = new ReflectionDatumReader<>(getSchema(type), type); UUID value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(uuid, value); }
@Test public void testList() throws IOException, UnsupportedTypeException { TypeToken<List<Long>> type = new TypeToken<List<Long>>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); List<Long> writeValue = ImmutableList.of(1L, 10L, 100L, 1000L); DatumWriter<List<Long>> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<List<Long>> reader = new ReflectionDatumReader<>(getSchema(type), type); List<Long> value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testRecord() throws IOException, UnsupportedTypeException { TypeToken<Record> type = new TypeToken<Record>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Record> writer = getWriter(type); Record writeValue = new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<Record> reader = new ReflectionDatumReader<>(getSchema(type), type); Record value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(writeValue, value); }
@Test public void testRecordArray() throws IOException, UnsupportedTypeException { TypeToken<Record[][]> type = new TypeToken<Record[][]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Record[][]> writer = getWriter(type); Record[][] writeValue = new Record[][] {{ new Record(10, "testing", ImmutableList.of("a", "b", "c"), TestEnum.VALUE2)}}; writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<Record[][]> reader = new ReflectionDatumReader<>(getSchema(type), type); Record[][] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testInt() throws UnsupportedTypeException, IOException { TypeToken<Integer> type = new TypeToken<Integer>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Integer> writer = getWriter(type); writer.encode(12234234, new BinaryEncoder(os)); ReflectionDatumReader<Integer> reader = new ReflectionDatumReader<>(getSchema(type), type); int value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(12234234, value); }
@Test public void testDouble() throws UnsupportedTypeException, IOException { TypeToken<Double> type = new TypeToken<Double>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Double> writer = getWriter(type); writer.encode(3.14d, new BinaryEncoder(os)); ReflectionDatumReader<Double> reader = new ReflectionDatumReader<>(getSchema(type), type); double value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals(3.14d, value, 0.000001d); }
@Test(expected = IOException.class) public void testCircularRef() throws UnsupportedTypeException, IOException { Schema schema = new ReflectionSchemaGenerator().generate(Node.class); Node head = new Node(); head.next = new Node(); head.next.next = head; ByteArrayOutputStream output = new ByteArrayOutputStream(); new ReflectionDatumWriter<Node>(schema).encode(head, new BinaryEncoder(output)); }
@Test public void testShort() throws UnsupportedTypeException, IOException { TypeToken<Short> type = new TypeToken<Short>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<Short> writer = getWriter(type); writer.encode((short) 3000, new BinaryEncoder(os)); ReflectionDatumReader<Short> reader = new ReflectionDatumReader<>(getSchema(type), type); short value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals((short) 3000, value); }
@Test public void testString() throws UnsupportedTypeException, IOException { TypeToken<String> type = new TypeToken<String>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); DatumWriter<String> writer = getWriter(type); writer.encode("Testing message", new BinaryEncoder(os)); ReflectionDatumReader<String> reader = new ReflectionDatumReader<>(getSchema(type), type); String value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertEquals("Testing message", value); }
@Test public void testPrimitiveArray() throws IOException, UnsupportedTypeException { TypeToken<int[]> type = new TypeToken<int[]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); int[] writeValue = {1, 2, 3, 4, -5, -6, -7, -8}; DatumWriter<int[]> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<int[]> reader = new ReflectionDatumReader<>(getSchema(type), type); int[] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }
@Test public void testReferenceArray() throws IOException, UnsupportedTypeException { TypeToken<String[]> type = new TypeToken<String[]>() { }; PipedOutputStream os = new PipedOutputStream(); PipedInputStream is = new PipedInputStream(os); String[] writeValue = new String[] {"1", "2", null, "3"}; DatumWriter<String[]> writer = getWriter(type); writer.encode(writeValue, new BinaryEncoder(os)); ReflectionDatumReader<String[]> reader = new ReflectionDatumReader<>(getSchema(type), type); String[] value = reader.read(new BinaryDecoder(is), getSchema(type)); Assert.assertArrayEquals(writeValue, value); }