private GenericRecord serializeWithWriterThenDeserializeWithReader(Schema writerSchema, GenericRecord datum, Schema readerSchema) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); DatumWriter<Object> datumWriter = new GenericDatumWriter<>(writerSchema); datumWriter.write(datum, encoder); encoder.flush(); byte[] bytes = baos.toByteArray(); Decoder decoder = DecoderFactory.get().resolvingDecoder( writerSchema, readerSchema, DecoderFactory.get().binaryDecoder(bytes, null)); DatumReader<Object> datumReader = new GenericDatumReader<>(readerSchema); return (GenericRecord)datumReader.read(null, decoder); }
private Record decodeGenericBlob(Schema expectedSchema, Schema schemaOfBlob, byte[] blob) throws IOException { if (blob == null) { return null; } GenericDatumReader<Record> reader = new GenericDatumReader<>(); reader.setExpected(expectedSchema); reader.setSchema(schemaOfBlob); Decoder decoder = encoderType == EncoderType.BINARY ? DecoderFactory.get().binaryDecoder(blob, null) : DecoderFactory.get().jsonDecoder(schemaOfBlob, new ByteArrayInputStream(blob)); return reader.read(null, decoder); } }
@Override public GenericContainer read(Kryo kryo, Input input, Class<GenericContainer> aClass) { Schema theSchema = this.getSchema(input.readString()); GenericDatumReader<GenericContainer> reader = new GenericDatumReader<>(theSchema); Decoder decoder = DecoderFactory .get() .directBinaryDecoder(input, null); GenericContainer foo; try { foo = reader.read(null, decoder); } catch (IOException e) { throw new RuntimeException(e); } return foo; } }
/** {@inheritDoc} */ @Override public void open(InputStream inputStream) throws IOException { mAvroDecoder = DecoderFactory.get().directBinaryDecoder(inputStream, mAvroDecoder); }
@Override protected BinaryDecoder initialValue() { InputStream dummyInputStream = new ByteArrayInputStream(new byte[0]); return DecoderFactory.get().binaryDecoder(dummyInputStream, null); } };
/** * Decode binary data. * * @param data the data * @param reuse the reuse * @return the decoded object * @throws IOException Signals that an I/O exception has occurred. */ public T decodeBinary(byte[] data, T reuse) throws IOException { binaryDecoder = DecoderFactory.get().binaryDecoder(data, binaryDecoder); return datumReader.read(reuse, binaryDecoder); }
@Test public void testAvroEncodeIO() throws IOException { Schema schm = ReflectData.get().getSchema(AvroEncRecord.class); ReflectDatumWriter<AvroEncRecord> writer = new ReflectDatumWriter<>(schm); ByteArrayOutputStream out = new ByteArrayOutputStream(); AvroEncRecord record = new AvroEncRecord(); record.date = new java.util.Date(948833323L); writer.write(record, factory.directBinaryEncoder(out, null)); ReflectDatumReader<AvroEncRecord> reader = new ReflectDatumReader<>(schm); AvroEncRecord decoded = reader.read(new AvroEncRecord(), DecoderFactory.get().binaryDecoder( out.toByteArray(), null)); assertEquals(record, decoded); }
private static void checkJson(Schema schema, Object datum, String json) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); DatumWriter<Object> writer = new GenericDatumWriter<>(); writer.setSchema(schema); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); String encoded = new String(data, StandardCharsets.UTF_8); assertEquals("Encoded data does not match.", json, encoded); DatumReader<Object> reader = new GenericDatumReader<>(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .jsonDecoder(schema, new ByteArrayInputStream(data))); assertEquals("Decoded data does not match.", datum, decoded); }
public TestBlockingIO2 (int bufferSize, int skipLevel, String calls) throws IOException { ByteArrayOutputStream os = new ByteArrayOutputStream(); EncoderFactory factory = new EncoderFactory() .configureBlockSize(bufferSize); Encoder encoder = factory.blockingBinaryEncoder(os, null); this.values = TestValidatingIO.randomValues(calls); TestValidatingIO.generate(encoder, calls, values); encoder.flush(); byte[] bb = os.toByteArray(); decoder = DecoderFactory.get().binaryDecoder(bb, null); this.calls = calls; }
@Test public void testWrite() throws IOException { String json = "{\"type\": \"record\", \"name\": \"r\", \"fields\": [" + "{ \"name\": \"f1\", \"type\": \"long\" }" + "]}"; Schema s = Schema.parse(json); GenericRecord r = new GenericData.Record(s); r.put("f1", 100L); ByteArrayOutputStream bao = new ByteArrayOutputStream(); GenericDatumWriter<GenericRecord> w = new GenericDatumWriter<>(s); Encoder e = EncoderFactory.get().jsonEncoder(s, bao); w.write(r, e); e.flush(); Object o = new GenericDatumReader<GenericRecord>(s).read(null, DecoderFactory.get().jsonDecoder(s, new ByteArrayInputStream(bao.toByteArray()))); assertEquals(r, o); }
@Test public void testSerialize() throws IOException { Schema writerSchema = Schema.create(Schema.Type.STRING); AvroSerializer<CharSequence> serializer = new AvroSerializer<>(writerSchema); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); serializer.open(outputStream); serializer.serialize(new AvroKey<>("record1")); ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); Schema readerSchema = Schema.create(Schema.Type.STRING); DatumReader<CharSequence> datumReader = new GenericDatumReader<>(readerSchema); Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); CharSequence record = null; record = datumReader.read(record, decoder); assertEquals("record1", record.toString()); record = datumReader.read(record, decoder); assertEquals("record2", record.toString()); inputStream.close();
@Test public void testNew() throws IOException { ByteBuffer payload = ByteBuffer.allocateDirect(8 * 1024); for (int i = 0; i < 500; i++) { payload.putInt(1); } payload.flip(); ByteBufferRecord bbr = new ByteBufferRecord(); bbr.setPayload(payload); bbr.setTp(TypeEnum.b); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ReflectDatumWriter<ByteBufferRecord> writer = new ReflectDatumWriter<ByteBufferRecord>(ByteBufferRecord.class); BinaryEncoder avroEncoder = EncoderFactory.get().blockingBinaryEncoder(outputStream, null); writer.write(bbr, avroEncoder); avroEncoder.flush(); byte[] bytes = outputStream.toByteArray(); ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); ReflectDatumReader<ByteBufferRecord> datumReader = new ReflectDatumReader<ByteBufferRecord>(ByteBufferRecord.class); BinaryDecoder avroDecoder = DecoderFactory.get().binaryDecoder(inputStream, null); ByteBufferRecord deserialized = datumReader.read(null, avroDecoder); Assert.assertEquals(bbr, deserialized); }
@Test public void test2() { Schema a = Schema.createRecord(ImmutableList.of( new Schema.Field("b", Schema.create(Schema.Type.NULL), "", null), new Schema.Field("a", Schema.create(Schema.Type.BOOLEAN), "", null))); GenericData.Record record = new GenericData.Record(a); record.put("a", false); ByteArrayOutputStream out = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); Schema b = Schema.createRecord(ImmutableList.of(new Schema.Field("a", Schema.create(Schema.Type.BOOLEAN), "", null))); GenericDatumReader<Object> reader = new GenericDatumReader<>(b); BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(new ByteArrayInputStream(out.toByteArray()), null); read = reader.read(null, decoder); } catch (IOException e) { throw new RuntimeException(e);
+ " not set and has no default value"); if (json.isNull() && (field.schema().getType() == Type.NULL || (field.schema().getType() == Type.UNION && field.schema().getTypes().get(0).getType() == Type.NULL))) { return null; ByteArrayOutputStream baos = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null); ResolvingGrammarGenerator.encode(encoder, field.schema(), json); encoder.flush(); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(baos.toByteArray(), null); defaultValue = createDatumReader(field.schema()).read(null, decoder);
public static void checkDirectBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(datum, encoder); // no flush for direct byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
public static void checkBlockingBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
public static void checkBinaryJson(String json) throws Exception { Object node = Json.parseJson(json); ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter<Object> writer = new Json.ObjectWriter(); Encoder encoder = EncoderFactory.get().binaryEncoder(out, null); encoder = EncoderFactory.get().validatingEncoder(Json.SCHEMA, encoder); writer.write(node, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); DatumReader<Object> reader = new Json.ObjectReader(); Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); decoder = DecoderFactory.get().validatingDecoder(Json.SCHEMA, decoder); Object decoded = reader.read(null, decoder); assertEquals("Decoded json does not match.", Json.toString(node), Json.toString(decoded)); }
public static Object checkBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader, Object reuse) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().binaryEncoder(out, null); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(reuse, DecoderFactory.get().binaryDecoder( data, null)); assertEquals("Decoded data does not match.", datum, decoded); return decoded; }
@Test public void testEnumMismatch() throws Exception { Schema actual = Schema.parse ("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"X\",\"Y\"]}"); Schema expected = Schema.parse ("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"Y\",\"Z\"]}"); ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter<Object> writer = new GenericDatumWriter<>(actual); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(new GenericData.EnumSymbol(actual, "Y"), encoder); writer.write(new GenericData.EnumSymbol(actual, "X"), encoder); encoder.flush(); byte[] data = out.toByteArray(); Decoder decoder = DecoderFactory.get().binaryDecoder( data, null); DatumReader<String> in = new GenericDatumReader<>(actual, expected); assertEquals("Wrong value", new GenericData.EnumSymbol(expected, "Y"), in.read(null, decoder)); try { in.read(null, decoder); fail("Should have thrown exception."); } catch (AvroTypeException e) { // expected } }
private static void checkBinary(ReflectData reflectData, Schema schema, Object datum, boolean equals, boolean blocking) throws IOException { ReflectDatumWriter<Object> writer = new ReflectDatumWriter<>(schema); ByteArrayOutputStream out = new ByteArrayOutputStream(); if (!blocking) { writer.write(datum, EncoderFactory.get().directBinaryEncoder(out, null)); } else { writer.write(datum, new EncoderFactory().configureBlockSize(64) .blockingBinaryEncoder(out, null)); } writer.write(datum, EncoderFactory.get().directBinaryEncoder(out, null)); byte[] data = out.toByteArray(); ReflectDatumReader<Object> reader = new ReflectDatumReader<>(schema); Object decoded = reader.read(null, DecoderFactory.get().binaryDecoder(data, null)); assertEquals(0, reflectData.compare(datum, decoded, schema, equals)); }