/** {@inheritDoc} */ @Override public void open(OutputStream outputStream) throws IOException { mOutputStream = outputStream; mAvroEncoder = mEncoderFactory.binaryEncoder(outputStream, mAvroEncoder); }
@Override protected Encoder getEncoder(Schema schema, OutputStream outputStream) { return EncoderFactory.get().binaryEncoder(outputStream, null); } }
public GenericAvroSchema(SchemaInfo schemaInfo) { super(schemaInfo); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, encoder); this.datumWriter = new GenericDatumWriter(schema); this.datumReader = new GenericDatumReader(schema); }
/** * Creates an Avro serialization schema for the given Avro schema string. * * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record */ public AvroRowSerializationSchema(String avroSchemaString) { Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); this.recordClazz = null; this.schemaString = avroSchemaString; try { this.schema = new Schema.Parser().parse(avroSchemaString); } catch (SchemaParseException e) { throw new IllegalArgumentException("Could not parse Avro schema string.", e); } this.datumWriter = new GenericDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
@Override public void init(final DataFileStream<GenericRecord> reader, final String codec, final OutputStream out) throws IOException { writer = new GenericDatumWriter<>(reader.getSchema()); encoder = EncoderFactory.get().binaryEncoder(out, null); }
@SuppressWarnings("unchecked") private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { recordClazz = (Class<? extends SpecificRecord>) inputStream.readObject(); schemaString = (String) inputStream.readObject(); if (recordClazz != null) { schema = SpecificData.get().getSchema(recordClazz); } else { schema = new Schema.Parser().parse(schemaString); } datumWriter = new SpecificDatumWriter<>(schema); arrayOutputStream = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); } }
/** * Creates an Avro serialization schema for the given specific record class. * * @param recordClazz Avro record class used to serialize Flink's row to Avro's record */ public AvroRowSerializationSchema(Class<? extends SpecificRecord> recordClazz) { Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); this.recordClazz = recordClazz; this.schema = SpecificData.get().getSchema(recordClazz); this.schemaString = schema.toString(); this.datumWriter = new SpecificDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
private AvroSchema(org.apache.avro.Schema schema, Map<String, String> properties) { this.schema = schema; this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); this.schemaInfo.setType(SchemaType.AVRO); this.schemaInfo.setSchema(this.schema.toString().getBytes()); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, this.encoder); this.datumWriter = new ReflectDatumWriter<>(this.schema); this.reader = new ReflectDatumReader<>(this.schema); }
/** * Writes given record using specified schema. * @param record record to serialize * @param schema schema to use for serialization * @return serialized record */ public static byte[] writeRecord(GenericRecord record, Schema schema) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(stream, null); new GenericDatumWriter<>(schema).write(record, encoder); encoder.flush(); return stream.toByteArray(); } }
private void writeObject(Schema schema, GenericRecord datum) throws Exception { BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(new ByteArrayOutputStream(), null); GenericDatumWriter<GenericData.Record> writer = new GenericDatumWriter<>(schema); writer.write(schema, datum, encoder); }
@Test public void testBinaryEncoderInit() throws IOException { OutputStream out = new ByteArrayOutputStream(); BinaryEncoder enc = factory.binaryEncoder(out, null); Assert.assertSame(enc, factory.binaryEncoder(out, enc)); }
protected Encoder newEncoder(ByteArrayOutputStream out) throws IOException { Encoder e = (USE_DIRECT_ENCODER ? encoder_factory.directBinaryEncoder(out, null) : encoder_factory.binaryEncoder(out, null)); // Encoder e = encoder_factory.blockingBinaryEncoder(out, null); // Encoder e = new LegacyBinaryEncoder(out); return e; }
@Test(expected=NullPointerException.class) public void testBadBinaryEncoderInit() { factory.binaryEncoder(null, null); }
private byte[] encodeGenericBlob(GenericRecord data) throws IOException { DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(data.getSchema()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); Encoder encoder = encoderType == EncoderType.BINARY ? EncoderFactory.get().binaryEncoder(outStream, null) : EncoderFactory.get().jsonEncoder(data.getSchema(), outStream); writer.write(data, encoder); encoder.flush(); outStream.close(); return outStream.toByteArray(); }
public static byte[] serializeRecord(FooBarSpecificRecord fooBarSpecificRecord) throws IOException { SpecificDatumWriter<FooBarSpecificRecord> datumWriter = new SpecificDatumWriter<>(FooBarSpecificRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(fooBarSpecificRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
private static <T> byte[] serializeWithReflectDatumWriter(T toSerialize, Class<T> toSerializeClass) throws IOException { ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter<>(toSerializeClass); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(toSerialize, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
public static byte[] serializeRecord(StringablesRecord stringablesRecord) throws IOException { SpecificDatumWriter<StringablesRecord> datumWriter = new SpecificDatumWriter<>(StringablesRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(stringablesRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
private void testWithBuffer(ByteBuffer buffer) throws IOException { assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); ByteArrayOutputStream output = new ByteArrayOutputStream(EXAMPLE_DATA_SIZE * 2); EncoderFactory encoderFactory = new EncoderFactory(); encoderFactory.configureBufferSize(ENCODER_BUFFER_SIZE); Encoder encoder = encoderFactory.binaryEncoder(output, null); new GenericDatumWriter<ByteBuffer>(Schema.create(Schema.Type.BYTES)).write(buffer, encoder); encoder.flush(); assertThat(output.toByteArray(), equalTo(avroEncoded(someBytes(EXAMPLE_DATA_SIZE)))); assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); // fails if buffer is not array-backed and buffer overflow occurs }
@Test public void test() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); EncoderFactory factory = EncoderFactory.get(); Encoder e = factory.validatingEncoder(schema, factory.binaryEncoder(baos, null)); ResolvingGrammarGenerator.encode(e, schema, data); e.flush(); }
@Test(expected = EOFException.class) public void testEOF() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder e = EncoderFactory.get().binaryEncoder(baos, null); e.writeLong(0x10000000000000L); e.flush(); Decoder d = newDecoder(new ByteArrayInputStream(baos.toByteArray())); Assert.assertEquals(0x10000000000000L, d.readLong()); d.readInt(); }