public LiAvroSerializerBase() { isKey = false; encoderFactory = EncoderFactory.get(); }
public GenericAvroSchema(SchemaInfo schemaInfo) { super(schemaInfo); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, encoder); this.datumWriter = new GenericDatumWriter(schema); this.datumReader = new GenericDatumReader(schema); }
@Override protected Encoder getEncoder(Schema schema, OutputStream outputStream) { return EncoderFactory.get().binaryEncoder(outputStream, null); } }
public Serializer(Schema schema) { try { this.writer = new GenericDatumWriter<>(schema); this.outputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().jsonEncoder(schema, this.outputStream); } catch (IOException ioe) { throw new RuntimeException("Could not initialize avro json encoder."); } }
@Override protected Encoder getEncoder(Schema schema, OutputStream outputStream) throws IOException { return EncoderFactory.get().jsonEncoder(schema, outputStream); } }
/** * Creates an Avro serialization schema for the given Avro schema string. * * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record */ public AvroRowSerializationSchema(String avroSchemaString) { Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); this.recordClazz = null; this.schemaString = avroSchemaString; try { this.schema = new Schema.Parser().parse(avroSchemaString); } catch (SchemaParseException e) { throw new IllegalArgumentException("Could not parse Avro schema string.", e); } this.datumWriter = new GenericDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
/** * Convert a GenericRecord to a byte array. */ public static byte[] recordToByteArray(GenericRecord record) throws IOException { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema()); writer.write(record, encoder); byte[] byteArray = out.toByteArray(); return byteArray; } }
private void dumpJson(PrintStream out, Schema schema, Object datum) throws IOException { DatumWriter<Object> writer = new GenericDatumWriter<>(schema); JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out, true); writer.write(datum, jsonEncoder); jsonEncoder.flush(); out.println(); out.flush(); }
@SuppressWarnings("unchecked") private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { recordClazz = (Class<? extends SpecificRecord>) inputStream.readObject(); schemaString = (String) inputStream.readObject(); if (recordClazz != null) { schema = SpecificData.get().getSchema(recordClazz); } else { schema = new Schema.Parser().parse(schemaString); } datumWriter = new SpecificDatumWriter<>(schema); arrayOutputStream = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); } }
@Override public void init(final DataFileStream<GenericRecord> reader, final String codec, final OutputStream out) throws IOException { writer = new GenericDatumWriter<>(reader.getSchema()); encoder = EncoderFactory.get().binaryEncoder(out, null); }
public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); encoder = EncoderFactory.get().jsonEncoder(schema, output); }
/** * Creates an Avro serialization schema for the given specific record class. * * @param recordClazz Avro record class used to serialize Flink's row to Avro's record */ public AvroRowSerializationSchema(Class<? extends SpecificRecord> recordClazz) { Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); this.recordClazz = recordClazz; this.schema = SpecificData.get().getSchema(recordClazz); this.schemaString = schema.toString(); this.datumWriter = new SpecificDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
@Override public void write(DataOutput out) throws IOException { // Write schema since we need it to pull the data out. (see point #1 above) String schemaString = record.getSchema().toString(false); out.writeUTF(schemaString); schemaString = fileSchema.toString(false); out.writeUTF(schemaString); recordReaderID.write(out); // Write record to byte buffer GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(); BinaryEncoder be = EncoderFactory.get().directBinaryEncoder((DataOutputStream)out, null); gdw.setSchema(record.getSchema()); gdw.write(record, be); }
@Override public byte[] getBytes(AvroGenericRecordWritable writable) { GenericRecord record = writable.getRecord(); byte[] valueBytes = null; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { BinaryEncoder be = EncoderFactory.get().directBinaryEncoder(out, null); gdw.setSchema(record.getSchema()); gdw.write(record, be); out.flush(); valueBytes = out.toByteArray(); } catch (IOException e) { Throwables.propagate(new SerDeException(e)); } return valueBytes; }
@Override public void write(Kryo kryo, Output output, GenericContainer record) { String fingerPrint = this.getFingerprint(record.getSchema()); output.writeString(fingerPrint); GenericDatumWriter<GenericContainer> writer = new GenericDatumWriter<>(record.getSchema()); BinaryEncoder encoder = EncoderFactory .get() .directBinaryEncoder(output, null); try { writer.write(record, encoder); } catch (IOException e) { throw new RuntimeException(e); } }
private AvroSchema(org.apache.avro.Schema schema, Map<String, String> properties) { this.schema = schema; this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); this.schemaInfo.setType(SchemaType.AVRO); this.schemaInfo.setSchema(this.schema.toString().getBytes()); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, this.encoder); this.datumWriter = new ReflectDatumWriter<>(this.schema); this.reader = new ReflectDatumReader<>(this.schema); }
/** * Writes given record using specified schema. * @param record record to serialize * @param schema schema to use for serialization * @return serialized record */ public static byte[] writeRecord(GenericRecord record, Schema schema) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(stream, null); new GenericDatumWriter<>(schema).write(record, encoder); encoder.flush(); return stream.toByteArray(); } }
private byte[] encodeGenericBlob(GenericRecord data) throws IOException { DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(data.getSchema()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); Encoder encoder = encoderType == EncoderType.BINARY ? EncoderFactory.get().binaryEncoder(outStream, null) : EncoderFactory.get().jsonEncoder(data.getSchema(), outStream); writer.write(data, encoder); encoder.flush(); outStream.close(); return outStream.toByteArray(); }
public static byte[] serializeRecord(FooBarSpecificRecord fooBarSpecificRecord) throws IOException { SpecificDatumWriter<FooBarSpecificRecord> datumWriter = new SpecificDatumWriter<>(FooBarSpecificRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(fooBarSpecificRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
private static <T> byte[] serializeWithReflectDatumWriter(T toSerialize, Class<T> toSerializeClass) throws IOException { ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter<>(toSerializeClass); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(toSerialize, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }