@Override protected Decoder getDecoder(byte[] payload) { return DecoderFactory.get().binaryDecoder(payload, null); } }
@Override protected BinaryDecoder initialValue() { InputStream dummyInputStream = new ByteArrayInputStream(new byte[0]); return DecoderFactory.get().binaryDecoder(dummyInputStream, null); } };
public void readFields(byte[] bytes, int offset, int length, Schema writerSchema, Schema readerSchema) throws IOException { fileSchema = writerSchema; record = new GenericData.Record(writerSchema); binaryDecoder = DecoderFactory.get().binaryDecoder(bytes, offset, length - offset, binaryDecoder); GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(writerSchema, readerSchema); record = gdr.read(null, binaryDecoder); }
@Override public T decode(byte[] bytes) { try { BinaryDecoder decoderFromCache = decoders.get(); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, decoderFromCache); if (decoderFromCache == null) { decoders.set(decoder); } return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, decoder)); } catch (IOException e) { throw new SchemaSerializationException(e); } }
@Override public GenericRecord parse(ByteBuffer bytes) { try { bytes.get(); // ignore first \0 byte int id = bytes.getInt(); // extract schema registry id int length = bytes.limit() - 1 - 4; int offset = bytes.position() + bytes.arrayOffset(); Schema schema = registry.getByID(id); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); return reader.read(null, DecoderFactory.get().binaryDecoder(bytes.array(), offset, length, null)); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } }
/** * Deserialize payload using payload schema */ public GenericRecord deserializePayload(byte[] payload, Schema payloadSchema) throws IOException, ExecutionException { Decoder decoder = this.decoderFactory.binaryDecoder(payload, null); GenericDatumReader<GenericRecord> reader = this.readers.get(payloadSchema); return reader.read(null, decoder); } }
public AvroReaderWithExplicitSchema(final InputStream in, final RecordSchema recordSchema, final Schema avroSchema) { this.in = in; this.recordSchema = recordSchema; datumReader = new NonCachingDatumReader<>(avroSchema); decoder = DecoderFactory.get().binaryDecoder(in, null); }
@Override public AvroJobSpec deserialize(String topic, byte[] data) { try (InputStream is = new ByteArrayInputStream(data)) { _versionWriter.readSchemaVersioningInformation(new DataInputStream(is)); Decoder decoder = DecoderFactory.get().binaryDecoder(is, _decoder); return _reader.read(null, decoder); } catch (IOException e) { throw new RuntimeException("Could not decode message"); } }
@Override public List<Object> deserialize(ByteBuffer ser) { try { Schema schema = schemas.getSchema(schemaString); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(Utils.toByteArray(ser), null); GenericRecord record = reader.read(null, decoder); ArrayList<Object> list = new ArrayList<>(fieldNames.size()); for (String field : fieldNames) { Object value = record.get(field); // Avro strings are stored using a special Avro Utf8 type instead of using Java primitives list.add(SerdeUtils.convertAvroUtf8(value)); } return list; } catch (IOException e) { throw new RuntimeException(e); } }
@SuppressWarnings("unchecked") private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { recordClazz = (Class<? extends SpecificRecord>) inputStream.readObject(); schemaString = inputStream.readUTF(); typeInfo = (RowTypeInfo) AvroSchemaConverter.<Row>convertToTypeInfo(schemaString); schema = new Schema.Parser().parse(schemaString); if (recordClazz != null) { record = (SpecificRecord) SpecificData.newInstance(recordClazz, schema); } else { record = new GenericData.Record(schema); } datumReader = new SpecificDatumReader<>(schema); this.inputStream = new MutableByteArrayInputStream(); decoder = DecoderFactory.get().binaryDecoder(this.inputStream, null); } }
@Override public void configure(Map<String, ?> configs, boolean isKey) { InputStream dummyInputStream = new ByteArrayInputStream(new byte[0]); _decoder = DecoderFactory.get().binaryDecoder(dummyInputStream, null); _reader = new SpecificDatumReader<AvroJobSpec>(AvroJobSpec.SCHEMA$); _versionWriter = new FixedSchemaVersionWriter(); }
void checkAvroInitialized() { if (datumReader != null) { return; } ClassLoader cl = Thread.currentThread().getContextClassLoader(); if (SpecificRecord.class.isAssignableFrom(recordClazz)) { SpecificData specificData = new SpecificData(cl); this.datumReader = new SpecificDatumReader<>(specificData); this.reader = specificData.getSchema(recordClazz); } else { this.reader = new Schema.Parser().parse(schemaString); GenericData genericData = new GenericData(cl); this.datumReader = new GenericDatumReader<>(null, this.reader, genericData); } this.inputStream = new MutableByteArrayInputStream(); this.decoder = DecoderFactory.get().binaryDecoder(inputStream, null); }
@Override public GenericRecord parse(ByteBuffer bytes) { try (ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes))) { return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null)); } catch (EOFException eof) { // waiting for avro v1.9.0 (#AVRO-813) throw new ParseException( eof, "Avro's unnecessary EOFException, detail: [%s]", "https://issues.apache.org/jira/browse/AVRO-813" ); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } } }
/** * Creates a Avro deserialization schema for the given Avro schema string. * * @param avroSchemaString Avro schema string to deserialize Avro's record to Flink's row */ public AvroRowDeserializationSchema(String avroSchemaString) { Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); recordClazz = null; final TypeInformation<?> typeInfo = AvroSchemaConverter.convertToTypeInfo(avroSchemaString); Preconditions.checkArgument(typeInfo instanceof RowTypeInfo, "Row type information expected."); this.typeInfo = (RowTypeInfo) typeInfo; schemaString = avroSchemaString; schema = new Schema.Parser().parse(avroSchemaString); record = new GenericData.Record(schema); datumReader = new GenericDatumReader<>(schema); inputStream = new MutableByteArrayInputStream(); decoder = DecoderFactory.get().binaryDecoder(inputStream, null); }
private AvroJobSpec decodeRecord(ByteArrayBasedKafkaRecord kafkaConsumerRecord) throws IOException { InputStream is = new ByteArrayInputStream(kafkaConsumerRecord.getMessageBytes()); _versionWriter.readSchemaVersioningInformation(new DataInputStream(is)); Decoder decoder = DecoderFactory.get().binaryDecoder(is, _decoder); return _reader.read(null, decoder); }
@Override public GenericRecord decode(byte[] bytes) { try { Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); org.apache.avro.generic.GenericRecord avroRecord = datumReader.read( null, decoder); return new GenericAvroRecord(schema, fields, avroRecord); } catch (IOException e) { throw new SchemaSerializationException(e); } }
@Override public GenericRecord parse(ByteBuffer bytes) { Pair<SUBJECT, ID> subjectAndId = subjectAndIdConverter.getSubjectAndId(bytes); Schema schema = typedRepository.getSchema(subjectAndId.lhs, subjectAndId.rhs); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); try (ByteBufferInputStream inputStream = new ByteBufferInputStream(Collections.singletonList(bytes))) { return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null)); } catch (EOFException eof) { // waiting for avro v1.9.0 (#AVRO-813) throw new ParseException( eof, "Avro's unnecessary EOFException, detail: [%s]", "https://issues.apache.org/jira/browse/AVRO-813" ); } catch (IOException e) { throw new ParseException(e, "Fail to decode avro message!"); } }
@Override public AvroGenericRecordWritable getWritable(byte[] value) { GenericRecord avroRecord = null; try { avroRecord = dataReader.read(null, DecoderFactory.get().binaryDecoder(value, null)); } catch (IOException e) { Throwables.propagate(new SerDeException(e)); } avroGenericRecordWritable.setRecord(avroRecord); avroGenericRecordWritable.setRecordReaderID(uid); avroGenericRecordWritable.setFileSchema(avroRecord.getSchema()); return avroGenericRecordWritable; } }
@Override public GenericRow decode(byte[] payload, int offset, int length, GenericRow destination) { try { GenericData.Record avroRecord = _reader.read(null, _decoderFactory.binaryDecoder(payload, offset, length, null)); return _rowGenerator.transform(avroRecord, destination); } catch (Exception e) { LOGGER.error("Caught exception", e); throw new RuntimeException(e); } } }
/** * Creates a Avro deserialization schema for the given specific record class. Having the * concrete Avro record class might improve performance. * * @param recordClazz Avro record class used to deserialize Avro's record to Flink's row */ public AvroRowDeserializationSchema(Class<? extends SpecificRecord> recordClazz) { Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); this.recordClazz = recordClazz; schema = SpecificData.get().getSchema(recordClazz); typeInfo = (RowTypeInfo) AvroSchemaConverter.convertToTypeInfo(recordClazz); schemaString = schema.toString(); record = (IndexedRecord) SpecificData.newInstance(recordClazz, schema); datumReader = new SpecificDatumReader<>(schema); inputStream = new MutableByteArrayInputStream(); decoder = DecoderFactory.get().binaryDecoder(inputStream, null); }