@Override public DatumReader createDatumReader(Schema schema) { return new ReflectDatumReader(schema, schema, this); }
@Override public DatumReader createDatumReader(Schema writer, Schema reader) { return new ReflectDatumReader(writer, reader, this); }
@Override public DatumReader createDatumReader(Schema schema) { return new ReflectDatumReader(schema, schema, this); }
@Override public DatumReader createDatumReader(Schema writer, Schema reader) { return new ReflectDatumReader(writer, reader, this); }
@InterfaceAudience.Private @Override public DatumReader getReader(Class<Object> clazz) { try { return new ReflectDatumReader(clazz); } catch (Exception e) { throw new RuntimeException(e); } }
@Override protected DatumReader<Object> getDatumReader(Schema actual, Schema expected) { return new ReflectDatumReader<>(actual, expected, getReflectData()); }
@Override protected DatumReader<Object> getDatumReader(Schema writer, Schema reader) { return new ReflectDatumReader<>(writer, reader, getReflectData()); }
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static <T> AvroFactory<T> fromReflective(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) { ReflectData reflectData = new ReflectData(cl); Schema newSchema = reflectData.getSchema(type); return new AvroFactory<>( reflectData, newSchema, new ReflectDatumReader<>(previousSchema.orElse(newSchema), newSchema, reflectData), new ReflectDatumWriter<>(newSchema, reflectData) ); }
/** * Constructor. * * @param writerSchema The Avro writer schema for the data to deserialize. * @param readerSchema The Avro reader schema for the data to deserialize (may be null). */ protected AvroDeserializer(Schema writerSchema, Schema readerSchema, ClassLoader classLoader) { mWriterSchema = writerSchema; mReaderSchema = null != readerSchema ? readerSchema : writerSchema; mAvroDatumReader = new ReflectDatumReader<>(mWriterSchema, mReaderSchema, new ReflectData(classLoader)); }
@Override public Object deserialise(final byte[] allBytes, final int offset, final int length) throws SerialisationException { final DatumReader<Object> datumReader = new ReflectDatumReader<>(); try (final InputStream inputStream = new ByteArrayInputStream(allBytes, offset, length); final DataFileStream<Object> in = new DataFileStream<>(inputStream, datumReader)) { return in.next(); } catch (final IOException e) { throw new SerialisationException("Unable to deserialise object, failed to read input bytes", e); } }
private AvroSchema(org.apache.avro.Schema schema, Map<String, String> properties) { this.schema = schema; this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); this.schemaInfo.setType(SchemaType.AVRO); this.schemaInfo.setSchema(this.schema.toString().getBytes()); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, this.encoder); this.datumWriter = new ReflectDatumWriter<>(this.schema); this.reader = new ReflectDatumReader<>(this.schema); }
private DataFileReader<E> initReader(FileInputSplit split) throws IOException { DatumReader<E> datumReader; if (org.apache.avro.generic.GenericRecord.class == avroValueType) { datumReader = new GenericDatumReader<E>(); } else { datumReader = org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType) ? new SpecificDatumReader<E>(avroValueType) : new ReflectDatumReader<E>(avroValueType); } if (LOG.isInfoEnabled()) { LOG.info("Opening split {}", split); } SeekableInput in = new FSDataInputStreamWrapper(stream, split.getPath().getFileSystem().getFileStatus(split.getPath()).getLen()); DataFileReader<E> dataFileReader = (DataFileReader) DataFileReader.openReader(in, datumReader); if (LOG.isDebugEnabled()) { LOG.debug("Loaded SCHEMA: {}", dataFileReader.getSchema()); } end = split.getStart() + split.getLength(); recordsReadSinceLastSync = 0; return dataFileReader; }
public T toObject(byte[] bytes) { Decoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null); ReflectDatumReader<T> reader = null; try { reader = new ReflectDatumReader<T>(clazz); return reader.read(null, decoder); } catch(IOException e) { throw new SerializationException(e); } } }
private void validateCountsFile(File file) throws Exception { DatumReader<WordCount> reader = new ReflectDatumReader<>(); InputStream in = new BufferedInputStream(new FileInputStream(file)); DataFileStream<WordCount> counts = new DataFileStream<>(in, reader); int numWords = 0; for (WordCount wc : counts) { assertEquals(wc.word, WordCountUtil.COUNTS.get(wc.word), (Long)wc.count); numWords++; } in.close(); assertEquals(WordCountUtil.COUNTS.size(), numWords); }
@SuppressWarnings("unchecked") ReflectTest(String name, T sample, int factor) throws IOException { super(name, ReflectData.get().getSchema(sample.getClass()).toString(), factor); clazz = (Class<T>) sample.getClass(); reader = new ReflectDatumReader<>(schema); writer = new ReflectDatumWriter<>(schema); }
private X readOneXFromAvro(Schema schema, ByteArrayOutputStream bout) throws IOException { SeekableByteArrayInput input = new SeekableByteArrayInput(bout.toByteArray()); ReflectDatumReader<X> datumReader = new ReflectDatumReader<>(schema); FileReader<X> reader = DataFileReader.openReader(input, datumReader); Iterator<X> it = reader.iterator(); assertTrue("missing first record",it.hasNext()); X record = it.next(); assertFalse("should be no more records - only wrote one out",it.hasNext()); return record; }
private void validateCompleteFile(File file) throws Exception { DatumReader<CompleteRecord> reader = new ReflectDatumReader<>(); int numRecs = 0; try(InputStream in = new BufferedInputStream(new FileInputStream(file))) { try (DataFileStream<CompleteRecord> records = new DataFileStream<>(in, reader)) { for (CompleteRecord rec : records) { assertEquals(rec.id, numRecs); assertEquals(rec.balance - 100, rec.id); assertEquals(rec.name, "record" + rec.id); numRecs++; } } } assertEquals(5, numRecs); }
@Test public void testAvroEncodeIO() throws IOException { Schema schm = ReflectData.get().getSchema(AvroEncRecord.class); ReflectDatumWriter<AvroEncRecord> writer = new ReflectDatumWriter<>(schm); ByteArrayOutputStream out = new ByteArrayOutputStream(); AvroEncRecord record = new AvroEncRecord(); record.date = new java.util.Date(948833323L); writer.write(record, factory.directBinaryEncoder(out, null)); ReflectDatumReader<AvroEncRecord> reader = new ReflectDatumReader<>(schm); AvroEncRecord decoded = reader.read(new AvroEncRecord(), DecoderFactory.get().binaryDecoder( out.toByteArray(), null)); assertEquals(record, decoded); }
@Test public void testRead_PojoWithList() throws IOException { PojoWithList pojoWithList = new PojoWithList(); pojoWithList.setId(42); pojoWithList.setRelatedIds(Arrays.asList(1, 2, 3)); byte[] serializedBytes = serializeWithReflectDatumWriter(pojoWithList, PojoWithList.class); Decoder decoder = DecoderFactory.get().binaryDecoder(serializedBytes, null); ReflectDatumReader<PojoWithList> reflectDatumReader = new ReflectDatumReader<>( PojoWithList.class); PojoWithList deserialized = new PojoWithList(); reflectDatumReader.read(deserialized, decoder); assertEquals(pojoWithList, deserialized); }
@Test public void testRead_PojoWithArray() throws IOException { PojoWithArray pojoWithArray = new PojoWithArray(); pojoWithArray.setId(42); pojoWithArray.setRelatedIds(new int[] { 1, 2, 3 }); byte[] serializedBytes = serializeWithReflectDatumWriter(pojoWithArray, PojoWithArray.class); Decoder decoder = DecoderFactory.get().binaryDecoder(serializedBytes, null); ReflectDatumReader<PojoWithArray> reflectDatumReader = new ReflectDatumReader<>( PojoWithArray.class); PojoWithArray deserialized = new PojoWithArray(); reflectDatumReader.read(deserialized, decoder); assertEquals(pojoWithArray, deserialized); }