/** * Converts object to byte array * * @param avroObject the avro object * @return the byte[] result of conversion * @throws IOException Signals that an I/O exception has occurred. */ public byte[] toByteArray(T avroObject) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(baos, encoder); avroWriter.write(avroObject, encoder); encoder.flush(); return baos.toByteArray(); }
protected static byte[] serializeAvroRecord(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length); serialized.put(out.toByteArray()); return serialized.array(); }
/** * Encode record to Json and then convert to byte array. * * @param record the object to encode * @return the byte[] * @throws IOException Signals that an I/O exception has occurred. */ public byte[] encodeToJsonBytes(T record) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); jsonEncoder = EncoderFactory.get().jsonEncoder(this.schema, baos, true); datumWriter.write(record, jsonEncoder); jsonEncoder.flush(); baos.flush(); return baos.toByteArray(); }
@Override protected Encoder getEncoder(Schema schema, OutputStream outputStream) { return EncoderFactory.get().binaryEncoder(outputStream, null); } }
@Override protected Encoder getEncoder(Schema schema, OutputStream outputStream) throws IOException { return EncoderFactory.get().jsonEncoder(schema, outputStream); } }
public GenericAvroSchema(SchemaInfo schemaInfo) { super(schemaInfo); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, encoder); this.datumWriter = new GenericDatumWriter(schema); this.datumReader = new GenericDatumReader(schema); }
/** * Returns the Avro binary encoded version of <tt>n</tt> according to * the schema <tt>s</tt>. * @param s The schema for encoding * @param n The Json node that has the value to be encoded. * @return The binary encoded version of <tt>n</tt>. * @throws IOException */ private static byte[] getBinary(Schema s, JsonNode n) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder e = factory.binaryEncoder(out, null); encode(e, s, n); e.flush(); return out.toByteArray(); }
@Test public void testAvroEncodeIO() throws IOException { Schema schm = ReflectData.get().getSchema(AvroEncRecord.class); ReflectDatumWriter<AvroEncRecord> writer = new ReflectDatumWriter<>(schm); ByteArrayOutputStream out = new ByteArrayOutputStream(); AvroEncRecord record = new AvroEncRecord(); record.date = new java.util.Date(948833323L); writer.write(record, factory.directBinaryEncoder(out, null)); ReflectDatumReader<AvroEncRecord> reader = new ReflectDatumReader<>(schm); AvroEncRecord decoded = reader.read(new AvroEncRecord(), DecoderFactory.get().binaryDecoder( out.toByteArray(), null)); assertEquals(record, decoded); }
Producer<byte[], byte[]> producer = new Producer<>(producerConfig); try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(65536)) { for (File avroFile : avroFiles) { try (DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(avroFile)) { BinaryEncoder binaryEncoder = new EncoderFactory().directBinaryEncoder(outputStream, null); GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(reader.getSchema()); outputStream.reset(); if (header != null && 0 < header.length) { outputStream.write(header); datumWriter.write(genericRecord, binaryEncoder); binaryEncoder.flush();
public byte[] serialize(String topic, GenericRecord data) throws SerializationException { Schema schema = data.getSchema(); MD5Digest schemaId = null; try { schemaId = schemaRegistry.register(topic, schema); ByteArrayOutputStream out = new ByteArrayOutputStream(); // MAGIC_BYTE | schemaId-bytes | avro_payload out.write(LiAvroSerDeHelper.MAGIC_BYTE); out.write(schemaId.asBytes()); BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema); writer.write(data, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); out.close(); return bytes; } catch (IOException | SchemaRegistryException e) { throw new SerializationException(e); } }
/** * Encode record to byte array. * * @param record the object to encode * @return the byte[] * @throws IOException Signals that an I/O exception has occurred. */ public byte[] encode(T record) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); binaryEncoder = EncoderFactory.get().binaryEncoder(baos, binaryEncoder); datumWriter.write(record, binaryEncoder); binaryEncoder.flush(); baos.flush(); return baos.toByteArray(); }
private byte[] serializeEvent(Event event, boolean useAvroEventFormat) throws IOException { byte[] bytes; if (useAvroEventFormat) { if (!tempOutStream.isPresent()) { tempOutStream = Optional.of(new ByteArrayOutputStream()); } if (!writer.isPresent()) { writer = Optional.of(new SpecificDatumWriter<AvroFlumeEvent>(AvroFlumeEvent.class)); } tempOutStream.get().reset(); AvroFlumeEvent e = new AvroFlumeEvent(toCharSeqMap(event.getHeaders()), ByteBuffer.wrap(event.getBody())); encoder = EncoderFactory.get().directBinaryEncoder(tempOutStream.get(), encoder); writer.get().write(e, encoder); encoder.flush(); bytes = tempOutStream.get().toByteArray(); } else { bytes = event.getBody(); } return bytes; }
+ " not set and has no default value"); if (json.isNull() && (field.schema().getType() == Type.NULL || (field.schema().getType() == Type.UNION && field.schema().getTypes().get(0).getType() == Type.NULL))) { return null; ByteArrayOutputStream baos = new ByteArrayOutputStream(); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null); ResolvingGrammarGenerator.encode(encoder, field.schema(), json); encoder.flush(); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(baos.toByteArray(), null); defaultValue = createDatumReader(field.schema()).read(null, decoder);
private GenericRecord serializeWithWriterThenDeserializeWithReader(Schema writerSchema, GenericRecord datum, Schema readerSchema) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); DatumWriter<Object> datumWriter = new GenericDatumWriter<>(writerSchema); datumWriter.write(datum, encoder); encoder.flush(); byte[] bytes = baos.toByteArray(); Decoder decoder = DecoderFactory.get().resolvingDecoder( writerSchema, readerSchema, DecoderFactory.get().binaryDecoder(bytes, null)); DatumReader<Object> datumReader = new GenericDatumReader<>(readerSchema); return (GenericRecord)datumReader.read(null, decoder); }
private static void checkJson(Schema schema, Object datum, String json) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); DatumWriter<Object> writer = new GenericDatumWriter<>(); writer.setSchema(schema); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); String encoded = new String(data, StandardCharsets.UTF_8); assertEquals("Encoded data does not match.", json, encoded); DatumReader<Object> reader = new GenericDatumReader<>(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .jsonDecoder(schema, new ByteArrayInputStream(data))); assertEquals("Decoded data does not match.", datum, decoded); }
@Test public void testDeserialize() throws IOException { Schema writerSchema = Schema.create(Schema.Type.STRING); Schema readerSchema = Schema.create(Schema.Type.STRING); ClassLoader classLoader = this.getClass().getClassLoader(); AvroValueDeserializer<CharSequence> deserializer = DatumWriter<CharSequence> datumWriter = new GenericDatumWriter<>(writerSchema); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); datumWriter.write("record1", encoder); datumWriter.write("record2", encoder); encoder.flush(); ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); deserializer.open(inputStream); AvroWrapper<CharSequence> record = null;
public static void checkBlockingBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
@Override public ByteBuffer write(List<Object> data, ByteBuffer buffer) { Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schemas"); try { Schema schema = schemas.getSchema(schemaString); GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < fieldNames.size(); i++) { record.put(fieldNames.get(i), data.get(i)); } ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema()); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); out.close(); return ByteBuffer.wrap(bytes); } catch (IOException e) { throw new RuntimeException(e); } } }
private byte[] encodeGenericBlob(GenericRecord data) throws IOException { DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(data.getSchema()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); Encoder encoder = encoderType == EncoderType.BINARY ? EncoderFactory.get().binaryEncoder(outStream, null) : EncoderFactory.get().jsonEncoder(data.getSchema(), outStream); writer.write(data, encoder); encoder.flush(); outStream.close(); return outStream.toByteArray(); }
private byte[] serialize(Object datum, Schema datumSchema) throws FlumeException { if (schema == null || !datumSchema.equals(schema)) { schema = datumSchema; out = new ByteArrayOutputStream(); writer = new ReflectDatumWriter<>(schema); encoder = EncoderFactory.get().binaryEncoder(out, null); } out.reset(); try { writer.write(datum, encoder); encoder.flush(); return out.toByteArray(); } catch (IOException e) { throw new FlumeException(e); } }