@Override public void flush() throws IOException { encoder.flush(); }
@Override public byte[] serialize(Row row) { try { // convert to record final GenericRecord record = convertRowToAvroRecord(schema, row); arrayOutputStream.reset(); datumWriter.write(record, encoder); encoder.flush(); return arrayOutputStream.toByteArray(); } catch (Exception e) { throw new RuntimeException("Failed to serialize row.", e); } }
protected static byte[] serializeAvroRecord(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length); serialized.put(out.toByteArray()); return serialized.array(); }
protected static byte[] serializeAvroRecord(SpecificDatumWriter<GenericRecord> writer, GenericRecord record) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); ByteBuffer serialized = ByteBuffer.allocate(out.toByteArray().length); serialized.put(out.toByteArray()); return serialized.array(); }
@Override public ByteBuffer write(List<Object> data, ByteBuffer buffer) { Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schemas"); try { Schema schema = schemas.getSchema(schemaString); GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < fieldNames.size(); i++) { record.put(fieldNames.get(i), data.get(i)); } ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema()); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(record, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); out.close(); return ByteBuffer.wrap(bytes); } catch (IOException e) { throw new RuntimeException(e); } } }
public static String jsonFromGenericRecord(GenericRecord record) { ByteArrayOutputStream out = new ByteArrayOutputStream(); GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(DefaultTopicSchema.MESSAGE_V0); try { Encoder encoder = new JsonEncoder(DefaultTopicSchema.MESSAGE_V0, out); writer.write(record, encoder); encoder.flush(); } catch (IOException e) { LOG.error("Unable to serialize avro record due to error " + e); } return out.toString(); }
public byte[] serialize(GenericRecord record) throws IOException { this.outputStream.reset(); this.writer.write(record, this.encoder); this.encoder.flush(); return this.outputStream.toByteArray(); } }
private static void writeBlock(Encoder vout, FileOutputStream out) throws IOException{ vout.writeLong(blockCount); bufOut.flush(); buffer.writeTo(out); buffer.reset(); blockCount = 0; }
@Override public final long writeTest() throws IOException { Encoder e = getEncoder(); long t = System.nanoTime(); writeInternal(e); e.flush(); return (System.nanoTime() - t); }
private byte[] encodeGenericBlob(GenericRecord data) throws IOException { DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(data.getSchema()); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); Encoder encoder = encoderType == EncoderType.BINARY ? EncoderFactory.get().binaryEncoder(outStream, null) : EncoderFactory.get().jsonEncoder(data.getSchema(), outStream); writer.write(data, encoder); encoder.flush(); outStream.close(); return outStream.toByteArray(); }
public static byte[] serializeRecord(StringablesRecord stringablesRecord) throws IOException { SpecificDatumWriter<StringablesRecord> datumWriter = new SpecificDatumWriter<>(StringablesRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(stringablesRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
public static byte[] serializeRecord(FooBarSpecificRecord fooBarSpecificRecord) throws IOException { SpecificDatumWriter<FooBarSpecificRecord> datumWriter = new SpecificDatumWriter<>(FooBarSpecificRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(fooBarSpecificRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
private static <T> byte[] render(T datum, Schema schema, DatumWriter<T> writer) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder enc = new EncoderFactory().directBinaryEncoder(out, null); writer.write(datum, enc); enc.flush(); return out.toByteArray(); } }
private static <T> byte[] serializeWithReflectDatumWriter(T toSerialize, Class<T> toSerializeClass) throws IOException { ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter<>(toSerializeClass); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(toSerialize, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
private <T> byte[] testJsonEncoder (String testType, T entityObj) throws IOException { ReflectData rdata = ReflectData.AllowNull.get(); Schema schema = rdata.getSchema(entityObj.getClass()); ByteArrayOutputStream os = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, os); ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter<>(schema, rdata); datumWriter.write(entityObj, encoder); encoder.flush(); byte[] bytes = os.toByteArray(); System.out.println ("JSON encoder output:\n" + new String(bytes)); return bytes; }
private void testWithBuffer(ByteBuffer buffer) throws IOException { assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); ByteArrayOutputStream output = new ByteArrayOutputStream(EXAMPLE_DATA_SIZE * 2); EncoderFactory encoderFactory = new EncoderFactory(); encoderFactory.configureBufferSize(ENCODER_BUFFER_SIZE); Encoder encoder = encoderFactory.binaryEncoder(output, null); new GenericDatumWriter<ByteBuffer>(Schema.create(Schema.Type.BYTES)).write(buffer, encoder); encoder.flush(); assertThat(output.toByteArray(), equalTo(avroEncoded(someBytes(EXAMPLE_DATA_SIZE)))); assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); // fails if buffer is not array-backed and buffer overflow occurs }
@Test public void test() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); EncoderFactory factory = EncoderFactory.get(); Encoder e = factory.validatingEncoder(schema, factory.binaryEncoder(baos, null)); ResolvingGrammarGenerator.encode(e, schema, data); e.flush(); }
@Test public void testJsonEncoderNewlineDelimited() throws IOException { OutputStream out = new ByteArrayOutputStream(); Schema ints = Schema.create(Type.INT); Encoder e = factory.jsonEncoder(ints, out); String separator = System.getProperty("line.separator"); GenericDatumWriter<Integer> writer = new GenericDatumWriter<>(ints); writer.write(1, e); writer.write(2, e); e.flush(); Assert.assertEquals("1"+separator+"2", out.toString()); }
@Override void init() throws IOException { genSourceData(); ByteArrayOutputStream baos = getOutputStream(); Encoder e = newEncoder(baos); writeInternal(e); e.flush(); data = baos.toByteArray(); encodedSize = data.length; //System.out.println(this.getClass().getSimpleName() + " encodedSize=" + encodedSize); }
@Test(expected = EOFException.class) public void testEOF() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder e = EncoderFactory.get().binaryEncoder(baos, null); e.writeLong(0x10000000000000L); e.flush(); Decoder d = newDecoder(new ByteArrayInputStream(baos.toByteArray())); Assert.assertEquals(0x10000000000000L, d.readLong()); d.readInt(); }