private static <T> byte[] render(T datum, Schema schema, DatumWriter<T> writer) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder enc = new EncoderFactory().directBinaryEncoder(out, null); writer.write(datum, enc); enc.flush(); return out.toByteArray(); } }
@Override protected Encoder newEncoder(ByteArrayOutputStream out) throws IOException { return new EncoderFactory().configureBlockSize(254).blockingBinaryEncoder(out, null); }
private void testWithBuffer(ByteBuffer buffer) throws IOException { assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); ByteArrayOutputStream output = new ByteArrayOutputStream(EXAMPLE_DATA_SIZE * 2); EncoderFactory encoderFactory = new EncoderFactory(); encoderFactory.configureBufferSize(ENCODER_BUFFER_SIZE); Encoder encoder = encoderFactory.binaryEncoder(output, null); new GenericDatumWriter<ByteBuffer>(Schema.create(Schema.Type.BYTES)).write(buffer, encoder); encoder.flush(); assertThat(output.toByteArray(), equalTo(avroEncoded(someBytes(EXAMPLE_DATA_SIZE)))); assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); // fails if buffer is not array-backed and buffer overflow occurs }
private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
private void init(OutputStream outs) throws IOException { this.underlyingStream = outs; this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }
public Tests(int bufferSize, int depth, String input) throws IOException { this.depth = depth; byte[] in = input.getBytes("UTF-8"); JsonFactory f = new JsonFactory(); JsonParser p = f.createJsonParser( new ByteArrayInputStream(input.getBytes("UTF-8"))); ByteArrayOutputStream os = new ByteArrayOutputStream(); EncoderFactory factory = new EncoderFactory() .configureBlockSize(bufferSize); Encoder cos = factory.blockingBinaryEncoder(os, null); serialize(cos, p, os); cos.flush(); byte[] bb = os.toByteArray(); // dump(bb); this.input = DecoderFactory.get().binaryDecoder(bb, null); this.parser = f.createJsonParser(new ByteArrayInputStream(in)); }
public TestBlockingIO2 (int bufferSize, int skipLevel, String calls) throws IOException { ByteArrayOutputStream os = new ByteArrayOutputStream(); EncoderFactory factory = new EncoderFactory() .configureBlockSize(bufferSize); Encoder encoder = factory.blockingBinaryEncoder(os, null); this.values = TestValidatingIO.randomValues(calls); TestValidatingIO.generate(encoder, calls, values); encoder.flush(); byte[] bb = os.toByteArray(); decoder = DecoderFactory.get().binaryDecoder(bb, null); this.calls = calls; }
for (File avroFile : avroFiles) { try (DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(avroFile)) { BinaryEncoder binaryEncoder = new EncoderFactory().directBinaryEncoder(outputStream, null); GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(reader.getSchema());
BinaryEncoder binaryEncoder = new EncoderFactory().directBinaryEncoder(outputStream, null); Schema avroSchema = reader.getSchema(); GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(avroSchema);
private static void checkBinary(ReflectData reflectData, Schema schema, Object datum, boolean equals, boolean blocking) throws IOException { ReflectDatumWriter<Object> writer = new ReflectDatumWriter<>(schema); ByteArrayOutputStream out = new ByteArrayOutputStream(); if (!blocking) { writer.write(datum, EncoderFactory.get().directBinaryEncoder(out, null)); } else { writer.write(datum, new EncoderFactory().configureBlockSize(64) .blockingBinaryEncoder(out, null)); } writer.write(datum, EncoderFactory.get().directBinaryEncoder(out, null)); byte[] data = out.toByteArray(); ReflectDatumReader<Object> reader = new ReflectDatumReader<>(schema); Object decoded = reader.read(null, DecoderFactory.get().binaryDecoder(data, null)); assertEquals(0, reflectData.compare(datum, decoded, schema, equals)); }
public ColumnEncoder(OutputStream out, ColumnEncoder reuse) { this.out = out; wrappedEncoder = new EncoderFactory().binaryEncoder(out, reuse.wrappedEncoder); }
public ColumnEncoder(OutputStream out, ColumnEncoder reuse) { this.out = out; wrappedEncoder = new EncoderFactory().binaryEncoder(out, reuse.wrappedEncoder); }
public ColumnEncoder(OutputStream out) { this.out = out; wrappedEncoder = new EncoderFactory().binaryEncoder(out, null); }
public ColumnEncoder(OutputStream out) { this.out = out; wrappedEncoder = new EncoderFactory().binaryEncoder(out, null); }
public ColumnEncoder(OutputStream out, ColumnEncoder reuse) { this.out = out; wrappedEncoder = new EncoderFactory().binaryEncoder(out, reuse.wrappedEncoder); }
public ColumnEncoder(OutputStream out) { this.out = out; wrappedEncoder = new EncoderFactory().binaryEncoder(out, null); }
@Override public void open(OutputStream out) throws IOException { this.out = out; this.encoder = new EncoderFactory().configureBlockSize(512).binaryEncoder(out, null); }
@Override public void open(OutputStream out) throws IOException { this.out = out; this.encoder = new EncoderFactory().configureBlockSize(512).binaryEncoder(out, null); }
private byte[] serialize(Map<String, Object> data) throws IOException { data = (data == null) ? Collections.emptyMap() : data; BulletAvro record = new BulletAvro(data); ByteArrayOutputStream stream = new ByteArrayOutputStream(2048); EncoderFactory encoderFactory = new EncoderFactory(); Encoder encoder = encoderFactory.binaryEncoder(stream, null); WRITER.write(record, encoder); encoder.flush(); return stream.toByteArray(); }
private void init(OutputStream outs) throws IOException { this.out = new BufferedFileOutputStream(outs); EncoderFactory efactory = new EncoderFactory(); this.vout = efactory.binaryEncoder(out, null); dout.setSchema(schema); buffer = new NonCopyingByteArrayOutputStream( Math.min((int)(syncInterval * 1.25), Integer.MAX_VALUE/2 -1)); this.bufOut = efactory.binaryEncoder(buffer, null); if (this.codec == null) { this.codec = CodecFactory.nullCodec().createInstance(); } this.isOpen = true; }