/** Initialize the stream without reading from it. */ void initialize(InputStream in, Header header) throws IOException { this.header = header; this.codec = resolveCodec(); reader.setSchema(header.schema); }
/** Initialize the stream without reading from it. */ void initialize(InputStream in, Header header) throws IOException { this.header = header; this.codec = resolveCodec(); reader.setSchema(header.schema); }
public static void checkDirectBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(datum, encoder); // no flush for direct byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
header.schema = Schema.parse(getMetaString(DataFileConstants.SCHEMA),false); this.codec = resolveCodec(); reader.setSchema(header.schema);
public static void checkBlockingBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
header.schema = Schema.parse(getMetaString(DataFileConstants.SCHEMA),false); this.codec = resolveCodec(); reader.setSchema(header.schema);
private static void checkJson(Schema schema, Object datum, String json) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); DatumWriter<Object> writer = new GenericDatumWriter<>(); writer.setSchema(schema); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); String encoded = new String(data, StandardCharsets.UTF_8); assertEquals("Encoded data does not match.", json, encoded); DatumReader<Object> reader = new GenericDatumReader<>(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .jsonDecoder(schema, new ByteArrayInputStream(data))); assertEquals("Decoded data does not match.", datum, decoded); }
private static void checkJson(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); writer.setSchema(schema); writer.write(datum, encoder); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data)); Object decoded = reader.read(null, decoder); assertEquals("Decoded data does not match.", datum, decoded); decoded = reader.read(decoded, decoder); assertEquals("Decoded data does not match.", datum, decoded); }
public static Object checkBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader, Object reuse) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().binaryEncoder(out, null); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(reuse, DecoderFactory.get().binaryDecoder( data, null)); assertEquals("Decoded data does not match.", datum, decoded); return decoded; }
this.reader = reader; reader.setSchema(schema);
this.reader = reader; reader.setSchema(schema);
@SuppressWarnings("unchecked") private DatumReader<D> newDatumReader() { DatumReader<D> reader = (DatumReader<D>) getReader.apply(readSchema); reader.setSchema(fileSchema); return reader; } }
/** Initialize the stream without reading from it. */ void initialize(InputStream in, Header header) throws IOException { this.header = header; this.codec = resolveCodec(); reader.setSchema(header.schema); }
/** Initialize the stream without reading from it. */ void initialize(InputStream in, Header header) throws IOException { this.header = header; this.codec = resolveCodec(); reader.setSchema(header.schema); }
/** * Creates a new {@link MessageDecoder} that constructs datum instances described by the * {@link Schema readSchema}. * <p> * The {@code readSchema} is used for the expected schema and the {@code writeSchema} is the * schema used to decode buffers. The {@code writeSchema} must be the schema that was used to * encode all buffers decoded by this class. * * @param readSchema the schema used to construct datum instances * @param writeSchema the schema used to decode buffers */ private RawDecoder(com.netflix.iceberg.Schema readSchema, org.apache.avro.Schema writeSchema) { this.reader = new ProjectionDatumReader<>(DataReader::create, readSchema, ImmutableMap.of()); this.reader.setSchema(writeSchema); }
protected DatumReader<T> getDatumReader(Configuration config) { String schemaStr = config.get(AvroJob.INPUT_SCHEMA); DatumReader<T> datumReader = "specific".equals(config.get("avro.input.api")) ? new SpecificDatumReader<T>() : new GenericDatumReader<T>(); // set the expected schema to be what is configured // if not configured, the initialization of the file reader // will set it to the file contents if (schemaStr != null) { Schema schema = Schema.parse(schemaStr); datumReader.setSchema(schema); } return datumReader; }
/** * Returns a input stream given a file path * * @param path * @return InputStream * @throws IOException */ @Override protected InputStream openFile(Path path) throws IOException { InputStream is = super.openFile(path); if (is != null) { DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); avroDataStream = new DataFileStream<GenericRecord>(is, datumReader); datumReader.setSchema(avroDataStream.getSchema()); } return is; }
/** * Returns a input stream given a file path * * @param path * @return InputStream * @throws IOException */ @Override protected InputStream openFile(Path path) throws IOException { InputStream is = super.openFile(path); if (is != null) { DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); avroDataStream = new DataFileStream<GenericRecord>(is, datumReader); datumReader.setSchema(avroDataStream.getSchema()); } return is; }
void initialize(InputStream in) throws IOException { this.vin = DecoderFactory.defaultFactory().createBinaryDecoder(in, vin); byte[] magic = new byte[DataFileConstants.MAGIC.length]; try { vin.readFixed(magic); // read magic } catch (IOException e) { throw new IOException("Not a data file."); } if (!Arrays.equals(DataFileConstants.MAGIC, magic)) throw new IOException("Not a data file."); long l = vin.readMapStart(); // read meta data if (l > 0) { do { for (long i = 0; i < l; i++) { String key = vin.readString(null).toString(); ByteBuffer value = vin.readBytes(null); byte[] bb = new byte[value.remaining()]; value.get(bb); meta.put(key, bb); } } while ((l = vin.mapNext()) != 0); } vin.readFixed(sync); // read sync this.codec = resolveCodec(); this.schema = Schema.parse(getMetaString(DataFileConstants.SCHEMA)); reader.setSchema(schema); }
void initialize(InputStream in) throws IOException { this.vin = DecoderFactory.defaultFactory().createBinaryDecoder(in, vin); byte[] magic = new byte[DataFileConstants.MAGIC.length]; try { vin.readFixed(magic); // read magic } catch (IOException e) { throw new IOException("Not a data file."); } if (!Arrays.equals(DataFileConstants.MAGIC, magic)) throw new IOException("Not a data file."); long l = vin.readMapStart(); // read meta data if (l > 0) { do { for (long i = 0; i < l; i++) { String key = vin.readString(null).toString(); ByteBuffer value = vin.readBytes(null); byte[] bb = new byte[value.remaining()]; value.get(bb); meta.put(key, bb); } } while ((l = vin.mapNext()) != 0); } vin.readFixed(sync); // read sync this.codec = resolveCodec(); this.schema = Schema.parse(getMetaString(DataFileConstants.SCHEMA)); reader.setSchema(schema); }