/** * Decode json data. * * @param data the data * @param reuse the reuse * @return the decoded object * @throws IOException Signals that an I/O exception has occurred. */ public T decodeJson(String data, T reuse) throws IOException { jsonDecoder = DecoderFactory.get().jsonDecoder(this.schema, data, true); return datumReader.read(null, jsonDecoder); }
/** * Decode json data. * * @param data the data * @param reuse the reuse * @return the decoded object * @throws IOException Signals that an I/O exception has occurred. */ public T decodeJson(String data, T reuse) throws IOException { jsonDecoder = DecoderFactory.get().jsonDecoder(this.schema, data, true); return avroReader.read(null, jsonDecoder); }
/** * Converts a String JSON object into a generic datum. * * This is inefficient (creates extra objects), so should be used * sparingly. */ static Object jsonToGenericDatum(Schema schema, String jsonData) throws IOException { GenericDatumReader<Object> reader = new GenericDatumReader<>(schema); Object datum = reader.read(null, DecoderFactory.get().jsonDecoder(schema, jsonData)); return datum; }
/** * Reads all records from a json file as {@link GenericRecord}s */ public static List<GenericRecord> readAllRecords(String jsonDataPath, String schemaPath) throws Exception { List<GenericRecord> records = new ArrayList<>(); File jsonDataFile = new File(jsonDataPath); File schemaFile = new File(schemaPath); Schema schema = new Schema.Parser().parse(schemaFile); GenericDatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema); try (InputStream is = new FileInputStream(jsonDataFile)) { Decoder decoder = DecoderFactory.get().jsonDecoder(schema, is); while (true) { records.add(datumReader.read(null, decoder)); } } catch (EOFException eof) { // read all records } return records; } }
Decoder decoder = DecoderFactory.get().jsonDecoder(MetricReport.SCHEMA$, inputStream); return READER.get().read(reuse, decoder); } catch (Throwable t) {
/** * Parses a {@link org.apache.gobblin.metrics.MetricReport} from a byte array representing a json input. * @param reuse MetricReport to reuse. * @param bytes Input bytes. * @return MetricReport. * @throws java.io.IOException */ public synchronized static GobblinTrackingEvent deserializeReportFromJson(GobblinTrackingEvent reuse, byte[] bytes) throws IOException { if (!reader.isPresent()) { reader = Optional.of(new SpecificDatumReader<>(GobblinTrackingEvent.class)); } Closer closer = Closer.create(); try { DataInputStream inputStream = closer.register(new DataInputStream(new ByteArrayInputStream(bytes))); // Check version byte int versionNumber = inputStream.readInt(); if (versionNumber != SCHEMA_VERSION) { throw new IOException(String .format("MetricReport schema version not recognized. Found version %d, expected %d.", versionNumber, SCHEMA_VERSION)); } // Decode the rest Decoder decoder = DecoderFactory.get().jsonDecoder(GobblinTrackingEvent.SCHEMA$, inputStream); return reader.get().read(reuse, decoder); } catch(Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
private List<Object> fromJson(Schema schema, File file) throws Exception { InputStream in = new FileInputStream(file); List<Object> data = new ArrayList<>(); try { DatumReader reader = new GenericDatumReader(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, in); while (true) data.add(reader.read(null, decoder)); } catch (EOFException e) { } finally { in.close(); } return data; }
@Test(expected=AvroTypeException.class) public void testJsonExcessFields() throws IOException { String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a0\": 45, \"a2\":true, \"a1\": null}}"; Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + "]}"); GenericDatumReader<Object> reader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value); reader.read(null, decoder); }
@Test public void testJsonRecordOrdering() throws IOException { String value = "{\"b\": 2, \"a\": 1}"; Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": \"int\"}" + "]}"); GenericDatumReader<Object> reader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value); Object o = reader.read(null, decoder); Assert.assertEquals("{\"a\": 1, \"b\": 2}", o.toString()); }
private void checkNumeric(String type, Object value) throws Exception { String def = "{\"type\":\"record\",\"name\":\"X\",\"fields\":" +"[{\"type\":\""+type+"\",\"name\":\"n\"}]}"; Schema schema = Schema.parse(def); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); String[] records = {"{\"n\":1}", "{\"n\":1.0}"}; for (String record : records) { Decoder decoder = DecoderFactory.get().jsonDecoder(schema, record); GenericRecord r = reader.read(null, decoder); Assert.assertEquals(value, r.get("n")); } }
private <T> GenericRecord testJsonDecoder (String testType, byte[] bytes, T entityObj) throws IOException { ReflectData rdata = ReflectData.AllowNull.get(); Schema schema = rdata.getSchema(entityObj.getClass()); GenericDatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new String(bytes)); GenericRecord r = datumReader.read(null, decoder); return r; }
@Test public void testJsonRecordOrdering2() throws IOException { String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true, \"a1\": null}}"; Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + "]}"); GenericDatumReader<Object> reader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value); Object o = reader.read(null, decoder); Assert.assertEquals("{\"a\": {\"a1\": null, \"a2\": true}, \"b\": {\"b1\": \"h\", \"b2\": 3.14, \"b3\": 1.4}}", o.toString()); }
@Test public void testJsonRecordOrderingWithProjection() throws IOException { String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true, \"a1\": null}}"; Schema writerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + "]}"); Schema readerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" + "]}"); GenericDatumReader<Object> reader = new GenericDatumReader<>(writerSchema, readerSchema); Decoder decoder = DecoderFactory.get().jsonDecoder(writerSchema, value); Object o = reader.read(null, decoder); Assert.assertEquals("{\"a\": {\"a1\": null, \"a2\": true}}", o.toString()); }
@Test public void testJsonRecordOrderingWithProjection2() throws IOException { String value = "{\"b\": { \"b1\": \"h\", \"b2\": [3.14, 3.56], \"b3\": 1.4}, \"a\": {\"a2\":true, \"a1\": null}}"; Schema writerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":{\"type\":\"array\", \"items\":\"float\"}}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + "]}"); Schema readerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" + "]}"); GenericDatumReader<Object> reader = new GenericDatumReader<>(writerSchema, readerSchema); Decoder decoder = DecoderFactory.get().jsonDecoder(writerSchema, value); Object o = reader.read(null, decoder); Assert.assertEquals("{\"a\": {\"a1\": null, \"a2\": true}}", o.toString()); }
private Record decodeGenericBlob(Schema expectedSchema, Schema schemaOfBlob, byte[] blob) throws IOException { if (blob == null) { return null; } GenericDatumReader<Record> reader = new GenericDatumReader<>(); reader.setExpected(expectedSchema); reader.setSchema(schemaOfBlob); Decoder decoder = encoderType == EncoderType.BINARY ? DecoderFactory.get().binaryDecoder(blob, null) : DecoderFactory.get().jsonDecoder(schemaOfBlob, new ByteArrayInputStream(blob)); return reader.read(null, decoder); } }
private static void checkJson(Schema schema, Object datum, String json) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); DatumWriter<Object> writer = new GenericDatumWriter<>(); writer.setSchema(schema); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); String encoded = new String(data, StandardCharsets.UTF_8); assertEquals("Encoded data does not match.", json, encoded); DatumReader<Object> reader = new GenericDatumReader<>(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .jsonDecoder(schema, new ByteArrayInputStream(data))); assertEquals("Decoded data does not match.", datum, decoded); }
private static void checkJson(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); writer.setSchema(schema); writer.write(datum, encoder); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data)); Object decoded = reader.read(null, decoder); assertEquals("Decoded data does not match.", datum, decoded); decoded = reader.read(decoded, decoder); assertEquals("Decoded data does not match.", datum, decoded); }
@Test public void testWrite() throws IOException { String json = "{\"type\": \"record\", \"name\": \"r\", \"fields\": [" + "{ \"name\": \"f1\", \"type\": \"long\" }" + "]}"; Schema s = Schema.parse(json); GenericRecord r = new GenericData.Record(s); r.put("f1", 100L); ByteArrayOutputStream bao = new ByteArrayOutputStream(); GenericDatumWriter<GenericRecord> w = new GenericDatumWriter<>(s); Encoder e = EncoderFactory.get().jsonEncoder(s, bao); w.write(r, e); e.flush(); Object o = new GenericDatumReader<GenericRecord>(s).read(null, DecoderFactory.get().jsonDecoder(s, new ByteArrayInputStream(bao.toByteArray()))); assertEquals(r, o); }
@Test public void testSingleSubRecord() throws IOException { final Schema child = SchemaBuilder.record("Child") .namespace("org.apache.avro.nested") .fields() .requiredString("childField").endRecord(); final Schema parent = SchemaBuilder.record("Parent") .namespace("org.apache.avro.nested") .fields() .requiredString("parentField1") .name("child1").type(child).noDefault() .requiredString("parentField2").endRecord(); final String inputAsExpected = "{\n" + " \"parentField1\": \"parentValue1\",\n" + " \"child1\":{\n" + " \"childField\":\"childValue1\"\n" + " },\n" + " \"parentField2\":\"parentValue2\"\n" + "}"; final ByteArrayInputStream inputStream = new ByteArrayInputStream(inputAsExpected.getBytes()); final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(parent, inputStream); final DatumReader<Object> reader = new GenericDatumReader<Object>(parent); final GenericData.Record decoded = (GenericData.Record) reader.read(null, decoder); assertThat(decoded.get("parentField1").toString(), equalTo("parentValue1")); assertThat(decoded.get("parentField2").toString(), equalTo("parentValue2")); assertThat(((GenericData.Record)decoded.get("child1")).get("childField").toString(), equalTo("childValue1")); }
@Test public void testReorderFields() throws Exception { String w = "{\"type\":\"record\",\"name\":\"R\",\"fields\":" +"[{\"type\":\"long\",\"name\":\"l\"}," +"{\"type\":{\"type\":\"array\",\"items\":\"int\"},\"name\":\"a\"}" +"]}"; Schema ws = Schema.parse(w); DecoderFactory df = DecoderFactory.get(); String data = "{\"a\":[1,2],\"l\":100}{\"l\": 200, \"a\":[1,2]}"; JsonDecoder in = df.jsonDecoder(ws, data); Assert.assertEquals(100, in.readLong()); in.skipArray(); Assert.assertEquals(200, in.readLong()); in.skipArray(); } }