Refine search
/** * Converts a String JSON object into a generic datum. * * This is inefficient (creates extra objects), so should be used * sparingly. */ static Object jsonToGenericDatum(Schema schema, String jsonData) throws IOException { GenericDatumReader<Object> reader = new GenericDatumReader<>(schema); Object datum = reader.read(null, DecoderFactory.get().jsonDecoder(schema, jsonData)); return datum; }
private Record decodeGenericBlob(Schema expectedSchema, Schema schemaOfBlob, byte[] blob) throws IOException { if (blob == null) { return null; } GenericDatumReader<Record> reader = new GenericDatumReader<>(); reader.setExpected(expectedSchema); reader.setSchema(schemaOfBlob); Decoder decoder = encoderType == EncoderType.BINARY ? DecoderFactory.get().binaryDecoder(blob, null) : DecoderFactory.get().jsonDecoder(schemaOfBlob, new ByteArrayInputStream(blob)); return reader.read(null, decoder); } }
@Override public Object readResponse(Schema writer, Schema reader, Decoder in) throws IOException { return new GenericDatumReader<>(writer, reader, data).read(null, in); }
public void readFields(byte[] bytes, int offset, int length, Schema writerSchema, Schema readerSchema) throws IOException { fileSchema = writerSchema; record = new GenericData.Record(writerSchema); binaryDecoder = DecoderFactory.get().binaryDecoder(bytes, offset, length - offset, binaryDecoder); GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(writerSchema, readerSchema); record = gdr.read(null, binaryDecoder); }
@Test(expected=AvroTypeException.class) public void testNoDefaultField() throws Exception { Schema expected = Schema.parse("{\"type\":\"record\", \"name\":\"Foo\", \"fields\":"+ "[{\"name\":\"f\", \"type\": \"string\"}]}"); DatumReader<Object> in = new GenericDatumReader<>(ACTUAL, expected); in.read(null, DecoderFactory.get().binaryDecoder( new ByteArrayInputStream(new byte[0]), null)); }
@Override public GenericRecord parse(ByteBuffer bytes) { try { bytes.get(); // ignore first \0 byte int id = bytes.getInt(); // extract schema registry id int length = bytes.limit() - 1 - 4; int offset = bytes.position() + bytes.arrayOffset(); Schema schema = registry.getByID(id); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); return reader.read(null, DecoderFactory.get().binaryDecoder(bytes.array(), offset, length, null)); } catch (Exception e) { throw new ParseException(e, "Fail to decode avro message!"); } }
throws Exception { Schema inputSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/envelope.avsc")); GenericDatumReader<GenericRecord> datumReader = new GenericDatumReader<>(inputSchema); when(mockRegistry.getSchemaByKey(any())).thenReturn(inputSchema.getField("nestedRecord").schema()); Assert.assertTrue(outputSchema.equals(latestPayloadSchema)); Schema expectedPayloadSchema = expectedPayload.getSchema(); Assert.assertTrue(expectedPayloadSchema.getName().equals(payloadSchema.getName())); Assert.assertTrue(expectedPayloadSchema.getNamespace().equals(payloadSchema.getNamespace())); Assert.assertTrue(expectedPayloadSchema.getFields().size() == payloadSchema.getFields().size());
protected Object readWithoutConversion(Object old, Schema expected, ResolvingDecoder in) throws IOException { switch (expected.getType()) { case RECORD: return readRecord(old, expected, in); case ENUM: return readEnum(expected, in); case ARRAY: return readArray(old, expected, in); case MAP: return readMap(old, expected, in); case UNION: return read(old, expected.getTypes().get(in.readIndex()), in); case FIXED: return readFixed(old, expected, in); case STRING: return readString(old, expected, in); case BYTES: return readBytes(old, expected, in); case INT: return readInt(old, expected, in); case LONG: return in.readLong(); case FLOAT: return in.readFloat(); case DOUBLE: return in.readDouble(); case BOOLEAN: return in.readBoolean(); case NULL: in.readNull(); return null; default: throw new AvroRuntimeException("Unknown type: " + expected); } }
public static Schema extractSchemaFromAvroWithoutTime(File avroFile) throws IOException { DataFileStream<GenericRecord> dataStream = new DataFileStream<GenericRecord>(new FileInputStream(avroFile), new GenericDatumReader<GenericRecord>()); Schema schema = new Schema(); for (final Field field : dataStream.getSchema().getFields()) { try { getColumnType(field); } catch (Exception e) { LOGGER.warn("Caught exception while converting Avro field {} of type {}, field will not be in schema.", field.name(), field.schema().getType()); continue; } final String columnName = field.name(); final String pinotType = field.getProp("pinotType"); final FieldSpec fieldSpec; if (pinotType != null && "METRIC".equals(pinotType)) { fieldSpec = new MetricFieldSpec(); } else { fieldSpec = new DimensionFieldSpec(); } fieldSpec.setName(columnName); fieldSpec.setDataType(getColumnType(dataStream.getSchema().getField(columnName))); fieldSpec.setSingleValueField(isSingleValueField(dataStream.getSchema().getField(columnName))); schema.addField(fieldSpec); } dataStream.close(); return schema; }
@Override public Exception readError(Schema writer, Schema reader, Decoder in) throws IOException { Object error = new GenericDatumReader<>(writer, reader, data) .read(null,in); if (error instanceof CharSequence) return new AvroRuntimeException(error.toString()); // system error return new AvroRemoteException(error); }
@VisibleForTesting InlineSchemaAvroBytesDecoder(Schema schemaObj) { this.schemaObj = schemaObj; this.reader = new GenericDatumReader<>(schemaObj); this.schema = null; }
@Test public void testSerialize() throws IOException { Schema writerSchema = Schema.create(Schema.Type.STRING); AvroSerializer<CharSequence> serializer = new AvroSerializer<>(writerSchema); Schema readerSchema = Schema.create(Schema.Type.STRING); DatumReader<CharSequence> datumReader = new GenericDatumReader<>(readerSchema); Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); CharSequence record = null;
@Override public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args) throws Exception { if (args.size() != 1) { err.println("Expected 1 argument: input_file"); return 1; } DataFileReader<Void> reader = new DataFileReader<>(Util.openSeekableFromFS(args.get(0)), new GenericDatumReader<>()); out.println(reader.getSchema().toString(true)); return 0; } }
@Override public GenericContainer read(Kryo kryo, Input input, Class<GenericContainer> aClass) { Schema theSchema = this.getSchema(input.readString()); GenericDatumReader<GenericContainer> reader = new GenericDatumReader<>(theSchema); Decoder decoder = DecoderFactory .get() .directBinaryDecoder(input, null); GenericContainer foo; try { foo = reader.read(null, decoder); } catch (IOException e) { throw new RuntimeException(e); } return foo; } }
Decoder in = DecoderFactory.get().binaryDecoder( new ByteBufferInputStream(buffers), null); ByteBufferOutputStream bbo = new ByteBufferOutputStream(); context.setRequestCallMeta(META_READER.read(null, in)); String messageName = in.readString(null).toString(); if (messageName.equals("")) // a handshake ping Message rm = remote.getMessages().get(messageName); if (rm == null) throw new AvroRuntimeException("No such remote message: "+messageName); Message m = getLocal().getMessages().get(messageName); if (m == null) throw new AvroRuntimeException("No message named "+messageName +" in "+getLocal()); throw new AvroRuntimeException("Not both one-way: "+messageName);
public static List<String> getColumnNamesFromAvro(File avro) throws IOException { List<String> ret = new ArrayList<String>(); DataFileStream<GenericRecord> dataStream = new DataFileStream<GenericRecord>(new FileInputStream(avro), new GenericDatumReader<GenericRecord>()); for (final Field field : dataStream.getSchema().getFields()) { ret.add(field.name()); } return ret; }
private List<Object> fromJson(Schema schema, File file) throws Exception { InputStream in = new FileInputStream(file); List<Object> data = new ArrayList<>(); try { DatumReader reader = new GenericDatumReader(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, in); while (true) data.add(reader.read(null, decoder)); } catch (EOFException e) { } finally { in.close(); } return data; }
/** * Convert the payload in the input record to a deserialized object with the latest schema * * @param inputRecord the input record * @return the schema'ed payload object */ protected P upConvertPayload(GenericRecord inputRecord) throws DataConversionException { try { Schema payloadSchema = getPayloadSchema(inputRecord); // Set writer schema latestPayloadReader.setSchema(payloadSchema); byte[] payloadBytes = getPayloadBytes(inputRecord); Decoder decoder = DecoderFactory.get().binaryDecoder(payloadBytes, null); // 'latestPayloadReader.read' will convert the record from 'payloadSchema' to the latest payload schema return latestPayloadReader.read(null, decoder); } catch (Exception e) { throw new DataConversionException(e); } } }
private GenericRecord serializeWithWriterThenDeserializeWithReader(Schema writerSchema, GenericRecord datum, Schema readerSchema) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); DatumWriter<Object> datumWriter = new GenericDatumWriter<>(writerSchema); datumWriter.write(datum, encoder); encoder.flush(); byte[] bytes = baos.toByteArray(); Decoder decoder = DecoderFactory.get().resolvingDecoder( writerSchema, readerSchema, DecoderFactory.get().binaryDecoder(bytes, null)); DatumReader<Object> datumReader = new GenericDatumReader<>(readerSchema); return (GenericRecord)datumReader.read(null, decoder); }
@Test() public void testUseMeta() throws IOException { DataFileWriter<?> w = new DataFileWriter<>(new GenericDatumWriter<>()); File f = new File(DIR.getRoot().getPath(), "testDataFileMeta.avro"); w.setMeta("hello", "bar"); w.create(Schema.create(Type.NULL), f); w.close(); DataFileStream<Void> r = new DataFileStream<>(new FileInputStream(f), new GenericDatumReader<>()); assertTrue(r.getMetaKeys().contains("hello")); assertEquals("bar", r.getMetaString("hello")); }