/** * Constructor accepting the schema definition as a JSON string. * * @param schema a serialized JSON object representing a Avro schema. */ public AvroGenericSerializer(String schema) { typeDef = Schema.parse(schema); }
private void validateProperties() { String keySchemaString = properties.get(CONF_OUTPUT_KEY_SCHEMA); if (keySchemaString == null) { throw new IllegalStateException("No key schema provided, set '" + CONF_OUTPUT_KEY_SCHEMA + "' property"); } Schema.parse(keySchemaString); //verifying that schema valid String valueSchemaString = properties.get(CONF_OUTPUT_VALUE_SCHEMA); if (valueSchemaString == null) { throw new IllegalStateException("No value schema provided, set '" + CONF_OUTPUT_VALUE_SCHEMA + "' property"); } Schema.parse(valueSchemaString); //verifying that schema valid }
public AvroVersionedGenericSerializer(Map<Integer, String> typeDefVersions) { this.typeDefVersions = new TreeMap<Integer, String>(typeDefVersions); newestVersion = this.typeDefVersions.lastKey(); typeDef = Schema.parse(typeDefVersions.get(newestVersion)); }
/** * Constructor accepting the schema definition as a JSON string. * * @param schema a serialized JSON object representing a Avro schema. */ public AvroVersionedGenericSerializer(String schema) { this.typeDefVersions = new TreeMap<Integer, String>(); this.typeDefVersions.put(0, schema); newestVersion = typeDefVersions.lastKey(); typeDef = Schema.parse(typeDefVersions.get(newestVersion)); }
@Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((newestVersion == null) ? 0 : newestVersion.hashCode()); for(Map.Entry<Integer, String> versionedSchema: typeDefVersions.entrySet()) { Schema schema = Schema.parse(versionedSchema.getValue()); result = (prime * result) + schema.hashCode(); } return result; }
/** * Auto-generate the avro struct from schema * * @param schemaLiteral schema for the avro struct as string * @param sb StringBuilder to hold the generated struct * @throws SerDeException if something goes wrong while generating the struct * */ private static void generateAvroStructFromSchema(String schemaLiteral, StringBuilder sb) throws SerDeException { Schema schema = Schema.parse(schemaLiteral); generateAvroStructFromSchema(schema, sb); }
private Integer getSchemaVersion(Schema s) throws SerializationException { for(Entry<Integer, String> entry: typeDefVersions.entrySet()) { Schema version = Schema.parse(entry.getValue()); if(s.equals(version)) return entry.getKey(); } throw new SerializationException("Writer's schema invalid!"); }
@Override @SuppressWarnings("deprecation") public void open(FileSystem fs, Path path) throws IOException { super.open(fs, path); try { CodecFactory compressionCodec = getCompressionCodec(properties); Schema keySchema = Schema.parse(properties.get(CONF_OUTPUT_KEY_SCHEMA)); Schema valueSchema = Schema.parse(properties.get(CONF_OUTPUT_VALUE_SCHEMA)); keyValueWriter = new AvroKeyValueWriter<K, V>( keySchema, valueSchema, compressionCodec, getStream()); } finally { if (keyValueWriter == null) { close(); } } }
@Test public void testAvroSchemaEvolutionWrite() throws IOException { String versionZero = "{\"type\": \"record\", \"name\": \"myrec\",\"fields\": [{ \"name\": \"original\", \"type\": \"string\" }]}"; String versionOne = "{\"type\": \"record\", \"name\": \"myrec\",\"fields\": [{ \"name\": \"original\", \"type\": \"string\" } ," + "{ \"name\": \"new-field\", \"type\": \"string\", \"default\":\"\" }]}"; Schema s0 = Schema.parse(versionZero); Schema s1 = Schema.parse(versionOne); Map<Integer, String> versions = new HashMap<Integer, String>(); versions.put(0, versionZero); versions.put(1, versionOne); byte[] versionZeroBytes = writeVersion0with1Present(versions, s0); } }
@Test public void testAvroSchemaEvolution() throws IOException { String versionZero = "{\"type\": \"record\", \"name\": \"myrec\",\"fields\": [{ \"name\": \"original\", \"type\": \"string\" }]}"; String versionOne = "{\"type\": \"record\", \"name\": \"myrec\",\"fields\": [{ \"name\": \"original\", \"type\": \"string\" } ," + "{ \"name\": \"new-field\", \"type\": \"string\", \"default\":\"\" }]}"; Schema s0 = Schema.parse(versionZero); Schema s1 = Schema.parse(versionOne); Map<Integer, String> versions = new HashMap<Integer, String>(); versions.put(0, versionZero); versions.put(1, versionOne); byte[] versionZeroBytes = writeVersion0(s0); GenericData.Record record = (Record) readVersion0(versions, versionZeroBytes); }
@Test public void testDefaultValue() throws Exception { String s = "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" +SIMPLE_FIELDS+"," +"{\"name\":\"z\",\"type\":\"int\"," +"\"default\":1,\""+RandomData.USE_DEFAULT+"\":true}" +"]}"; checkWrite(Schema.parse(SIMPLE_RECORD)); checkRead(Schema.parse(s)); }
@Test public void testMapInRecord() throws Exception { String json = "{\"type\":\"record\", \"name\":\"Test\", \"fields\":" +"[{\"name\":\"f\", \"type\": {\"type\":\"map\", \"values\":\"long\"}}]}"; Schema schema = Schema.parse(json); HashMap<Utf8,Long> map = new HashMap<>(); map.put(new Utf8("a"), 1L); GenericData.Record record = new GenericData.Record(schema); record.put("f", map); check(new File(DIR.getRoot(), name.getMethodName()),json, "{\"f\":{\"a\":1}}", record, false); }
@Test public void testNull() throws Exception { Schema schema = Schema.parse("\"null\""); byte[] b = render(null, schema, new GenericDatumWriter<>()); assertEquals(0, BinaryData.compare(b, 0, b, 0, schema)); }
@Test public void testBytes() throws Exception { assertEquals(Schema.create(Type.BYTES), Schema.parse("\"bytes\"")); assertEquals(Schema.create(Type.BYTES), Schema.parse("{\"type\":\"bytes\"}")); check(new File(DIR.getRoot(), name.getMethodName()),"\"bytes\"", "\"\\u0000ABC\\u00FF\"", ByteBuffer.wrap(new byte[]{0,65,66,67,-1})); }
@Test public void testFieldDocs() { String schemaStr = "{\"name\": \"Rec\",\"type\": \"record\",\"fields\" : ["+ "{\"name\": \"f\", \"type\": \"int\", \"doc\": \"test\"}]}"; // check field doc is parsed correctly Schema schema = Schema.parse(schemaStr); assertEquals("test", schema.getField("f").doc()); // check print/read cycle preserves field doc schema = Schema.parse(schema.toString()); assertEquals("test", schema.getField("f").doc()); }
public void testRoundtripAvroWithGenericRecord() throws Exception { String jsonSchema = "{\"name\": \"Compact Disk\", \"type\": \"record\", " + "\"fields\": [" + "{\"name\": \"name\", \"type\": \"string\", \"order\": \"ascending\"}" + "]}"; AvroGenericSerializer serializer = new AvroGenericSerializer(jsonSchema); Record record = new Record(Schema.parse(jsonSchema)); // we need to use a Utf8 instance to map to a String. record.put("name", new Utf8("Hello")); byte[] bytes = serializer.toBytes(record); assertTrue(serializer.toObject(bytes).equals(record)); }
@Test public void testArray() throws Exception { String json = "{\"type\":\"array\", \"items\": \"long\"}"; Schema schema = Schema.parse(json); Collection<Long> array = new GenericData.Array<>(1, schema); array.add(1L); check(new File(DIR.getRoot(), name.getMethodName()),json, "[1]", array); array = new ArrayList<>(1); array.add(1L); check(new File(DIR.getRoot(), name.getMethodName()),json, "[1]", array); checkParseError("{\"type\":\"array\"}"); // items required }
private static void checkProp(Schema s0) throws Exception { if(s0.getType().equals(Schema.Type.UNION)) return; // unions have no props assertNull(s0.getProp("foo")); Schema s1 = Schema.parse(s0.toString()); s1.addProp("foo", "bar"); assertEquals("bar", s1.getProp("foo")); assertNotEquals(s0, s1); Schema s2 = Schema.parse(s1.toString()); assertEquals("bar", s2.getProp("foo")); assertEquals(s1, s2); assertNotEquals(s0, s2); }
@Test public void testFloat() throws Exception { assertEquals(Schema.create(Type.FLOAT), Schema.parse("\"float\"")); assertEquals(Schema.create(Type.FLOAT), Schema.parse("{\"type\":\"float\"}")); check(new File(DIR.getRoot(), name.getMethodName()),"\"float\"", "1.1", 1.1f); checkDefault("\"float\"", "\"NaN\"", Float.NaN); checkDefault("\"float\"", "\"Infinity\"", Float.POSITIVE_INFINITY); checkDefault("\"float\"", "\"-Infinity\"", Float.NEGATIVE_INFINITY); }