@Test public void testWriteNullableUUIDReadRequiredString() throws IOException { Schema nullableUuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().optionalString("uuid").endRecord(); LogicalTypes.uuid().addToSchema( nullableUuidSchema.getField("uuid").schema().getTypes().get(1)); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithUUID r1 = new RecordWithUUID(); r1.uuid = u1; RecordWithUUID r2 = new RecordWithUUID(); r2.uuid = u2; List<RecordWithStringUUID> expected = Arrays.asList( new RecordWithStringUUID(), new RecordWithStringUUID()); expected.get(0).uuid = u1.toString(); expected.get(1).uuid = u2.toString(); File test = write(REFLECT, nullableUuidSchema, r1, r2); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); Assert.assertEquals("Should read uuid as String without UUID conversion", expected, read(REFLECT.createDatumReader(uuidStringSchema), test)); }
@Test public void testUnionWithNull() { Schema nestedRecord = SchemaBuilder.record("nested").fields().requiredDouble("double") .requiredString("string").endRecord(); Schema union = SchemaBuilder.unionOf().nullType().and().type(nestedRecord).endUnion(); Schema schema = SchemaBuilder.record("record").fields().name("union").type(union).noDefault().endRecord(); Schema doubleSchema = AvroUtils.getFieldSchema(schema, "union.double").get(); Assert.assertEquals(doubleSchema.getType(), Schema.Type.DOUBLE); GenericRecord nested = new GenericData.Record(nestedRecord); nested.put("double", 10); nested.put("string", "testString"); GenericRecord record = new GenericData.Record(schema); record.put("union", nested); String stringValue = AvroUtils.getFieldValue(record, "union.string").get().toString(); Assert.assertEquals(stringValue, "testString"); } }
@Test public void testGenericRecordReadWithCompatibleSchema() throws IOException { RegistryAvroDeserializationSchema<GenericRecord> deserializer = new RegistryAvroDeserializationSchema<>( GenericRecord.class, SchemaBuilder.record("Address") .fields() .requiredString("street") .requiredInt("num") .optionalString("country") .endRecord(), () -> new SchemaCoder() { @Override public Schema readSchema(InputStream in) { return Address.getClassSchema(); } } ); GenericRecord genericRecord = deserializer.deserialize(writeRecord( address, Address.getClassSchema())); assertEquals(address.getNum(), genericRecord.get("num")); assertEquals(address.getStreet(), genericRecord.get("street").toString()); assertNull(genericRecord.get("city")); assertNull(genericRecord.get("state")); assertNull(genericRecord.get("zip")); assertNull(genericRecord.get("country")); }
@Test public void testResolveTypeObjectToGeneric() { Class<Object> type = Object.class; Schema schema = SchemaBuilder.record("User").fields() .requiredString("name") .requiredString("color") .endRecord(); Class expResult = GenericData.Record.class; Class result = DataModelUtil.resolveType(type, schema); assertEquals(expResult, result); }
@Override public void run() { new DatasetDescriptor.Builder() .schema(SchemaBuilder.record("Record").fields() .requiredString("field") .endRecord()) .partitionStrategy(new PartitionStrategy.Builder() .hash("array", 48) .build()) .build(); } });
public static GenericData.Record[] splitSentenceMethodReturnAvro(String sentence) { Schema wordSchema = record("word").fields().requiredString("word").endRecord(); String[] words = sentence.split(" "); GenericData.Record[] events = new GenericData.Record[words.length]; for (int i = 0; i < words.length; i++) { events[i] = new GenericData.Record(wordSchema); events[i].put(0, words[i]); } return events; }
public static GenericData.Record[] splitWordMethodReturnAvro(String word) { Schema schema = record("chars").fields().requiredString("char").endRecord(); GenericData.Record[] records = new GenericData.Record[word.length()]; for (int i = 0; i < word.length(); i++) { records[i] = new GenericData.Record(schema); records[i].put("char", Character.toString(word.charAt(i))); } return records; }
fields.add(new Field("enum2", enumSchema, null, (Object)null)); Schema recordSchema = SchemaBuilder.record("aRecord").fields().requiredString("myString").endRecord(); fields.add(new Field("record1", recordSchema, null, (Object)null)); fields.add(new Field("record2", recordSchema, null, (Object)null));
.optionalDouble("odouble") .nullableDouble("ndouble", 99.9d) .requiredString("rstring") .optionalString("ostring") .nullableString("nstring", "def")
@Test public void testSingleSubRecordExtraField() throws IOException { final Schema child = SchemaBuilder.record("Child") .namespace("org.apache.avro.nested") .fields() .requiredString("childField").endRecord(); final Schema parent = SchemaBuilder.record("Parent") .namespace("org.apache.avro.nested") .fields() .requiredString("parentField1") .name("child1").type(child).noDefault() .requiredString("parentField2").endRecord(); final String inputAsExpected = "{\n" + " \"parentField1\": \"parentValue1\",\n" + " \"child1\":{\n" + " \"childField\":\"childValue1\",\n" + //this field should be safely ignored " \"extraField\":\"extraValue\"\n" + " },\n" + " \"parentField2\":\"parentValue2\"\n" + "}"; final ByteArrayInputStream inputStream = new ByteArrayInputStream(inputAsExpected.getBytes()); final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(parent, inputStream); final DatumReader<Object> reader = new GenericDatumReader<Object>(parent); final GenericData.Record decoded = (GenericData.Record) reader.read(null, decoder); assertThat(decoded.get("parentField1").toString(), equalTo("parentValue1")); assertThat(decoded.get("parentField2").toString(), equalTo("parentValue2")); assertThat(((GenericData.Record)decoded.get("child1")).get("childField").toString(), equalTo("childValue1")); }
@Test public void testSingleSubRecord() throws IOException { final Schema child = SchemaBuilder.record("Child") .namespace("org.apache.avro.nested") .fields() .requiredString("childField").endRecord(); final Schema parent = SchemaBuilder.record("Parent") .namespace("org.apache.avro.nested") .fields() .requiredString("parentField1") .name("child1").type(child).noDefault() .requiredString("parentField2").endRecord(); final String inputAsExpected = "{\n" + " \"parentField1\": \"parentValue1\",\n" + " \"child1\":{\n" + " \"childField\":\"childValue1\"\n" + " },\n" + " \"parentField2\":\"parentValue2\"\n" + "}"; final ByteArrayInputStream inputStream = new ByteArrayInputStream(inputAsExpected.getBytes()); final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(parent, inputStream); final DatumReader<Object> reader = new GenericDatumReader<Object>(parent); final GenericData.Record decoded = (GenericData.Record) reader.read(null, decoder); assertThat(decoded.get("parentField1").toString(), equalTo("parentValue1")); assertThat(decoded.get("parentField2").toString(), equalTo("parentValue2")); assertThat(((GenericData.Record)decoded.get("child1")).get("childField").toString(), equalTo("childValue1")); }
@Test public void testWriteUUID() throws IOException { Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithUUID r1 = new RecordWithUUID(); r1.uuid = u1; RecordWithUUID r2 = new RecordWithUUID(); r2.uuid = u2; List<RecordWithStringUUID> expected = Arrays.asList( new RecordWithStringUUID(), new RecordWithStringUUID()); expected.get(0).uuid = u1.toString(); expected.get(1).uuid = u2.toString(); File test = write(REFLECT, uuidSchema, r1, r2); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); Assert.assertEquals("Should read uuid as String without UUID conversion", expected, read(REFLECT.createDatumReader(uuidStringSchema), test)); LogicalTypes.uuid().addToSchema(uuidStringSchema.getField("uuid").schema()); Assert.assertEquals("Should read uuid as String without UUID logical type", expected, read(ReflectData.get().createDatumReader(uuidStringSchema), test)); }
@Test public void testReadUUID() throws IOException { Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithStringUUID r1 = new RecordWithStringUUID(); r1.uuid = u1.toString(); RecordWithStringUUID r2 = new RecordWithStringUUID(); r2.uuid = u2.toString(); List<RecordWithUUID> expected = Arrays.asList( new RecordWithUUID(), new RecordWithUUID()); expected.get(0).uuid = u1; expected.get(1).uuid = u2; File test = write( ReflectData.get().getSchema(RecordWithStringUUID.class), r1, r2); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidSchema), test)); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidStringSchema.getField("uuid").schema()); Assert.assertEquals("Should not convert to UUID if accessor is String", Arrays.asList(r1, r2), read(REFLECT.createDatumReader(uuidStringSchema), test)); }
@Test public void testReadUUIDMissingLogicalTypeUnsafe() throws IOException { String unsafeValue = System.getProperty("avro.disable.unsafe"); try { // only one FieldAccess can be set per JVM System.clearProperty("avro.disable.unsafe"); Assume.assumeTrue( ReflectionUtil.getFieldAccess() instanceof FieldAccessUnsafe); Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); RecordWithStringUUID r1 = new RecordWithStringUUID(); r1.uuid = u1.toString(); File test = write( ReflectData.get().getSchema(RecordWithStringUUID.class), r1); RecordWithUUID datum = (RecordWithUUID) read( ReflectData.get().createDatumReader(uuidSchema), test).get(0); Object uuid = datum.uuid; Assert.assertTrue("UUID should be a String (unsafe)", uuid instanceof String); } finally { if (unsafeValue != null) { System.setProperty("avro.disable.unsafe", unsafeValue); } } }
@Test public void testReadUUIDGenericRecord() throws IOException { Schema uuidSchema = SchemaBuilder.record("RecordWithUUID") .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithStringUUID r1 = new RecordWithStringUUID(); r1.uuid = u1.toString(); RecordWithStringUUID r2 = new RecordWithStringUUID(); r2.uuid = u2.toString(); List<GenericData.Record> expected = Arrays.asList( new GenericData.Record(uuidSchema), new GenericData.Record(uuidSchema)); expected.get(0).put("uuid", u1); expected.get(1).put("uuid", u2); File test = write( ReflectData.get().getSchema(RecordWithStringUUID.class), r1, r2); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidSchema), test)); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); Assert.assertEquals("Should not convert to UUID if accessor is String", Arrays.asList(r1, r2), read(REFLECT.createDatumReader(uuidStringSchema), test)); }
@Test public void testAllTypes() throws SchemaValidationException { Schema s = SchemaBuilder.record("r").fields() .requiredBoolean("boolF") .requiredInt("intF") .requiredLong("longF") .requiredFloat("floatF") .requiredDouble("doubleF") .requiredString("stringF") .requiredBytes("bytesF") .name("fixedF1").type().fixed("F1").size(1).noDefault() .name("enumF").type().enumeration("E1").symbols("S").noDefault() .name("mapF").type().map().values().stringType().noDefault() .name("arrayF").type().array().items().stringType().noDefault() .name("recordF").type().record("inner").fields() .name("f").type().intType().noDefault() .endRecord().noDefault() .optionalBoolean("boolO") .endRecord(); testValidatorPasses(builder.mutualReadStrategy().validateLatest(), s, s); }
@Test(expected = IllegalArgumentException.class) public void testReadUUIDMissingLogicalTypeReflect() throws IOException { String unsafeValue = System.getProperty("avro.disable.unsafe"); try { // only one FieldAccess can be set per JVM System.setProperty("avro.disable.unsafe", "true"); Assume.assumeTrue( ReflectionUtil.getFieldAccess() instanceof FieldAccessReflect); Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); RecordWithStringUUID r1 = new RecordWithStringUUID(); r1.uuid = u1.toString(); File test = write( ReflectData.get().getSchema(RecordWithStringUUID.class), r1); read(ReflectData.get().createDatumReader(uuidSchema), test).get(0); } finally { if (unsafeValue != null) { System.setProperty("avro.disable.unsafe", unsafeValue); } } }
@Test(expected = DataFileWriter.AppendWriteException.class) public void testWriteUUIDMissingLogicalType() throws IOException { Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName()) .fields().requiredString("uuid").endRecord(); LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); RecordWithUUID r1 = new RecordWithUUID(); r1.uuid = u1; RecordWithUUID r2 = new RecordWithUUID(); r2.uuid = u2; // write without using REFLECT, which has the logical type File test = write(uuidSchema, r1, r2); // verify that the field's type overrides the logical type Schema uuidStringSchema = SchemaBuilder .record(RecordWithStringUUID.class.getName()) .fields().requiredString("uuid").endRecord(); // this fails with an AppendWriteException wrapping ClassCastException // because the UUID isn't converted to a CharSequence expected internally read(ReflectData.get().createDatumReader(uuidStringSchema), test); }
@Test public void testValidateUnion() { Schema type1Schema = SchemaBuilder.record("Type1") .fields() .requiredString("myString") .requiredInt("myInt") .endRecord(); Schema type2Schema = SchemaBuilder.record("Type2") .fields() .requiredString("myString") .endRecord(); Schema unionSchema = SchemaBuilder.unionOf() .type(type1Schema).and().type(type2Schema) .endUnion(); GenericRecord record = new GenericData.Record(type2Schema); record.put("myString", "myValue"); assertTrue(GenericData.get().validate(unionSchema, record)); }
@Test /** check that GenericArray.reset() retains reusable elements and that GenericArray.prune() cleans * them up properly. */ public void testGenericArrayPeek() { Schema elementSchema = SchemaBuilder.record("element").fields().requiredString("value").endRecord(); Schema arraySchema = Schema.createArray(elementSchema); GenericRecord record = new GenericData.Record( elementSchema ); record.put("value", "string"); GenericArray<GenericRecord> list = new GenericData.Array<GenericRecord>(1, arraySchema); list.add(record); list.reset(); assertTrue( record == list.peek() ); list.prune(); assertNull( list.peek() ); }