private JSONSchema(Class<T> pojo, Map<String, String> properties) { this.pojo = pojo; this.properties = properties; this.schema = ReflectData.AllowNull.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); this.schemaInfo.setType(SchemaType.JSON); this.schemaInfo.setSchema(this.schema.toString().getBytes()); this.objectMapper = JSON_MAPPER.get(); }
private <T> GenericRecord testJsonDecoder (String testType, byte[] bytes, T entityObj) throws IOException { ReflectData rdata = ReflectData.AllowNull.get(); Schema schema = rdata.getSchema(entityObj.getClass()); GenericDatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new String(bytes)); GenericRecord r = datumReader.read(null, decoder); return r; }
private <T> byte[] testJsonEncoder (String testType, T entityObj) throws IOException { ReflectData rdata = ReflectData.AllowNull.get(); Schema schema = rdata.getSchema(entityObj.getClass()); ByteArrayOutputStream os = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, os); ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter<>(schema, rdata); datumWriter.write(entityObj, encoder); encoder.flush(); byte[] bytes = os.toByteArray(); System.out.println ("JSON encoder output:\n" + new String(bytes)); return bytes; }
@Test public void testReadNullsWithPrimitivesAllowNullSchema() { final String name = "allowNullPrimitives"; try { repo.create(NAMESPACE, name, new DatasetDescriptor.Builder() .schema(ReflectData.AllowNull.get().getSchema(ObjectPoJo.class)) .build(), ObjectPoJo.class); // should load the dataset because PrimitivePoJo can be used to write final Dataset<PrimitivePoJo> dataset = repo.load( NAMESPACE, name, PrimitivePoJo.class); TestHelpers.assertThrows("AllowNull primitives cannot read nullable type", IncompatibleSchemaException.class, new Runnable() { @Override public void run() { dataset.newReader(); } }); } catch (RuntimeException e) { throw e; } finally { repo.delete(NAMESPACE, name); } }
/** * Returns a {@link GenericData} instance based on the mode type. * @return a {@link GenericData} instance based on the mode type. */ public GenericData getData() { if (factory != null) { return factory.getData(); } switch(this.modeType) { case REFLECT: return ReflectData.AllowNull.get(); case SPECIFIC: return SpecificData.get(); default: return GenericData.get(); } }
public ReflectData getReflectData() { return ReflectData.AllowNull.get(); }
public ReflectData getReflectData() { return ReflectData.AllowNull.get(); }
@Override public ReflectData getData() { return ReflectData.AllowNull.get(); }
public Schema getSchema(Class<?> datasetClass) { if (schema != null) { return schema; } Schema s; if (allowNullValues) { s = ReflectData.AllowNull.get().getSchema(datasetClass); } else { s = ReflectData.get().getSchema(datasetClass); } return s; } }
@Test public void testCreateWithAllowNullSchema() { String name = "allowNull"; try { repo.create(NAMESPACE, name, new DatasetDescriptor.Builder() .schema(ReflectData.AllowNull.get().getSchema(ObjectPoJo.class)) .build()); } catch (RuntimeException e) { throw e; } finally { repo.delete(NAMESPACE, name); } }
@Test public void testNull() throws IOException { File file = new File(DIR.getRoot().getPath(), "testNull.avro"); CheckList<BarRecord> check = new CheckList<>(); try(FileOutputStream fos = new FileOutputStream(file)) { ReflectData reflectData = ReflectData.AllowNull.get(); Schema schema = reflectData.getSchema(BarRecord.class); try(DataFileWriter<BarRecord> writer = new DataFileWriter<>(new ReflectDatumWriter<>(BarRecord.class, reflectData))) { writer.create(schema, fos); // test writing to a file write(writer, new BarRecord("One beer please"), check); // null record here, fails when using the default reflectData instance write(writer, new BarRecord(), check); write(writer, new BarRecord("Two beers please"), check); } } ReflectDatumReader<BarRecord> din = new ReflectDatumReader<>(); try(SeekableFileInput sin = new SeekableFileInput(file)) { try (DataFileReader<BarRecord> reader = new DataFileReader<>(sin, din)) { int count = 0; for (BarRecord datum : reader) { check.assertEquals(datum, count++); } Assert.assertEquals(count, check.size()); } } }
@Test public void testAllowNullWithNullableAnnotation() { Schema withNullable = ReflectData.AllowNull.get() .getSchema(AllowNullWithNullable.class); Assert.assertEquals("Should produce a nullable double", nullableSchema(double.class), withNullable.getField("aDouble").schema()); Schema nullableDoubleOrLong = Schema.createUnion(Arrays.asList( Schema.create(Schema.Type.NULL), Schema.create(Schema.Type.DOUBLE), Schema.create(Schema.Type.LONG))); Assert.assertEquals("Should add null to a non-null union", nullableDoubleOrLong, withNullable.getField("doubleOrLong").schema()); Assert.assertEquals("Should add null to a non-null union", nullableDoubleOrLong, withNullable.getField("doubleOrLongOrNull1").schema()); Schema doubleOrLongOrNull = Schema.createUnion(Arrays.asList( Schema.create(Schema.Type.DOUBLE), Schema.create(Schema.Type.LONG), Schema.create(Schema.Type.NULL))); Assert.assertEquals("Should add null to a non-null union", doubleOrLongOrNull, withNullable.getField("doubleOrLongOrNull2").schema()); Assert.assertEquals("Should add null to a non-null union", doubleOrLongOrNull, withNullable.getField("doubleOrLongOrNull3").schema()); }
@Test public void testWrappers() { // AllowNull only makes fields nullable, so testing must use a base record Schema wrappers = ReflectData.AllowNull.get().getSchema(Wrappers.class); Assert.assertEquals(nullableSchema(boolean.class), wrappers.getField("aBoolean").schema()); Assert.assertEquals(nullableSchema(byte.class), wrappers.getField("aByte").schema()); Assert.assertEquals(nullableSchema(short.class), wrappers.getField("aShort").schema()); Assert.assertEquals(nullableSchema(int.class), wrappers.getField("anInt").schema()); Assert.assertEquals(nullableSchema(long.class), wrappers.getField("aLong").schema()); Assert.assertEquals(nullableSchema(float.class), wrappers.getField("aFloat").schema()); Assert.assertEquals(nullableSchema(double.class), wrappers.getField("aDouble").schema()); Assert.assertEquals(nullableSchema(Primitives.class), wrappers.getField("anObject").schema()); }
@Test public void testRecordWithNullIO() throws IOException { ReflectData reflectData = ReflectData.AllowNull.get(); Schema schm = reflectData.getSchema(AnotherSampleRecord.class); ReflectDatumWriter<AnotherSampleRecord> writer = new ReflectDatumWriter<>(schm); ByteArrayOutputStream out = new ByteArrayOutputStream(); // keep record.a null and see if that works Encoder e = factory.directBinaryEncoder(out, null); AnotherSampleRecord a = new AnotherSampleRecord(); writer.write(a, e); AnotherSampleRecord b = new AnotherSampleRecord(10); writer.write(b, e); e.flush(); ReflectDatumReader<AnotherSampleRecord> reader = new ReflectDatumReader<>(schm); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Decoder d = DecoderFactory.get().binaryDecoder(in, null); AnotherSampleRecord decoded = reader.read(null, d); assertEquals(a, decoded); decoded = reader.read(null, d); assertEquals(b, decoded); }
/** * Test serialization of non-string map-key POJOs */ public <T> byte[] testSerialization(String testType, T ... entityObjs) throws Exception { log ("---- Beginning " + testType + " ----"); T entityObj1 = entityObjs[0]; ReflectData rdata = ReflectData.AllowNull.get(); Schema schema = rdata.getSchema(entityObj1.getClass()); assertNotNull("Unable to get schema for " + testType, schema); log (schema.toString(true)); ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter (entityObj1.getClass(), rdata); DataFileWriter<T> fileWriter = new DataFileWriter<>(datumWriter); ByteArrayOutputStream baos = new ByteArrayOutputStream(); fileWriter.create(schema, baos); for (T entityObj : entityObjs) { fileWriter.append(entityObj); } fileWriter.close(); byte[] bytes = baos.toByteArray(); return bytes; }
@Test public void testPrimitives() { // AllowNull only makes fields nullable, so testing must use a base record Schema primitives = ReflectData.AllowNull.get().getSchema(Primitives.class); Assert.assertEquals(requiredSchema(boolean.class), primitives.getField("aBoolean").schema()); Assert.assertEquals(requiredSchema(byte.class), primitives.getField("aByte").schema()); Assert.assertEquals(requiredSchema(short.class), primitives.getField("aShort").schema()); Assert.assertEquals(requiredSchema(int.class), primitives.getField("anInt").schema()); Assert.assertEquals(requiredSchema(long.class), primitives.getField("aLong").schema()); Assert.assertEquals(requiredSchema(float.class), primitives.getField("aFloat").schema()); Assert.assertEquals(requiredSchema(double.class), primitives.getField("aDouble").schema()); }
@Test public void testNullableStringableField() throws Exception { NullableStringable datum = new NullableStringable(); datum.number = java.math.BigDecimal.TEN; Schema schema = ReflectData.AllowNull.get().getSchema(NullableStringable.class); checkBinary(schema, datum); }
@Test public void testReflectWithAllowNullMatchStructure() throws SchemaValidationException { testValidatorPasses(builder.canBeReadStrategy().validateAll(), circleSchemaDifferentNames, ReflectData.AllowNull.get().getSchema(Circle.class)); }
private static <T> org.apache.avro.Schema createAvroSchema(Class<T> pojo) { return ReflectData.AllowNull.get().getSchema(pojo); }