@Test public void shouldSerializeArray() { shouldSerializeTypeCorrectly( SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build(), ImmutableList.of(1, 2, 3), org.apache.avro.SchemaBuilder.array().items( org.apache.avro.SchemaBuilder.builder() .unionOf().nullType().and().intType().endUnion()) ); }
SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion(); tree.get(currentParrentValue).add(new Field(currentElement, outputHierarchicalSchema, "", ""));
SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion(); tree.get(currentParrentValue).add(new Field(currentElement, outputHierarchicalSchema, "", ""));
String fname = getName().replaceAll("-", "_"); fs = SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion(); switch (getDataType()) { case ("string"): case ("lead_function"): case ("reference"): fs = SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion(); break; case ("integer"): fs = SchemaBuilder.builder().unionOf().nullType().and().intType().endUnion(); break; case ("boolean"): fs = SchemaBuilder.builder().unionOf().nullType().and().booleanType().endUnion(); break; case ("float"): case ("currency"): fs = SchemaBuilder.builder().unionOf().nullType().and().floatType().endUnion(); break; case ("date"): case ("datetime"): fs = SchemaBuilder.builder().unionOf().nullType().and().longType().endUnion(); fs.getTypes().get(1).addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, getName()); fs.getTypes().get(1).addProp(SchemaConstants.TALEND_COLUMN_PATTERN, MarketoConstants.DATETIME_PATTERN_REST);
String fname = getName().replaceAll("-", "_"); fs = SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion(); switch (getDataType()) { case ("string"): case ("lead_function"): case ("reference"): fs = SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion(); break; case ("integer"): fs = SchemaBuilder.builder().unionOf().nullType().and().intType().endUnion(); break; case ("boolean"): fs = SchemaBuilder.builder().unionOf().nullType().and().booleanType().endUnion(); break; case ("float"): case ("currency"): fs = SchemaBuilder.builder().unionOf().nullType().and().floatType().endUnion(); break; case ("date"): case ("datetime"): fs = SchemaBuilder.builder().unionOf().nullType().and().longType().endUnion(); fs.getTypes().get(1).addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, getName()); fs.getTypes().get(1).addProp(SchemaConstants.TALEND_COLUMN_PATTERN, MarketoConstants.DATETIME_PATTERN_REST);
fieldAssembler = fieldAssembler.name(pom.getPredicate().getId().replaceAll("[^\\w]", "_")).type().unionOf().map().values().unionOf().map().values().stringType().and().stringType().and().nullType().endUnion().and().nullType().endUnion().noDefault(); fieldAssembler = fieldAssembler.name(pom.getPredicate().getId().replaceAll("[^\\w]", "_")).type().unionOf().map().values().unionOf().map().values(targetSchema).and().type(targetSchema).and().nullType().endUnion().and().nullType().endUnion().noDefault();
@Test public void testSchemaMergeUnionTypes() throws Exception { String jsonArraySample = "{\"id\": 1}\n{\"id\": 2}" + "{\"id\": \"socket\", \"union\": [33, 34, 35]}" + "{\"id\": 3, \"union\": [\"badger\", \"porcupine\"]}"; String jsonMapSample = "{\"id\": 1}\n{\"id\": 2}" + "{\"id\": \"socket\", \"union\": {\"coffee\": 17}}" + "{\"id\": 3, \"union\": {\"badger\": \"porcupine\"}}"; Schema schema = SchemaBuilder.record("Test").fields() .name("id").type() .unionOf().intType().and().stringType().endUnion().noDefault() .name("union").type().unionOf() .nullType() .and() .array().items().unionOf().intType().and().stringType().endUnion() .and() .map().values().unionOf().intType().and().stringType().endUnion() .endUnion().nullDefault() .endRecord(); Assert.assertEquals("Should match expected schema", schema, SchemaUtil.merge( mergeSchemas(jsonArraySample), mergeSchemasWithMaps(jsonMapSample)) ); }
@Test public void testSchemaInferenceNullableMap() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("aMap").type().map().values() .unionOf().nullType().and().stringType().endUnion().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"aMap\": { \"left\": null, \"right\": \"dictionary\" }" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchemaWithMaps(datum, "Test")); Map<String, Object> aMap = Maps.newLinkedHashMap(); aMap.put("left", null); aMap.put("right", "dictionary"); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("aMap", aMap); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaInferenceMultipleTypes() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("anArray").type().array().items() .unionOf().nullType().and().intType().and().stringType().endUnion().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"anArray\": [ null, 1, 2, 3, \"winter\" ]" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchema(datum, "Test")); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("anArray", Lists.newArrayList(null, 1, 2, 3, "winter")); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaInferenceNullablePrimitiveArray() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("anArray").type().array().items() .unionOf().nullType().and().intType().endUnion().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"anArray\": [ null, 1, 2, 3, 4 ]" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchema(datum, "Test")); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("anArray", Lists.newArrayList(null, 1, 2, 3, 4)); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaMergeMapTypes() throws Exception { String jsonSample = "{\"id\": 1}\n{\"id\": 2}" + "{\"id\": \"socket\", \"aMap\": {\"coffee\": 17}}" + "{\"id\": 3, \"aMap\": {\"badger\": \"porcupine\"}}"; Schema schema = SchemaBuilder.record("Test").fields() .name("id").type() .unionOf().intType().and().stringType().endUnion().noDefault() .name("aMap").type().optional().map().values() .unionOf().intType().and().stringType().endUnion() .endRecord(); Assert.assertEquals("Should match expected schema", schema, mergeSchemasWithMaps(jsonSample)); }
private Schema wrapAsNullable(Schema avroType) { return SchemaBuilder.builder().unionOf().nullType().and().type(avroType).endUnion(); } }
@Test public void testSchemaMergeArrayTypes() throws Exception { String jsonSample = "{\"id\": 1}\n{\"id\": 2}" + "{\"id\": \"socket\", \"anArray\": [33, 34, 35]}" + "{\"id\": 3, \"anArray\": [\"badger\", \"porcupine\"]}"; Schema schema = SchemaBuilder.record("Test").fields() .name("id").type() .unionOf().intType().and().stringType().endUnion().noDefault() .name("anArray").type().optional().array().items() .unionOf().intType().and().stringType().endUnion() .endRecord(); Assert.assertEquals("Should match expected schema", schema, mergeSchemas(jsonSample)); }
/** * This method will create a schema a union field consisting of null and the specified type. * * @param dataType The data type of the field */ public static Schema getUnionFieldType(String dataType) { return SchemaBuilder.builder().unionOf().nullBuilder().endNull().and().type(getSchemaForType(dataType)).endUnion(); }
/** * Create a builder for an Avro union * This is equivalent to: * <pre> * builder().unionOf(); * </pre> */ public static BaseTypeBuilder<UnionAccumulator<Schema>> unionOf() { return builder().unionOf(); }
/** * Create a builder for an Avro union * This is equivalent to: * <pre> * builder().unionOf(); * </pre> */ public static BaseTypeBuilder<UnionAccumulator<Schema>> unionOf() { return builder().unionOf(); }
/** * Create a builder for an Avro union * This is equivalent to: * <pre> * builder().unionOf(); * </pre> */ public static BaseTypeBuilder<UnionAccumulator<Schema>> unionOf() { return builder().unionOf(); }
/** * Create a builder for an Avro union * This is equivalent to: * <pre> * builder().unionOf(); * </pre> */ public static BaseTypeBuilder<UnionAccumulator<Schema>> unionOf() { return builder().unionOf(); }
/** * This method will create a schema a union field consisting of null and the specified type. * * @param dataType The data type of the field */ protected static Schema getUnionFieldType(String dataType) { return SchemaBuilder.builder().unionOf().nullBuilder().endNull().and().type(getSchemaForType(dataType)).endUnion(); }