public static JsonNode parse(InputStream in) { return parse(in, JsonNode.class); }
public static JsonNode parse(String json) { return parse(json, JsonNode.class); }
public static JsonNode parse(File file) { return parse(file, JsonNode.class); }
public static Schema removeEmbeddedMapping(Schema schema) { // TODO: avoid embedding mappings in the schema // Avro considers Props read-only and uses an older Jackson version // Parse the Schema as a String because Avro uses com.codehaus.jackson ObjectNode schemaJson = JsonUtil.parse(schema.toString(), ObjectNode.class); schemaJson.remove(MAPPING); return new Schema.Parser().parse(schemaJson.toString()); }
/** * Parses a PartitionStrategy from a JSON string. * * @param json * The JSON string * @return The PartitionStrategy. */ public static PartitionStrategy parse(String json) { return buildPartitionStrategy(JsonUtil.parse(json)); }
/** * Parses a PartitionStrategy from an input stream * * @param in * The input stream that contains the PartitionStrategy in JSON * format. * @return The PartitionStrategy. */ public static PartitionStrategy parse(InputStream in) { return buildPartitionStrategy(JsonUtil.parse(in)); }
/** * Parses the Mapping Descriptor from an input stream * * @param in * The input stream that contains the Mapping Descriptor in JSON * format. * @return ColumnMapping. */ public static ColumnMapping parse(InputStream in) { return buildColumnMapping(JsonUtil.parse(in)); }
public static Schema removeEmbeddedStrategy(Schema schema) { // TODO: avoid embedding strategies in the schema // Avro considers Props read-only and uses an older Jackson version // Parse the Schema as a String because Avro uses com.codehaus.jackson ObjectNode schemaJson = JsonUtil.parse(schema.toString(), ObjectNode.class); schemaJson.remove(PARTITIONS); return new Schema.Parser().parse(schemaJson.toString()); }
/** * Parses the Mapping Descriptor from a File * * @param file * The File that contains the Mapping Descriptor in JSON format. * @return ColumnMapping. */ public static ColumnMapping parse(File file) { return buildColumnMapping(JsonUtil.parse(file)); }
/** * Parses a PartitionStrategy from a File * * @param file * The File that contains the PartitionStrategy in JSON format. * @return The PartitionStrategy. */ public static PartitionStrategy parse(File file) { return buildPartitionStrategy(JsonUtil.parse(file)); }
/** * Parses the Mapping Descriptor as a JSON string. * * @param mappingDescriptor * The mapping descriptor as a JSON string * @return ColumnMapping */ public static ColumnMapping parse(String mappingDescriptor) { return buildColumnMapping(JsonUtil.parse(mappingDescriptor)); }
public static Schema embedPartitionStrategy(Schema schema, PartitionStrategy strategy) { // TODO: avoid embedding strategies in the schema // Avro considers Props read-only and uses an older Jackson version // Parse the Schema as a String because Avro uses com.codehaus.jackson ObjectNode schemaJson = JsonUtil.parse(schema.toString(), ObjectNode.class); schemaJson.set(PARTITIONS, toJson(strategy)); return new Schema.Parser().parse(schemaJson.toString()); }
public static Schema embedColumnMapping(Schema schema, ColumnMapping mapping) { // TODO: avoid embedding mappings in the schema // Avro considers Props read-only and uses an older Jackson version // Parse the Schema as a String because Avro uses com.codehaus.jackson ObjectNode schemaJson = JsonUtil.parse(schema.toString(), ObjectNode.class); schemaJson.set(MAPPING, toJson(mapping)); return new Schema.Parser().parse(schemaJson.toString()); }
public static Map<Integer, FieldMapping> parseKeyMappingsFromSchemaFields( Schema schema) { Map<Integer, FieldMapping> keyMappings = Maps.newHashMap(); if (Schema.Type.RECORD == schema.getType()) { for (Schema.Field field : schema.getFields()) { if (field.getJsonProp(MAPPING) != null) { // parse the String because Avro uses com.codehaus.jackson JsonNode mappingNode = JsonUtil.parse( field.getJsonProp(MAPPING).toString()); FieldMapping fm = parseFieldMapping(field.name(), mappingNode); if (FieldMapping.MappingType.KEY == fm.getMappingType() && mappingNode.has(VALUE)) { Integer index = mappingNode.get(VALUE).asInt(); keyMappings.put(index, fm); } } } return keyMappings; } throw new IllegalArgumentException( "Cannot parse field-level mappings from non-Record"); }
public static ColumnMapping parseFromSchemaFields(Schema schema) { if (Schema.Type.RECORD == schema.getType()) { ColumnMapping.Builder builder = new ColumnMapping.Builder(); for (Schema.Field field : schema.getFields()) { if (field.getJsonProp(MAPPING) != null) { // parse the String because Avro uses com.codehaus.jackson builder.fieldMapping(parseFieldMapping(field.name(), JsonUtil.parse(field.getJsonProp(MAPPING).toString()))); } } return builder.build(); } throw new IllegalArgumentException( "Cannot parse field-level mappings from non-Record"); }
@Test public void testSchemaInferenceMap() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("aMap").type().map().values().stringType().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"aMap\": { \"left\": \"timid\", \"right\": \"dictionary\" }" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchemaWithMaps(datum, "Test")); Map<String, Object> aMap = Maps.newLinkedHashMap(); aMap.put("left", "timid"); aMap.put("right", "dictionary"); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("aMap", aMap); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaInferenceNullableMap() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("aMap").type().map().values() .unionOf().nullType().and().stringType().endUnion().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"aMap\": { \"left\": null, \"right\": \"dictionary\" }" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchemaWithMaps(datum, "Test")); Map<String, Object> aMap = Maps.newLinkedHashMap(); aMap.put("left", null); aMap.put("right", "dictionary"); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("aMap", aMap); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaInferencePrimitiveArray() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("anArray").type().array().items().intType().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"anArray\": [ 1, 2, 3, 4 ]" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchema(datum, "Test")); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("anArray", Lists.newArrayList(1, 2, 3, 4)); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaInferenceNullablePrimitiveArray() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("anArray").type().array().items() .unionOf().nullType().and().intType().endUnion().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"anArray\": [ null, 1, 2, 3, 4 ]" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchema(datum, "Test")); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("anArray", Lists.newArrayList(null, 1, 2, 3, 4)); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }
@Test public void testSchemaInferenceMultipleTypes() throws Exception { Schema recordSchema = SchemaBuilder.record("Test").fields() .requiredString("aString") .name("anArray").type().array().items() .unionOf().nullType().and().intType().and().stringType().endUnion().noDefault() .endRecord(); String jsonSample = "{" + "\"aString\": \"triangle\"," + "\"anArray\": [ null, 1, 2, 3, \"winter\" ]" + "}"; JsonNode datum = JsonUtil.parse(jsonSample); Assert.assertEquals("Should produce expected schema", recordSchema, JsonUtil.inferSchema(datum, "Test")); GenericData.Record expected = new GenericData.Record(recordSchema); expected.put("aString", "triangle"); expected.put("anArray", Lists.newArrayList(null, 1, 2, 3, "winter")); Assert.assertEquals("Should convert to record", expected, convertGeneric(datum, recordSchema)); }