@Override Object convertField(JsonElement value) { ParquetGroup array = new ParquetGroup((GroupType) schema()); JsonElementConverter converter = this.elementConverter; for (JsonElement elem : (JsonArray) value) { array.add(ARRAY_KEY, converter.convert(elem)); } return array; }
public CollectionConverter(JsonSchema collectionSchema, InputType elementType, boolean repeated) { super(collectionSchema); this.elementType = elementType; this.elementConverter = getConverter(getElementSchema(), repeated); this.schema = buildSchema(); }
public JsonElementConverter getKeyConverter() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(STRING); jsonSchema.setColumnName(MAP_KEY_COLUMN_NAME); return getConverter(jsonSchema, false); } }
@Override Object convertField(JsonElement value) { ParquetGroup mapGroup = new ParquetGroup((GroupType) schema()); JsonElementConverter converter = this.elementConverter; JsonObject map = (JsonObject) value; for (Map.Entry<String, JsonElement> entry : map.entrySet()) { ParquetGroup entrySet = (ParquetGroup) mapGroup.addGroup(MAP_KEY); entrySet.add(MAP_KEY_COLUMN_NAME, entry.getKey()); entrySet.add(MAP_VALUE_COLUMN_NAME, converter.convert(entry.getValue())); } return mapGroup; }
@Override public MessageType convertSchema(JsonArray inputSchema, WorkUnitState workUnit) throws SchemaConversionException { String fieldName = workUnit.getExtract().getTable(); JsonSchema jsonSchema = new JsonSchema(inputSchema); jsonSchema.setColumnName(fieldName); recordConverter = new RecordConverter(jsonSchema, ROOT); return (MessageType) recordConverter.schema(); }
@Override JsonSchema getElementSchema() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(STRING); jsonSchema.setColumnName(this.jsonSchema.getColumnName()); return jsonSchema; } }
@Override JsonSchema getElementSchema() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(this.elementType); jsonSchema.setColumnName(ARRAY_KEY); return jsonSchema; } }
/** * Builds a {@link JsonSchema} object for a given {@link InputType} object. * @param type * @return */ public static JsonSchema buildBaseSchema(InputType type) { JsonObject jsonObject = new JsonObject(); JsonObject dataType = new JsonObject(); jsonObject.addProperty(COLUMN_NAME_KEY, DEFAULT_RECORD_COLUMN_NAME); dataType.addProperty(TYPE_KEY, type.toString()); jsonObject.add(DATA_TYPE_KEY, dataType); return new JsonSchema(jsonObject); }
public JsonSchema(JsonArray jsonArray) { JsonObject jsonObject = new JsonObject(); JsonObject dataType = new JsonObject(); jsonObject.addProperty(COLUMN_NAME_KEY, DEFAULT_RECORD_COLUMN_NAME); dataType.addProperty(TYPE_KEY, RECORD.toString()); dataType.add(RECORD_FIELDS_KEY, jsonArray); jsonObject.add(DATA_TYPE_KEY, dataType); setJsonSchemaProperties(jsonObject); this.type = RECORD; }
public MapConverter(JsonSchema mapSchema) { super(mapSchema, mapSchema.getElementTypeUsingKey(MAP_ITEMS_KEY), false); }
@Override protected Type buildSchema() { return this.elementConverter.schema(); }
/** * @param jsonSchema * @param repeated * @param outputType */ public PrimitiveConverter(JsonSchema jsonSchema, boolean repeated, PrimitiveTypeName outputType) { super(jsonSchema); this.repeated = repeated; this.outputType = outputType; this.schema = buildSchema(); }
/** * Parquet {@link Repetition} for this {@link JsonSchema}. * @return */ public Repetition optionalOrRequired() { return this.isNullable() ? OPTIONAL : REQUIRED; }
public StringConverter(JsonSchema schema, boolean repeated) { super(schema, repeated, BINARY); this.schema = buildSchema(); }
public String toString() { return this.toString(""); }
public Group getGroup(int fieldIndex, int index) { return (Group) this.getValue(fieldIndex, index); }
@Test public void testRecordType() throws Exception { testCase("record"); }
@Override JsonSchema getElementSchema() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(this.elementType); jsonSchema.setColumnName(MAP_VALUE_COLUMN_NAME); return jsonSchema; }
public ArrayConverter(JsonSchema arraySchema) { super(arraySchema, arraySchema.getElementTypeUsingKey(ARRAY_ITEMS_KEY), true); }
@Test public void testMapType() throws Exception { testCase("map"); }