public static List<InputRow> parseGenericRecord( GenericRecord record, MapInputRowParser mapParser, ObjectFlattener<GenericRecord> avroFlattener ) { return mapParser.parseBatch(avroFlattener.flatten(record)); } }
@Override public List<InputRow> parseBatch(Group group) { return parser.parseBatch(groupFlattener.flatten(group)); } }
/** * @param input JSON string. The root must be a JSON object, not an array. * e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported * but [{"invalid": "true"}] and [1,2,3] are not. * * @return A map of field names and values */ @Override public Map<String, Object> parseToMap(String input) { try { JsonNode document = mapper.readValue(input, JsonNode.class); return flattener.flatten(document); } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } } }
/** * imitate avro extension {@link org.apache.druid.data.input.avro.AvroParsers#parseGenericRecord} */ @Override public List<InputRow> parseBatch(GenericRecord record) { Map<String, Object> row = recordFlattener.flatten(record); final List<String> dimensions; if (!this.dimensions.isEmpty()) { dimensions = this.dimensions; } else { dimensions = Lists.newArrayList( Sets.difference(row.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); } // check for parquet Date // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date LogicalType logicalType = determineTimestampSpecLogicalType(record.getSchema(), timestampSpec.getTimestampColumn()); DateTime dateTime; if (logicalType instanceof LogicalTypes.Date) { int daysSinceEpoch = (Integer) record.get(timestampSpec.getTimestampColumn()); dateTime = DateTimes.utc(TimeUnit.DAYS.toMillis(daysSinceEpoch)); } else { // Fall back to a binary format that will be parsed using joda-time dateTime = timestampSpec.extractTimestamp(row); } return ImmutableList.of(new MapBasedInputRow(dateTime, dimensions, row)); }
/** * @param input JSON string. The root must be a JSON object, not an array. * e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported * but [{"invalid": "true"}] and [1,2,3] are not. * * @return A map of field names and values */ @Override public Map<String, Object> parseToMap(String input) { try { JsonNode document = mapper.readValue(input, JsonNode.class); return flattener.flatten(document); } catch (Exception e) { throw new ParseException(e, "Unable to parse row [%s]", input); } } }
public static List<InputRow> parseGenericRecord( GenericRecord record, ParseSpec parseSpec, ObjectFlattener<GenericRecord> avroFlattener ) { return new MapInputRowParser(parseSpec).parseBatch(avroFlattener.flatten(record)); } }