@Override public RecordSchema getSchema() { return new SimpleRecordSchema(fields); } };
/** * Converts an Avro Schema to a RecordSchema * * @param avroSchema the Avro Schema to convert * @param schemaText the textual representation of the schema * @param schemaId the identifier of the schema * @return the Corresponding Record Schema */ public static RecordSchema createSchema(final Schema avroSchema, final String schemaText, final SchemaIdentifier schemaId) { if (avroSchema == null) { throw new IllegalArgumentException("Avro Schema cannot be null"); } final String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName(); final SimpleRecordSchema recordSchema = schemaText == null ? new SimpleRecordSchema(schemaId) : new SimpleRecordSchema(schemaText, AVRO_SCHEMA_FORMAT, schemaId); recordSchema.setSchemaName(avroSchema.getName()); recordSchema.setSchemaNamespace(avroSchema.getNamespace()); final DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); final Map<String, DataType> knownRecords = new HashMap<>(); knownRecords.put(schemaFullName, recordSchemaType); final List<RecordField> recordFields = new ArrayList<>(avroSchema.getFields().size()); for (final Field field : avroSchema.getFields()) { final String fieldName = field.name(); final Schema fieldSchema = field.schema(); final DataType dataType = AvroTypeUtil.determineDataType(fieldSchema, knownRecords); final boolean nullable = isNullable(fieldSchema); addFieldToList(recordFields, field, fieldName, fieldSchema, dataType, nullable); } recordSchema.setFields(recordFields); return recordSchema; }
public SimpleRecordSchema(final List<RecordField> fields, final SchemaIdentifier id) { this(fields, createText(fields), null, false, id); }
return knownRecordTypes.get(schemaFullName); } else { SimpleRecordSchema recordSchema = new SimpleRecordSchema(SchemaIdentifier.EMPTY); DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); knownRecordTypes.put(schemaFullName, recordSchemaType); recordSchema.setFields(recordFields); return recordSchemaType;
private SimpleRecordSchema(final List<RecordField> fields, final String text, final String schemaFormat, final boolean textAvailable, final SchemaIdentifier id) { this(text, schemaFormat, textAvailable, id); setFields(fields); }
@Override public List<String> getFieldNames() { return getFields().stream().map(RecordField::getFieldName) .collect(Collectors.toList()); }
/** * Converts an Avro Schema to a RecordSchema * * @param avroSchema the Avro Schema to convert * @param schemaText the textual representation of the schema * @param schemaId the identifier of the schema * @return the Corresponding Record Schema */ public static RecordSchema createSchema(final Schema avroSchema, final String schemaText, final SchemaIdentifier schemaId) { if (avroSchema == null) { throw new IllegalArgumentException("Avro Schema cannot be null"); } final String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName(); final SimpleRecordSchema recordSchema = schemaText == null ? new SimpleRecordSchema(schemaId) : new SimpleRecordSchema(schemaText, AVRO_SCHEMA_FORMAT, schemaId); final DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); final Map<String, DataType> knownRecords = new HashMap<>(); knownRecords.put(schemaFullName, recordSchemaType); final List<RecordField> recordFields = new ArrayList<>(avroSchema.getFields().size()); for (final Field field : avroSchema.getFields()) { final String fieldName = field.name(); final Schema fieldSchema = field.schema(); final DataType dataType = AvroTypeUtil.determineDataType(fieldSchema, knownRecords); final boolean nullable = isNullable(fieldSchema); addFieldToList(recordFields, field, fieldName, fieldSchema, dataType, nullable); } recordSchema.setFields(recordFields); return recordSchema; }
private SimpleRecordSchema(final List<RecordField> fields, final String text, final String schemaFormat, final boolean textAvailable, final SchemaIdentifier id) { this(text, schemaFormat, textAvailable, id); setFields(fields); }
@Override public List<DataType> getDataTypes() { return getFields().stream().map(RecordField::getDataType) .collect(Collectors.toList()); }
@Override public RecordSchema getSchema() { return new SimpleRecordSchema(fields); } };
return knownRecordTypes.get(schemaFullName); } else { SimpleRecordSchema recordSchema = new SimpleRecordSchema(SchemaIdentifier.EMPTY); DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); knownRecordTypes.put(schemaFullName, recordSchemaType); recordSchema.setFields(recordFields); return recordSchemaType;
public SimpleRecordSchema(final List<RecordField> fields) { this(fields, createText(fields), null, false, SchemaIdentifier.EMPTY); }
@Override public List<String> getFieldNames() { return getFields().stream().map(RecordField::getFieldName) .collect(Collectors.toList()); }
@Override public RecordSchema getSchema(Map<String, String> variables, RecordSchema readSchema) throws SchemaNotFoundException, IOException { return new SimpleRecordSchema(Collections.emptyList()); }
@Override public String toString() { String textValue = text.get(); if (textValue != null) { return textValue; } textValue = createText(fields); final boolean updated = text.compareAndSet(null, textValue); if (updated) { return textValue; } else { return text.get(); } }
@Override public List<DataType> getDataTypes() { return getFields().stream().map(RecordField::getDataType) .collect(Collectors.toList()); }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException { if (failAfterN > -1 && recordCount >= failAfterN) { throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read"); } final String nextLine = reader.readLine(); if (nextLine == null) { return null; } recordCount++; final String[] values = nextLine.split(","); final Map<String, Object> valueMap = new HashMap<>(); int i = 0; for (final RecordField field : fields) { final String fieldName = field.getFieldName(); valueMap.put(fieldName, values[i++].trim()); } return new MapRecord(new SimpleRecordSchema(fields), valueMap); }
public SimpleRecordSchema(final List<RecordField> fields) { this(fields, createText(fields), null, false, SchemaIdentifier.EMPTY); }
@Override public Record nextRecord(final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException { if (failAfterN >= 0 && recordCount >= failAfterN) { throw new MalformedRecordException("Intentional Unit Test Exception because " + recordCount + " records have been read"); } recordCount++; if (!itr.hasNext()) { return null; } final Object[] values = itr.next(); final Map<String, Object> valueMap = new HashMap<>(); int i = 0; for (final RecordField field : fields) { final String fieldName = field.getFieldName(); valueMap.put(fieldName, values[i++]); } return new MapRecord(new SimpleRecordSchema(fields), valueMap); }
public SimpleRecordSchema(final List<RecordField> fields, final SchemaIdentifier id) { this(fields, createText(fields), null, false, id); }