/** * Initialize converters per each schema field * * @param schema design schema * @review */ protected void initConverters(Schema schema) { schemaFields = schema.getFields(); avroConverters = new AvroConverter[schemaFields.size()]; for (int i = 0; i < schemaFields.size(); i++) { Schema.Field field = schemaFields.get(i); Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (LogicalTypeUtils.isLogicalTimestampMillis(fieldSchema)) { String datePattern = field.getProp( SchemaConstants.TALEND_COLUMN_PATTERN); avroConverters[i] = new StringTimestampConverter(datePattern); } else { Schema.Type type = fieldSchema.getType(); avroConverters[i] = _converterRegistry.get(type); } } }
protected String getIndexedRecordId(IndexedRecord indexedRecord) throws IOException { Schema indexRecordSchema = indexedRecord.getSchema(); List<Schema.Field> indexRecordFields = indexRecordSchema.getFields(); Stream<Schema.Field> stream = indexRecordFields.stream(); Schema.Field idField = stream.filter( field -> AvroConstants.ID.equals(field.name()) ).findFirst( ).orElseThrow( () -> new IOException( String.format( "Unable to find '%s' field in the incoming indexed record", AvroConstants.ID)) ); Schema fieldSchema = idField.schema(); Schema unwrappedSchema = AvroUtils.unwrapIfNullable(fieldSchema); Schema.Type fieldType = unwrappedSchema.getType(); if (fieldType == Schema.Type.STRING) { return (String)indexedRecord.get(idField.pos()); } throw new IOException( i18nMessages.getMessage( "error.unsupported.field.schema", idField.name(), fieldType.getName())); }
private ObjectNode _createApioExpectedForm( IndexedRecord indexedRecord, boolean excludeId) throws IOException { Schema indexRecordSchema = indexedRecord.getSchema(); List<Schema.Field> indexRecordFields = indexRecordSchema.getFields(); ObjectNode objectNode = _mapper.createObjectNode(); for (Schema.Field field : indexRecordFields) { String fieldName = field.name(); if (excludeId && fieldName.equals(AvroConstants.ID)) { continue; } Schema fieldSchema = field.schema(); Schema unwrappedSchema = AvroUtils.unwrapIfNullable(fieldSchema); Schema.Type fieldType = unwrappedSchema.getType(); if (fieldType == Schema.Type.STRING) { objectNode.put( fieldName, (String)indexedRecord.get(field.pos())); } else if (fieldType == Schema.Type.NULL) { objectNode.put(fieldName, ""); } else { throw new IOException( i18nMessages.getMessage( "error.unsupported.field.schema", field.name(), fieldType.getName())); } } return objectNode; }
public static Schema getUnwrappedSchema(Field field) { return AvroUtils.unwrapIfNullable(field.schema()); }
private static Schema getUnwrappedSchema(Field field) { return AvroUtils.unwrapIfNullable(field.schema()); }
public static Schema getUnwrappedSchema(Schema.Field field) { return AvroUtils.unwrapIfNullable(field.schema()); }
private static Schema getUnwrappedSchema(Field field) { return AvroUtils.unwrapIfNullable(field.schema()); }
public static Schema getUnwrappedSchema(Schema.Field field) { return AvroUtils.unwrapIfNullable(field.schema()); }
public static Schema getUnwrappedSchema(Field field) { return AvroUtils.unwrapIfNullable(field.schema()); }
public static Schema getUnwrappedSchema(IndexedRecord record) { return AvroUtils.unwrapIfNullable(record.getSchema()); }
public static Schema getUnwrappedSchema(IndexedRecord record) { return AvroUtils.unwrapIfNullable(record.getSchema()); }
/** * @param srcSchema THe schema to investigate. * @return The Date/Time output type that corresponds to the logical type of the schema, or null if none. */ public static TypeConverterOutputTypes getDateTimeTypeFromLogicalType(Schema srcSchema) { LogicalType srcLogicalType = AvroUtils.unwrapIfNullable(srcSchema).getLogicalType(); if (srcLogicalType instanceof LogicalTypes.Date) { return TypeConverterOutputTypes.Date; } else if (srcLogicalType instanceof LogicalTypes.TimeMillis) { return TypeConverterOutputTypes.Time; } else if (srcLogicalType instanceof LogicalTypes.TimestampMillis) { return TypeConverterOutputTypes.DateTime; } else { return null; } }
/** * Get the child schema of a field, and check if this is correctly a Record. If not, thow a TalendRuntimeException. * * @param parentSchema the schema of the parent element * @param field the field to extract * @return the schema of the element extracted */ public static Schema getChildSchemaAsRecord(Schema parentSchema, Schema.Field field) { Schema childSchema = AvroUtils.unwrapIfNullable(parentSchema.getField(field.name()).schema()); if (childSchema.getType().equals(Schema.Type.RECORD)) { return childSchema; } else { throw new TalendRuntimeException(CommonErrorCodes.UNEXPECTED_EXCEPTION, new Throwable(String.format("The field %s has the type %s but should be a Record on the schema %s", field.name(), childSchema.getType(), parentSchema.toString()))); } }
static VisitableStructure createVisitableStructure(Schema schema, Object value, TraversalPath path) { Schema unwrappedSchema = AvroUtils.unwrapIfNullable(schema); switch (unwrappedSchema.getType()) { case ARRAY: return new VisitableArray(ensureArray((List) value, unwrappedSchema), path); default: return createWrapperForType(unwrappedSchema.getType(), value, path); } }
@Override public Iterable<AvroConverter<?, ?>> getNestedAvroConverters() { for (Schema.Field f : AvroUtils.unwrapIfNullable(getSchema()).getFields()) { int i = f.pos(); if (fieldType[i] == null) { fieldType[i] = getFieldDataSpec(i); } if (fieldConverter[i] == null) { fieldConverter[i] = getConverter(fieldType[i], f.schema(), null); } } return Arrays.asList((AvroConverter<?, ?>[]) fieldConverter); }
public AvroConverter<String, ?> convertToString(Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); switch (fieldSchema.getType()) { case LONG: String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (pattern != null && !pattern.isEmpty()) { fieldSchema.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, pattern); return new DateToStringConvert(fieldSchema); } else { return super.getConverter(String.class); } default: return super.getConverter(String.class); } }
public AvroConverter<String, ?> convertToString(Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); switch (fieldSchema.getType()) { case LONG: String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (pattern != null && !pattern.isEmpty()) { fieldSchema.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, pattern); return new DateToStringConvert(fieldSchema); } else { return super.getConverter(String.class); } default: return super.getConverter(String.class); } }
private TableFieldSchema tryArrayFieldSchema(org.apache.avro.Schema.Field field) { String fieldName = field.name(); TableFieldSchema tableFieldSchema = new TableFieldSchema().setName(fieldName); boolean nullable = AvroUtils.isNullable(field.schema()); if (!nullable) { tableFieldSchema = tableFieldSchema.setMode(REQUIRED_MODE); } org.apache.avro.Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (fieldSchema.getType() == org.apache.avro.Schema.Type.ARRAY) { return tryFieldSchema(tableFieldSchema.setMode(REPEATED_MODE), fieldSchema.getElementType()); } return tryFieldSchema(tableFieldSchema, fieldSchema); }
private TableFieldSchema tryArrayFieldSchema(org.apache.avro.Schema.Field field) { String fieldName = field.name(); TableFieldSchema tableFieldSchema = new TableFieldSchema().setName(fieldName); boolean nullable = AvroUtils.isNullable(field.schema()); if (!nullable) { tableFieldSchema = tableFieldSchema.setMode(REQUIRED_MODE); } org.apache.avro.Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (fieldSchema.getType() == org.apache.avro.Schema.Type.ARRAY) { return tryFieldSchema(tableFieldSchema.setMode(REPEATED_MODE), fieldSchema.getElementType()); } return tryFieldSchema(tableFieldSchema, fieldSchema); }
@Override public ValidationResult validate(RuntimeContainer arg0) { for (FilterDescriptor descriptor : filterDescriptors) { Schema.Field f = inputSchema.getField(descriptor.getColumnName()); ValidationResult filteringValidationResult = new FilterPrerequisitesValidator().validate(descriptor.getFunctionType(), descriptor.getOperatorType(), AvroUtils.unwrapIfNullable(f.schema()).getType(), descriptor.getPredefinedValue()); if (filteringValidationResult.getStatus() != Result.OK) { return filteringValidationResult; } } return ValidationResult.OK; }