private static void _addIdSchemaField( List<Schema.Field> fields, Set<String> names) { String safeIdFieldName = "_id"; names.add(safeIdFieldName); Schema.Field designField = new Schema.Field( safeIdFieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); // This is the first column in the schema fields.add(0, designField); }
/** * Initialize converters per each schema field * * @param schema design schema * @review */ protected void initConverters(Schema schema) { schemaFields = schema.getFields(); avroConverters = new AvroConverter[schemaFields.size()]; for (int i = 0; i < schemaFields.size(); i++) { Schema.Field field = schemaFields.get(i); Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (LogicalTypeUtils.isLogicalTimestampMillis(fieldSchema)) { String datePattern = field.getProp( SchemaConstants.TALEND_COLUMN_PATTERN); avroConverters[i] = new StringTimestampConverter(datePattern); } else { Schema.Type type = fieldSchema.getType(); avroConverters[i] = _converterRegistry.get(type); } } }
private static Schema _getDeleteSchema() { List<Schema.Field> schemaFields = new ArrayList<>(1); Schema.Field designField = new Schema.Field( AvroConstants.ID, AvroUtils._string(), null, (Object)null); designField.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); schemaFields.add(designField); Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields); return schema; }
public static Schema inferSchemaByResourceType( ApioApiDocumentation.SupportedClass resourceSupportedClass) { List<Property> supportedProperties = resourceSupportedClass.getSupportedProperties(); List<Schema.Field> schemaFields = new ArrayList<>( supportedProperties.size() + 1); // Already used names for the fields Set<String> fieldNames = new HashSet<>(); _addIdSchemaField(schemaFields, fieldNames); int i = 1; for (Property supportedProperty : supportedProperties) { String fieldName = NameUtil.correct( supportedProperty.getName(), i, fieldNames); fieldNames.add(fieldName); Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); schemaFields.add(i, designField); i++; } Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields); return schema; }
protected String getIndexedRecordId(IndexedRecord indexedRecord) throws IOException { Schema indexRecordSchema = indexedRecord.getSchema(); List<Schema.Field> indexRecordFields = indexRecordSchema.getFields(); Stream<Schema.Field> stream = indexRecordFields.stream(); Schema.Field idField = stream.filter( field -> AvroConstants.ID.equals(field.name()) ).findFirst( ).orElseThrow( () -> new IOException( String.format( "Unable to find '%s' field in the incoming indexed record", AvroConstants.ID)) ); Schema fieldSchema = idField.schema(); Schema unwrappedSchema = AvroUtils.unwrapIfNullable(fieldSchema); Schema.Type fieldType = unwrappedSchema.getType(); if (fieldType == Schema.Type.STRING) { return (String)indexedRecord.get(idField.pos()); } throw new IOException( i18nMessages.getMessage( "error.unsupported.field.schema", idField.name(), fieldType.getName())); }
Map.Entry<String, JsonNode> field = jsonFields.next(); String fieldName = NameUtil.correct( field.getKey(), i, normalizedJsonFieldNames); JsonNode resourceJsonNode = field.getValue(); Object value = avroConverters[pos].convertToAvro( resourceJsonNode.asText());
protected Schema getSchema() throws IOException { if (schema == null) { schema = liferayConnectionResourceBaseProperties.getSchema(); if (AvroUtils.isIncludeAllFields(schema)) { String resourceURL = null; if (liferayConnectionResourceBaseProperties instanceof TLiferayInputProperties) { resourceURL = liferayConnectionResourceBaseProperties.resource. resourceProperty.getResourceURL(); } BoundedSource boundedSource = getCurrentSource(); schema = boundedSource.getEndpointSchema( runtimeContainer, resourceURL); } } return schema; }
@Override public org.apache.avro.Schema getSchema() { return AvroUtils._long(); }
/** * Internal utility method to add the primitive class to the shared converters and factory adapters to be reused by * all components. */ private static <DatumT> void registerSharedPrimitiveClass(Class<DatumT> primitiveClass, Schema schema) { mapSharedConverter.put(primitiveClass, new Unconverted<>(primitiveClass, schema)); mapSharedIndexedRecordConverter.put(primitiveClass, new LambdaSingleColumnIndexedRecordConverterSupplier<>(primitiveClass, schema)); }
@Override public ConverterT get() { return createANew(converterClass, datumClass); }
String fieldName = NameUtil.correct( fieldNames.get(i), i, filedNames); fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null);
private void _setupSchemas() { Schema.Field docIdField = new Schema.Field( "resourceId", AvroUtils._string(), null, (Object)null, Schema.Field.Order.ASCENDING); docIdField.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true"); docIdField.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); List<Schema.Field> fields = new ArrayList<>(); fields.add(docIdField); Schema initialSchema = Schema.createRecord( "liferay", null, null, false, fields); resource.main.schema.setValue(initialSchema); _updateOutputSchemas(); }
public static Schema createRejectSchema(Schema inputSchema) { final List<Schema.Field> rejectFields = new ArrayList<>(); Schema.Field field = new Schema.Field( FIELD_ERROR_MESSAGE, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255"); field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true"); field.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); rejectFields.add(field); Schema rejectSchema = SchemaUtils.newSchema( inputSchema, "rejectOutput", rejectFields); return rejectSchema; }
private ObjectNode _createApioExpectedForm( IndexedRecord indexedRecord, boolean excludeId) throws IOException { Schema indexRecordSchema = indexedRecord.getSchema(); List<Schema.Field> indexRecordFields = indexRecordSchema.getFields(); ObjectNode objectNode = _mapper.createObjectNode(); for (Schema.Field field : indexRecordFields) { String fieldName = field.name(); if (excludeId && fieldName.equals(AvroConstants.ID)) { continue; } Schema fieldSchema = field.schema(); Schema unwrappedSchema = AvroUtils.unwrapIfNullable(fieldSchema); Schema.Type fieldType = unwrappedSchema.getType(); if (fieldType == Schema.Type.STRING) { objectNode.put( fieldName, (String)indexedRecord.get(field.pos())); } else if (fieldType == Schema.Type.NULL) { objectNode.put(fieldName, ""); } else { throw new IOException( i18nMessages.getMessage( "error.unsupported.field.schema", field.name(), fieldType.getName())); } } return objectNode; }
String fieldName = NameUtil.correct( fieldNames.get(i), i, filedNames); fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null);
/** * Returns schema of avro data */ @Override public Schema getSchema() { return AvroUtils._string(); }
/** * Returns schema of avro data */ @Override public Schema getSchema() { return AvroUtils._string(); }
@Override public Schema getSchema() { return AvroUtils._string(); }
@Override public Schema getSchema() { return AvroUtils._string(); }
@Override public Schema getSchema() { return AvroUtils._string(); }