private static void _addIdSchemaField( List<Schema.Field> fields, Set<String> names) { String safeIdFieldName = "_id"; names.add(safeIdFieldName); Schema.Field designField = new Schema.Field( safeIdFieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); // This is the first column in the schema fields.add(0, designField); }
fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null);
public static Schema createRejectSchema(Schema inputSchema) { final List<Schema.Field> rejectFields = new ArrayList<>(); Schema.Field field = new Schema.Field( FIELD_ERROR_MESSAGE, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255"); field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true"); field.addProp(SchemaConstants.TALEND_IS_LOCKED, "true"); rejectFields.add(field); Schema rejectSchema = SchemaUtils.newSchema( inputSchema, "rejectOutput", rejectFields); return rejectSchema; }
fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null);
public static Schema inferSchemaByResourceType( ApioApiDocumentation.SupportedClass resourceSupportedClass) { List<Property> supportedProperties = resourceSupportedClass.getSupportedProperties(); List<Schema.Field> schemaFields = new ArrayList<>( supportedProperties.size() + 1); // Already used names for the fields Set<String> fieldNames = new HashSet<>(); _addIdSchemaField(schemaFields, fieldNames); int i = 1; for (Property supportedProperty : supportedProperties) { String fieldName = NameUtil.correct( supportedProperty.getName(), i, fieldNames); fieldNames.add(fieldName); Schema.Field designField = new Schema.Field( fieldName, AvroUtils.wrapAsNullable(AvroUtils._string()), null, (Object)null); schemaFields.add(i, designField); i++; } Schema schema = Schema.createRecord( "Runtime", null, null, false, schemaFields); return schema; }
private Schema salesforceField2AvroTypeSchema(Field field) { Schema base = AvroUtils._string(); return field.getNillable() ? AvroUtils.wrapAsNullable(base) : base; }
/** * Get an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. * * @param node Json node. * @return an Avro schema using {@link AvroUtils#wrapAsNullable(Schema)} by node type. */ @VisibleForTesting Schema getAvroSchema(JsonNode node) { if (node instanceof TextNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else if (node instanceof IntNode) { return AvroUtils.wrapAsNullable(AvroUtils._int()); } else if (node instanceof LongNode) { return AvroUtils.wrapAsNullable(AvroUtils._long()); } else if (node instanceof DoubleNode) { return AvroUtils.wrapAsNullable(AvroUtils._double()); } else if (node instanceof BooleanNode) { return AvroUtils.wrapAsNullable(AvroUtils._boolean()); } else if (node instanceof NullNode) { return AvroUtils.wrapAsNullable(AvroUtils._string()); } else { return createSubRecord(node); } }
private Schema salesforceField2AvroTypeSchema(Field field) { Schema base = AvroUtils._string(); return field.getNillable() ? AvroUtils.wrapAsNullable(base) : base; }
/** * Creates new actual schema from incoming specification <code>schema</code> * Actual schema fields has the same names as specification schema, but they have String type * Note, getSchema() will return schema, which differs from specification schema passed to this method */ // TODO this one more kind of copySchema() method which should be implemented in Daikon see TDKN-96 @Override public void setSchema(Schema schema) { actualSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); List<Schema.Field> stringFields = new ArrayList<>(); for (Schema.Field specField : schema.getFields()) { boolean nullable = AvroUtils.isNullable(specField.schema()); Schema stringSchema = AvroUtils._string(); if (nullable) { stringSchema = AvroUtils.wrapAsNullable(stringSchema); } Schema.Field stringField = new Schema.Field(specField.name(), stringSchema, specField.doc(), specField.defaultVal(), specField.order()); for (Map.Entry<String, Object> entry : specField.getObjectProps().entrySet()) { stringField.addProp(entry.getKey(), entry.getValue()); } stringFields.add(stringField); } actualSchema.setFields(stringFields); for (Map.Entry<String, Object> entry : schema.getObjectProps().entrySet()) { actualSchema.addProp(entry.getKey(), entry.getValue()); } }
/** * Generate new field, * if the user did not set an output field path, use the input field name and the operation name * if the user set an output field path, use the name of the last element in the path. * * @param originalField the field to copy * @param operationProps the operation to execute * @return */ public static Schema.Field genField(Schema.Field originalField, AggregateOperationProperties operationProps) { Schema newFieldSchema = AvroUtils.wrapAsNullable(genFieldType(originalField.schema(), operationProps.operation.getValue())); String outputFieldPath = operationProps.outputFieldPath.getValue(); String newFieldName; if (StringUtils.isEmpty(outputFieldPath)) { newFieldName = genOutputFieldNameByOpt(operationProps.fieldPath.getValue(), operationProps.operation.getValue()); } else { newFieldName = outputFieldPath.contains(".") ? StringUtils.substringAfterLast(outputFieldPath, ".") : outputFieldPath; } return new Schema.Field(newFieldName, newFieldSchema, originalField.doc(), originalField.defaultVal()); }
/** * Creates new actual schema from incoming specification <code>schema</code> * Actual schema fields has the same names as specification schema, but they have String type * Note, getSchema() will return schema, which differs from specification schema passed to this method */ // TODO this one more kind of copySchema() method which should be implemented in Daikon see TDKN-96 @Override public void setSchema(Schema schema) { actualSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); List<Schema.Field> stringFields = new ArrayList<>(); for (Schema.Field specField : schema.getFields()) { boolean nullable = AvroUtils.isNullable(specField.schema()); Schema stringSchema = AvroUtils._string(); if (nullable) { stringSchema = AvroUtils.wrapAsNullable(stringSchema); } Schema.Field stringField = new Schema.Field(specField.name(), stringSchema, specField.doc(), specField.defaultVal(), specField.order()); for (Map.Entry<String, Object> entry : specField.getObjectProps().entrySet()) { stringField.addProp(entry.getKey(), entry.getValue()); } stringFields.add(stringField); } actualSchema.setFields(stringFields); for (Map.Entry<String, Object> entry : schema.getObjectProps().entrySet()) { actualSchema.addProp(entry.getKey(), entry.getValue()); } }
/** * Generate new field, * if the user did not set an output field path, use the input field name and the operation name * if the user set an output field path, use the name of the last element in the path. * * @param originalField the field to copy * @param operationProps the operation to execute * @return */ public static Schema.Field genField(Schema.Field originalField, AggregateOperationProperties operationProps) { Schema newFieldSchema = AvroUtils.wrapAsNullable(genFieldType(originalField.schema(), operationProps.operation.getValue())); String outputFieldPath = operationProps.outputFieldPath.getValue(); String newFieldName; if (StringUtils.isEmpty(outputFieldPath)) { newFieldName = genOutputFieldNameByOpt(operationProps.fieldPath.getValue(), operationProps.operation.getValue()); } else { newFieldName = outputFieldPath.contains(".") ? StringUtils.substringAfterLast(outputFieldPath, ".") : outputFieldPath; } return new Schema.Field(newFieldName, newFieldSchema, originalField.doc(), originalField.defaultVal()); }
public static Schema getEventSchema() { Schema.Field[] fields = new Schema.Field[10]; Schema.Field field; fields[EventSchemaField.EVENT_IDX] = new Schema.Field("event", AvroUtils._string(), "Type of event", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.PARTITION_IDX] = new Schema.Field("partition", AvroUtils._short(), "Partition number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.KEY_IDX] = new Schema.Field("key", AvroUtils._string(), "Key", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.CAS_IDX] = new Schema.Field("cas", AvroUtils._long(), "CAS", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.SEQNO_IDX] = new Schema.Field("bySeqno", AvroUtils._long(), "Sequence number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.REV_SEQNO_IDX] = new Schema.Field("revSeqno", AvroUtils._long(), "Revision sequence number", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.EXPIRATION_IDX] = new Schema.Field("expiration", AvroUtils.wrapAsNullable(AvroUtils._int()), "Expiration", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.FLAGS_IDX] = new Schema.Field("flags", AvroUtils.wrapAsNullable(AvroUtils._int()), "Flags", (Object) null, Schema.Field.Order.ASCENDING); fields[EventSchemaField.LOCK_TIME_IDX] = new Schema.Field("lockTime", AvroUtils.wrapAsNullable(AvroUtils._int()), "Lock time", (Object) null, Schema.Field.Order.ASCENDING); field = new Schema.Field("content", AvroUtils.wrapAsNullable(AvroUtils._bytes()), "Content", (Object) null, Schema.Field.Order.ASCENDING); field.addProp(TALEND_IS_LOCKED, "false"); fields[EventSchemaField.CONTENT_IDX] = field; Schema schema = Schema.createRecord("DcpMessage", "Couchbase DCP message", "com.couchbase", false, Arrays.asList(fields)); schema.addProp(TALEND_IS_LOCKED, "true"); return schema; }
private org.apache.avro.Schema inferSchemaField(Field field) { String name = field.getName(); Field.Type sqlType = field.getType(); Field.Mode mode = field.getMode(); // Get the "basic" type of the field. org.apache.avro.Schema fieldSchema = inferSchemaFieldWithoutMode(field); // BigQuery fields are NULLABLE by default. if (Field.Mode.NULLABLE == mode || mode == null) { fieldSchema = AvroUtils.wrapAsNullable(fieldSchema); } else if (Field.Mode.REPEATED == mode) { // Determine if the field is an array. // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type fieldSchema = SchemaBuilder.array().items(fieldSchema); } return fieldSchema; }
private org.apache.avro.Schema inferSchemaField(Field field) { String name = field.getName(); Field.Type sqlType = field.getType(); Field.Mode mode = field.getMode(); // Get the "basic" type of the field. org.apache.avro.Schema fieldSchema = inferSchemaFieldWithoutMode(field); // BigQuery fields are NULLABLE by default. if (Field.Mode.NULLABLE == mode || mode == null) { fieldSchema = AvroUtils.wrapAsNullable(fieldSchema); } else if (Field.Mode.REPEATED == mode) { // Determine if the field is an array. // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type fieldSchema = SchemaBuilder.array().items(fieldSchema); } return fieldSchema; }
/** * add field to Schema. */ private FieldAssembler<Schema> addField(FieldAssembler<Schema> record, String name, Class<?> type, AvroRegistry avroReg) { Schema base = avroReg.getConverter(type).getSchema(); FieldBuilder<Schema> fieldBuilder = record.name(name); fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault(); return record; }
private void setupDefaultSchema() { AvroRegistry avroReg = new AvroRegistry(); FieldAssembler<Schema> record = SchemaBuilder.record("Main").fields(); for (SplunkJSONEventField metadataField : SplunkJSONEventField.getMetadataFields()) { Schema base = avroReg.getConverter(metadataField.getDataType()).getSchema(); FieldBuilder<Schema> fieldBuilder = record.name(metadataField.getName()); if (metadataField.getName().equals(SplunkJSONEventField.TIME.getName())) { fieldBuilder.prop(SchemaConstants.TALEND_COLUMN_PATTERN, "dd-MM-yyyy"); } fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault(); } Schema defaultSchema = record.endRecord(); schema.schema.setValue(defaultSchema); }
private void setupDefaultSchema() { AvroRegistry avroReg = new AvroRegistry(); FieldAssembler<Schema> record = SchemaBuilder.record("Main").fields(); for (SplunkJSONEventField metadataField : SplunkJSONEventField.getMetadataFields()) { Schema base = avroReg.getConverter(metadataField.getDataType()).getSchema(); FieldBuilder<Schema> fieldBuilder = record.name(metadataField.getName()); if (metadataField.getName().equals(SplunkJSONEventField.TIME.getName())) { fieldBuilder.prop(SchemaConstants.TALEND_COLUMN_PATTERN, "dd-MM-yyyy"); } fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault(); } Schema defaultSchema = record.endRecord(); schema.schema.setValue(defaultSchema); }
private Schema initDefaultSchema(Schema designSchema) { AvroRegistry avroReg = new AvroRegistry(); FieldAssembler<Schema> record = SchemaBuilder.record("Main").fields(); for (SplunkJSONEventField metadataField : SplunkJSONEventField.getMetadataFields()) { Schema base = avroReg.getConverter(metadataField.getDataType()).getSchema(); FieldBuilder<Schema> fieldBuilder = record.name(metadataField.getName()); if (metadataField.getName().equals(SplunkJSONEventField.TIME.getName())) { String datePattern; Field designField = designSchema.getField(metadataField.getName()); if (designField != null) { datePattern = designField.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); } else { datePattern = designSchema.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); } if (datePattern == null || datePattern.isEmpty()) { datePattern = "dd-MM-yyyy"; } fieldBuilder.prop(SchemaConstants.TALEND_COLUMN_PATTERN, datePattern); } fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault(); } Schema defaultSchema = record.endRecord(); return defaultSchema; }
private Schema initDefaultSchema(Schema designSchema) { AvroRegistry avroReg = new AvroRegistry(); FieldAssembler<Schema> record = SchemaBuilder.record("Main").fields(); for (SplunkJSONEventField metadataField : SplunkJSONEventField.getMetadataFields()) { Schema base = avroReg.getConverter(metadataField.getDataType()).getSchema(); FieldBuilder<Schema> fieldBuilder = record.name(metadataField.getName()); if (metadataField.getName().equals(SplunkJSONEventField.TIME.getName())) { String datePattern; Field designField = designSchema.getField(metadataField.getName()); if (designField != null) { datePattern = designField.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); } else { datePattern = designSchema.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); } if (datePattern == null || datePattern.isEmpty()) { datePattern = "dd-MM-yyyy"; } fieldBuilder.prop(SchemaConstants.TALEND_COLUMN_PATTERN, datePattern); } fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault(); } Schema defaultSchema = record.endRecord(); return defaultSchema; }