protected Schema inferSchemaDynamicTableEntity(DynamicTableEntity entity) { List<Field> fields = new ArrayList<>(); fields.add(new Field("PartitionKey", AvroUtils._string(), null, (Object) null)); fields.add(new Field("RowKey", AvroUtils._string(), null, (Object) null)); fields.add(new Field("Timestamp", AvroUtils._date(), null, (Object) null)); // FIXME set tableName properly and manage nameMappings String tableName = "schemaInfered"; for (Entry<String, EntityProperty> f : entity.getProperties().entrySet()) { String fieldName = f.getKey(); Field field = getAvroMapping(fieldName, f.getValue()); fields.add(field); } return Schema.createRecord(tableName, null, null, false, fields); }
protected Schema inferSchemaDynamicTableEntity(DynamicTableEntity entity) { List<Field> fields = new ArrayList<>(); fields.add(new Field("PartitionKey", AvroUtils._string(), null, (Object) null)); fields.add(new Field("RowKey", AvroUtils._string(), null, (Object) null)); fields.add(new Field("Timestamp", AvroUtils._date(), null, (Object) null)); // FIXME set tableName properly and manage nameMappings String tableName = "schemaInfered"; for (Entry<String, EntityProperty> f : entity.getProperties().entrySet()) { String fieldName = f.getKey(); Field field = getAvroMapping(fieldName, f.getValue()); fields.add(field); } return Schema.createRecord(tableName, null, null, false, fields); }
protected Object getFieldValue(Object inputValue, Field field) { Schema s = AvroUtils.unwrapIfNullable(field.schema()); if (inputValue != null && inputValue instanceof String && ((String) inputValue).isEmpty()) { return emptyStringValue; } else if (null == inputValue || inputValue instanceof String) { return inputValue; } else if (AvroUtils.isSameType(s, AvroUtils._date())) { Date date = (Date) inputValue; return date.getTime(); } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.timeMillis()) { return formatter.formatTimeMillis(inputValue); } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.date()) { return formatter.formatDate(inputValue); } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.timestampMillis()) { return formatter.formatTimestampMillis(inputValue); } else { return inputValue; } }
/** * Checks {@link TalendType#convertFromAvro(Schema)} converts long avro type with java-class flag "java.util.Date" to * "id_Date" * di type */ @Test public void testConvertFromAvroDate() { TalendType expectedType = TalendType.DATE; Schema fieldSchema = AvroUtils._date(); assertEquals(expectedType, TalendType.convertFromAvro(fieldSchema)); }
@Override public void setupProperties() { super.setupProperties(); Schema s = SchemaBuilder.record("Main").fields().name("ServerTimeStamp") .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'.000Z'") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "20")//$NON-NLS-1$ .type(AvroUtils._date()).noDefault().endRecord(); schema.schema.setValue(s); }
public static Schema getCustomObjectRecordSchema() { return record("CustomObjectRecordREST") .fields()// .name(FIELD_MARKETO_GUID) .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .type() .stringType() .noDefault()// .name(FIELD_SEQ) .type() .intType() .noDefault()// .name(FIELD_CREATED_AT) // .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)// .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) // .type(AvroUtils._date()) .noDefault()// .name(FIELD_UPDATED_AT) // .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)// .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) // .type(AvroUtils._date()) .noDefault()// // .name(FIELD_DEDUPE_FIELDS).type().nullable().stringType().noDefault()// // .endRecord(); }
/** * A helper method to convert the String representation of a datum in the DataPrep system to the Avro type that * matches the Schema generated for it. * * @param f is field in Avro Schema. * @return converter for a given type. */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
public static Schema getCustomObjectRecordSchema() { return record("CustomObjectRecordREST") .fields()// .name(FIELD_MARKETO_GUID) .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .type() .stringType() .noDefault()// .name(FIELD_SEQ) .type() .intType() .noDefault()// .name(FIELD_CREATED_AT) // .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)// .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) // .type(AvroUtils._date()) .noDefault()// .name(FIELD_UPDATED_AT) // .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)// .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) // .type(AvroUtils._date()) .noDefault()// // .name(FIELD_DEDUPE_FIELDS).type().nullable().stringType().noDefault()// // .endRecord(); }
public Schema getQueueSchema() { return SchemaBuilder.builder().record("Main").fields()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(AzureStorageQueueProperties.FIELD_INSERTION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_EXPIRATION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_NEXT_VISIBLE_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(AzureStorageQueueProperties.FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); }
public Schema getQueueSchema() { return SchemaBuilder.builder().record("Main").fields()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(AzureStorageQueueProperties.FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(AzureStorageQueueProperties.FIELD_INSERTION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_EXPIRATION_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_NEXT_VISIBLE_TIME) .prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss").type(AvroUtils._date()).noDefault() // .name(AzureStorageQueueProperties.FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(AzureStorageQueueProperties.FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); }
/** * A helper method to convert the String representation of a datum in the Salesforce system to the Avro type that * matches the Schema generated for it. * * @param f * @return */ public AvroConverter<String, ?> getConverterFromString(org.apache.avro.Schema.Field f) { Schema fieldSchema = AvroUtils.unwrapIfNullable(f.schema()); // FIXME use avro type to decide the converter is not correct if the user change the avro type, Date to String // for instance if (AvroUtils.isSameType(fieldSchema, AvroUtils._boolean())) { return new StringToBooleanConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._decimal())) { return new StringToDecimalConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._double())) { return new StringToDoubleConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._int())) { return new StringToIntegerConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._date())) { return new StringToDateConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._bytes())) { return new StringToBytesConverter(f); } else if (AvroUtils.isSameType(fieldSchema, AvroUtils._string())) { return super.getConverter(String.class); } throw new UnsupportedOperationException("The type " + fieldSchema.getType() + " is not supported."); //$NON-NLS-1$ //$NON-NLS-2$ }
public static Schema getOpportunitySchema() { return record("OpportunityREST") .fields()// .name(FIELD_MARKETO_GUID) .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .type() .stringType() .noDefault()// .name(FIELD_SEQ) .type() .intType() .noDefault()// .name("externalOpportunityId") .type() .stringType() .noDefault()// .name(FIELD_CREATED_AT) // .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)// .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) // .type(AvroUtils._date()) .noDefault()// .name(FIELD_UPDATED_AT) // .prop(SchemaConstants.TALEND_COLUMN_PATTERN, DATETIME_PATTERN_REST)// .prop(SchemaConstants.JAVA_CLASS_FLAG, Date.class.getCanonicalName()) // .type(AvroUtils._date()) .noDefault()// .endRecord(); }
@Override public void setupProperties() { super.setupProperties(); dieOnError.setValue(true); Schema s = SchemaBuilder.builder().record("Main").fields()// .name(FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(FIELD_INSERTION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_EXPIRATION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_NEXT_VISIBLE_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); schema.schema.setValue(s); numberOfMessages.setValue(32); peekMessages.setValue(true); deleteMessages.setValue(false); visibilityTimeoutInSeconds.setValue(30); }
@Override public void setupProperties() { super.setupProperties(); dieOnError.setValue(true); Schema s = SchemaBuilder.builder().record("Main").fields()// .name(FIELD_MESSAGE_ID).prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "100").type(AvroUtils._string()).noDefault()// .name(FIELD_MESSAGE_CONTENT).type(AvroUtils._string()).noDefault() // .name(FIELD_INSERTION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_EXPIRATION_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_NEXT_VISIBLE_TIME).prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss") .type(AvroUtils._date()).noDefault() // .name(FIELD_DEQUEUE_COUNT).type(AvroUtils._int()).noDefault() // .name(FIELD_POP_RECEIPT).type(AvroUtils._string()).noDefault() // .endRecord(); schema.schema.setValue(s); numberOfMessages.setValue(32); peekMessages.setValue(true); deleteMessages.setValue(false); visibilityTimeoutInSeconds.setValue(30); }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
public static DTEConverter createConverter(final Field f, final String mappedName) { Schema basicSchema = AvroUtils.unwrapIfNullable(f.schema()); AzureStorageDTEConverters converters = new AzureStorageDTEConverters(); DTEConverter dteConverter; if (AvroUtils.isSameType(basicSchema, AvroUtils._string()) || AvroUtils.isSameType(basicSchema, AvroUtils._character())) { dteConverter = converters.new StringDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int()) || AvroUtils.isSameType(basicSchema, AvroUtils._short()) || AvroUtils.isSameType(basicSchema, AvroUtils._byte())) { dteConverter = converters.new IntegerDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) { dteConverter = converters.new DateDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal()) || AvroUtils.isSameType(basicSchema, AvroUtils._double()) || AvroUtils.isSameType(basicSchema, AvroUtils._float())) { dteConverter = converters.new DoubleDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) { dteConverter = converters.new LongDTEConverter(f, mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) { dteConverter = converters.new BooleanDTEConverter(mappedName); } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) { dteConverter = converters.new ByteArrayDTEConverter(mappedName); } else { dteConverter = converters.new StringDTEConverter(f, mappedName); } return dteConverter; }
@Override public void setupProperties() { super.setupProperties(); dieOnError.setValue(true); // default Input schema Schema s = SchemaBuilder.record("Main").fields() // .name("PartitionKey").prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255")// $NON-NLS-3$ .prop(SchemaConstants.TALEND_IS_LOCKED, "true").type(AvroUtils._string()).noDefault() // .name("RowKey").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255")// $NON-NLS-3$ .prop(SchemaConstants.TALEND_IS_LOCKED, "true").type(AvroUtils._string()).noDefault() // .name("Timestamp").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss")// $NON-NLS-3$ .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "20")// $NON-NLS-3$ .type(AvroUtils._date()).noDefault() // .endRecord(); schema.schema.setValue(s); // useFilterExpression.setValue(false); producedFilter.setValue(""); producedFilter.setTaggedValue(ADD_QUOTES, true); }
@Override public void setupProperties() { super.setupProperties(); dieOnError.setValue(true); // default Input schema Schema s = SchemaBuilder.record("Main").fields() // .name("PartitionKey").prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255")// $NON-NLS-3$ .prop(SchemaConstants.TALEND_IS_LOCKED, "true").type(AvroUtils._string()).noDefault() // .name("RowKey").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true") .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255")// $NON-NLS-3$ .prop(SchemaConstants.TALEND_IS_LOCKED, "true").type(AvroUtils._string()).noDefault() // .name("Timestamp").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd hh:mm:ss")// $NON-NLS-3$ .prop(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "20")// $NON-NLS-3$ .type(AvroUtils._date()).noDefault() // .endRecord(); schema.schema.setValue(s); // useFilterExpression.setValue(false); producedFilter.setValue(""); producedFilter.setTaggedValue(ADD_QUOTES, true); }
private Schema getMakeRowSchema() { SchemaBuilder.FieldAssembler<Schema> fa = SchemaBuilder.builder().record("MakeRowRecord").fields() // .name("col_1").type().nullable().stringType().noDefault() // .name("col_2").type().nullable().stringType().noDefault() // .name("col_3").type().nullable().intType().noDefault() // .name("col_4").type().nullable().doubleType().noDefault() // .name("col_5").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd'T'HH:mm:ss'.000Z'").type(AvroUtils._date()).noDefault() // .name("col_6").type().nullable().stringType().noDefault() //the UI model may pass a empty string as the pattern, please see MetadataToolAvroHelper .name("col_7").prop(SchemaConstants.TALEND_COLUMN_PATTERN, "").type().nullable().longType().noDefault(); Schema schema = fa.endRecord(); return schema; }
public static Schema getBasicSchema(String pattern) { return SchemaBuilder.builder().record("Schema").fields() // .name("TestBoolean").type().booleanType().noDefault() // .name("TestByte").type(AvroUtils._byte()).noDefault() // .name("TestBytes").type(AvroUtils._bytes()).noDefault() // .name("TestChar").type(AvroUtils._character()).noDefault() // .name("TestDate").prop(SchemaConstants.TALEND_COLUMN_PATTERN, pattern)// .type(AvroUtils._date()).noDefault() // .name("TestDouble").type().doubleType().noDefault() // .name("TestFloat").type().floatType().noDefault() // .name("TestBigDecimal").type(AvroUtils._decimal()).noDefault()// .name("TestInteger").type().intType().noDefault() // .name("TestLong").type().longType().noDefault() // .name("TestObject").type(AvroUtils._bytes()).noDefault().endRecord(); }