/** * Returns a table schema under the given key if it exists. */ public Optional<TableSchema> getOptionalTableSchema(String key) { // filter for number of fields final int fieldCount = properties.keySet().stream() .filter((k) -> k.startsWith(key) && k.endsWith('.' + TABLE_SCHEMA_NAME)) .mapToInt((k) -> 1) .sum(); if (fieldCount == 0) { return Optional.empty(); } // validate fields and build schema final TableSchema.Builder schemaBuilder = TableSchema.builder(); for (int i = 0; i < fieldCount; i++) { final String nameKey = key + '.' + i + '.' + TABLE_SCHEMA_NAME; final String typeKey = key + '.' + i + '.' + TABLE_SCHEMA_TYPE; final String name = optionalGet(nameKey).orElseThrow(exceptionSupplier(nameKey)); final TypeInformation<?> type = optionalGet(typeKey) .map(TypeStringUtils::readTypeInfo) .orElseThrow(exceptionSupplier(typeKey)); schemaBuilder.field(name, type); } return Optional.of(schemaBuilder.build()); }
@Test public void testSchemaDerivation() { final Map<String, String> properties = new HashMap<>(); properties.put("schema.0.name", "otherField"); properties.put("schema.0.type", "VARCHAR"); properties.put("schema.0.from", "csvField"); properties.put("schema.1.name", "abcField"); properties.put("schema.1.type", "VARCHAR"); properties.put("schema.2.name", "p"); properties.put("schema.2.type", "TIMESTAMP"); properties.put("schema.2.proctime", "true"); properties.put("schema.3.name", "r"); properties.put("schema.3.type", "TIMESTAMP"); properties.put("schema.3.rowtime.timestamps.type", "from-source"); properties.put("schema.3.rowtime.watermarks.type", "from-source"); final TableSchema actualSchema = TableFormatFactoryBase.deriveSchema(properties); final TableSchema expectedSchema = TableSchema.builder() .field("csvField", Types.STRING) // aliased .field("abcField", Types.STRING) .build(); assertEquals(expectedSchema, actualSchema); }
@Test public void testSchemaDerivationWithRowtime() { final Map<String, String> properties = new HashMap<>(); properties.put("schema.0.name", "otherField"); properties.put("schema.0.type", "VARCHAR"); properties.put("schema.0.from", "csvField"); properties.put("schema.1.name", "abcField"); properties.put("schema.1.type", "VARCHAR"); properties.put("schema.2.name", "p"); properties.put("schema.2.type", "TIMESTAMP"); properties.put("schema.2.proctime", "true"); properties.put("schema.3.name", "r"); properties.put("schema.3.type", "TIMESTAMP"); properties.put("schema.3.rowtime.timestamps.type", "from-field"); // from-field strategy properties.put("schema.3.rowtime.timestamps.from", "myTime"); properties.put("schema.3.rowtime.watermarks.type", "from-source"); final TableSchema actualSchema = TableFormatFactoryBase.deriveSchema(properties); final TableSchema expectedSchema = TableSchema.builder() .field("csvField", Types.STRING) // aliased .field("abcField", Types.STRING) .field("myTime", Types.SQL_TIMESTAMP) .build(); assertEquals(expectedSchema, actualSchema); } }
private static TableSchema removeTimeAttributes(TableSchema schema) { final TableSchema.Builder builder = TableSchema.builder(); for (int i = 0; i < schema.getFieldCount(); i++) { final TypeInformation<?> type = schema.getFieldTypes()[i]; final TypeInformation<?> convertedType; if (FlinkTypeFactory.isTimeIndicatorType(type)) { convertedType = Types.SQL_TIMESTAMP; } else { convertedType = type; } builder.field(schema.getFieldNames()[i], convertedType); } return builder.build(); } }
descriptorProperties.putProperties(properties); final TableSchema.Builder builder = TableSchema.builder();
protected TableSchema createTestSchema() { return TableSchema.builder() .field(FIELD_KEY, Types.LONG()) .field(FIELD_FRUIT_NAME, Types.STRING()) .field(FIELD_COUNT, Types.DECIMAL()) .field(FIELD_TS, Types.SQL_TIMESTAMP()) .build(); }
final TableSchema schema = TableSchema.builder() .field(FRUIT_NAME, Types.STRING()) .field(COUNT, Types.DECIMAL())
/** * Returns a table schema under the given key if it exists. */ public Optional<TableSchema> getOptionalTableSchema(String key) { // filter for number of fields final int fieldCount = properties.keySet().stream() .filter((k) -> k.startsWith(key) && k.endsWith('.' + TABLE_SCHEMA_NAME)) .mapToInt((k) -> 1) .sum(); if (fieldCount == 0) { return Optional.empty(); } // validate fields and build schema final TableSchema.Builder schemaBuilder = TableSchema.builder(); for (int i = 0; i < fieldCount; i++) { final String nameKey = key + '.' + i + '.' + TABLE_SCHEMA_NAME; final String typeKey = key + '.' + i + '.' + TABLE_SCHEMA_TYPE; final String name = optionalGet(nameKey).orElseThrow(exceptionSupplier(nameKey)); final InternalType type = optionalGet(typeKey) .map(TypeStringUtils::readDataType) .orElseThrow(exceptionSupplier(typeKey)); schemaBuilder.field(name, type); } return Optional.of(schemaBuilder.build()); }
/** * Returns a table schema under the given key if it exists. */ public Optional<TableSchema> getOptionalTableSchema(String key) { // filter for number of fields final int fieldCount = properties.keySet().stream() .filter((k) -> k.startsWith(key) && k.endsWith('.' + TABLE_SCHEMA_NAME)) .mapToInt((k) -> 1) .sum(); if (fieldCount == 0) { return Optional.empty(); } // validate fields and build schema final TableSchema.Builder schemaBuilder = TableSchema.builder(); for (int i = 0; i < fieldCount; i++) { final String nameKey = key + '.' + i + '.' + TABLE_SCHEMA_NAME; final String typeKey = key + '.' + i + '.' + TABLE_SCHEMA_TYPE; final String name = optionalGet(nameKey).orElseThrow(exceptionSupplier(nameKey)); final TypeInformation<?> type = optionalGet(typeKey) .map(TypeStringUtils::readTypeInfo) .orElseThrow(exceptionSupplier(typeKey)); schemaBuilder.field(name, type); } return Optional.of(schemaBuilder.build()); }
descriptorProperties.putProperties(properties); final TableSchema.Builder builder = TableSchema.builder();
descriptorProperties.putProperties(properties); final TableSchema.Builder builder = TableSchema.builder();