properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME()); properties.add(SCHEMA() + ".#." + SCHEMA_FROM()); properties.add(SCHEMA() + ".#." + SCHEMA_PROCTIME()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_TYPE()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_FROM()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_CLASS()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_SERIALIZED()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_TYPE()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_CLASS()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_SERIALIZED()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_DELAY());
private DescriptorProperties getValidatedProperties(Map<String, String> properties) { final DescriptorProperties descriptorProperties = new DescriptorProperties(true); descriptorProperties.putProperties(properties); // allow Kafka timestamps to be used, watermarks can not be received from source new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties); new KafkaValidator().validate(descriptorProperties); return descriptorProperties; }
@Override public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties); final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic); return createKafkaTableSource( descriptorProperties.getTableSchema(SCHEMA()), SchemaValidator.deriveProctimeAttribute(descriptorProperties), SchemaValidator.deriveRowtimeAttributes(descriptorProperties), SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(deserializationSchema.getProducedType())), topic, getKafkaProperties(descriptorProperties), deserializationSchema, startupOptions.startupMode, startupOptions.specificOffsets); }
properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME());
@Override public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA()); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = SchemaValidator.deriveRowtimeAttributes(descriptorProperties); // see also FLINK-9870 if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() || checkForCustomFieldMapping(descriptorProperties, schema)) { throw new TableException("Time attributes and custom field mappings are not supported yet."); } return createKafkaTableSink( schema, topic, getKafkaProperties(descriptorProperties), getFlinkKafkaPartitioner(descriptorProperties), getSerializationSchema(properties)); }
@Override public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); return createElasticsearchUpsertTableSink( descriptorProperties.isValue(UPDATE_MODE(), UPDATE_MODE_VALUE_APPEND()), descriptorProperties.getTableSchema(SCHEMA()), getHosts(descriptorProperties), descriptorProperties.getString(CONNECTOR_INDEX), descriptorProperties.getString(CONNECTOR_DOCUMENT_TYPE), descriptorProperties.getOptionalString(CONNECTOR_KEY_DELIMITER).orElse(DEFAULT_KEY_DELIMITER), descriptorProperties.getOptionalString(CONNECTOR_KEY_NULL_LITERAL).orElse(DEFAULT_KEY_NULL_LITERAL), getSerializationSchema(properties), SUPPORTED_CONTENT_TYPE, getFailureHandler(descriptorProperties), getSinkOptions(descriptorProperties)); }
private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) { final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type return fieldMapping.size() != schema.getFieldNames().length || !fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue())); }
properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME());
@Override public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA()); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = SchemaValidator.deriveRowtimeAttributes(descriptorProperties); // see also FLINK-9870 if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() || checkForCustomFieldMapping(descriptorProperties, schema)) { throw new TableException("Time attributes and custom field mappings are not supported yet."); } return createKafkaTableSink( schema, topic, getKafkaProperties(descriptorProperties), getFlinkKafkaPartitioner(descriptorProperties), getSerializationSchema(properties)); }
public void putSchemaIntoProperties(RichTableSchema schema) { try { byte[] serialized = InstantiationUtil.serializeObject(schema); String encoded = Base64.getEncoder().encodeToString(serialized); setString(SchemaValidator.SCHEMA(), encoded); } catch (IOException ioe) { LOG.error("Exception when put rich table schema to configuration: {}", ioe.getCause()); throw new RuntimeException(ioe.getMessage()); } }
private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) { final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type return fieldMapping.size() != schema.getFieldNames().length || !fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue())); }
properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME()); properties.add(SCHEMA() + ".#." + SCHEMA_FROM()); properties.add(SCHEMA() + ".#." + SCHEMA_PROCTIME()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_TYPE()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_FROM()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_CLASS()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_SERIALIZED()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_TYPE()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_CLASS()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_SERIALIZED()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_DELAY());
properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME());
private DescriptorProperties getValidatedProperties(Map<String, String> properties) { final DescriptorProperties descriptorProperties = new DescriptorProperties(true); descriptorProperties.putProperties(properties); new StreamTableDescriptorValidator(true, false, true).validate(descriptorProperties); new SchemaValidator(true, false, false).validate(descriptorProperties); new ElasticsearchValidator().validate(descriptorProperties); return descriptorProperties; }
@Override public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties); final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic); return createKafkaTableSource( descriptorProperties.getTableSchema(SCHEMA()), SchemaValidator.deriveProctimeAttribute(descriptorProperties), SchemaValidator.deriveRowtimeAttributes(descriptorProperties), SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(deserializationSchema.getProducedType())), topic, getKafkaProperties(descriptorProperties), deserializationSchema, startupOptions.startupMode, startupOptions.specificOffsets); }
@Override public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA()); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = SchemaValidator.deriveRowtimeAttributes(descriptorProperties); // see also FLINK-9870 if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() || checkForCustomFieldMapping(descriptorProperties, schema)) { throw new TableException("Time attributes and custom field mappings are not supported yet."); } return createKafkaTableSink( schema, topic, getKafkaProperties(descriptorProperties), getFlinkKafkaPartitioner(descriptorProperties), getSerializationSchema(properties)); }
public RichTableSchema readSchemaFromProperties(ClassLoader classLoader) { try { String encoded = getString(SchemaValidator.SCHEMA(), null); return InstantiationUtil.deserializeObject(Base64.getDecoder().decode(encoded), classLoader); } catch (ClassNotFoundException | IOException cne) { LOG.error("Exception when put rich table schema to configuration: {}", cne.getCause()); throw new RuntimeException(cne.getMessage()); } }
private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) { final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type return fieldMapping.size() != schema.getFieldNames().length || !fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue())); }
properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME()); properties.add(SCHEMA() + ".#." + SCHEMA_FROM()); properties.add(SCHEMA() + ".#." + SCHEMA_PROCTIME()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_TYPE()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_FROM()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_CLASS()); properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_SERIALIZED()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_TYPE()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_CLASS()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_SERIALIZED()); properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_DELAY());
private DescriptorProperties getValidatedProperties(Map<String, String> properties) { final DescriptorProperties descriptorProperties = new DescriptorProperties(true); descriptorProperties.putProperties(properties); // allow Kafka timestamps to be used, watermarks can not be received from source new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties); new KafkaValidator().validate(descriptorProperties); return descriptorProperties; }