ViewEntry(String name, DescriptorProperties properties) { super(name, properties); query = properties.getString(TABLES_QUERY); }
private Properties getKafkaProperties(DescriptorProperties descriptorProperties) { final Properties kafkaProperties = new Properties(); final List<Map<String, String>> propsList = descriptorProperties.getFixedIndexedProperties( CONNECTOR_PROPERTIES, Arrays.asList(CONNECTOR_PROPERTIES_KEY, CONNECTOR_PROPERTIES_VALUE)); propsList.forEach(kv -> kafkaProperties.put( descriptorProperties.getString(kv.get(CONNECTOR_PROPERTIES_KEY)), descriptorProperties.getString(kv.get(CONNECTOR_PROPERTIES_VALUE)) )); return kafkaProperties; }
private List<Host> getHosts(DescriptorProperties descriptorProperties) { final List<Map<String, String>> hosts = descriptorProperties.getFixedIndexedProperties( CONNECTOR_HOSTS, Arrays.asList(CONNECTOR_HOSTS_HOSTNAME, CONNECTOR_HOSTS_PORT, CONNECTOR_HOSTS_PROTOCOL)); return hosts.stream() .map(host -> new Host( descriptorProperties.getString(host.get(CONNECTOR_HOSTS_HOSTNAME)), descriptorProperties.getInt(host.get(CONNECTOR_HOSTS_PORT)), descriptorProperties.getString(host.get(CONNECTOR_HOSTS_PROTOCOL)))) .collect(Collectors.toList()); }
private static FunctionEntry create(DescriptorProperties properties) { properties.validateString(FUNCTIONS_NAME, false, 1); final String name = properties.getString(FUNCTIONS_NAME); final DescriptorProperties cleanedProperties = properties.withoutKeys(Collections.singletonList(FUNCTIONS_NAME)); return new FunctionEntry(name, cleanedProperties); }
@Override public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); // create and configure if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) { return new AvroRowDeserializationSchema( descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class)); } else { return new AvroRowDeserializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA)); } }
@Override public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); // create and configure if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) { return new AvroRowSerializationSchema( descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class)); } else { return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA)); } }
@Override public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); return createElasticsearchUpsertTableSink( descriptorProperties.isValue(UPDATE_MODE(), UPDATE_MODE_VALUE_APPEND()), descriptorProperties.getTableSchema(SCHEMA()), getHosts(descriptorProperties), descriptorProperties.getString(CONNECTOR_INDEX), descriptorProperties.getString(CONNECTOR_DOCUMENT_TYPE), descriptorProperties.getOptionalString(CONNECTOR_KEY_DELIMITER).orElse(DEFAULT_KEY_DELIMITER), descriptorProperties.getOptionalString(CONNECTOR_KEY_NULL_LITERAL).orElse(DEFAULT_KEY_NULL_LITERAL), getSerializationSchema(properties), SUPPORTED_CONTENT_TYPE, getFailureHandler(descriptorProperties), getSinkOptions(descriptorProperties)); }
private TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) { if (descriptorProperties.containsKey(JsonValidator.FORMAT_SCHEMA)) { return (RowTypeInfo) descriptorProperties.getType(JsonValidator.FORMAT_SCHEMA); } else if (descriptorProperties.containsKey(JsonValidator.FORMAT_JSON_SCHEMA)) { return JsonRowSchemaConverter.convert(descriptorProperties.getString(JsonValidator.FORMAT_JSON_SCHEMA)); } else { return deriveSchema(descriptorProperties.asMap()).toRowType(); } }
@Override public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA()); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = SchemaValidator.deriveRowtimeAttributes(descriptorProperties); // see also FLINK-9870 if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() || checkForCustomFieldMapping(descriptorProperties, schema)) { throw new TableException("Time attributes and custom field mappings are not supported yet."); } return createKafkaTableSink( schema, topic, getKafkaProperties(descriptorProperties), getFlinkKafkaPartitioner(descriptorProperties), getSerializationSchema(properties)); }
@Override public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties); final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic); return createKafkaTableSource( descriptorProperties.getTableSchema(SCHEMA()), SchemaValidator.deriveProctimeAttribute(descriptorProperties), SchemaValidator.deriveRowtimeAttributes(descriptorProperties), SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(deserializationSchema.getProducedType())), topic, getKafkaProperties(descriptorProperties), deserializationSchema, startupOptions.startupMode, startupOptions.specificOffsets); }
descriptorProperties.isValue(timestampKey, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD)) { final String aliasName = descriptorProperties .getString(SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_FROM); builder.field(aliasName, fieldType);
static AthenaXTableSinkProvider getProvider(ExternalCatalogTable table) { DescriptorProperties properties = new DescriptorProperties(true); table.addProperties(properties); String connectorType = properties.getString(ConnectorDescriptorValidator.CONNECTOR_TYPE()); return PROVIDERS.get(connectorType); } }
@Override public StreamTableSource<Row> create(Map<String, String> properties) { DescriptorProperties params = new DescriptorProperties(true); params.putProperties(properties); TableSchema schema = params.getTableSchema(TOPIC_SCHEMA_KEY); String topic = params.getString(TOPIC_NAME_KEY); Properties conf = new Properties(); conf.putAll(params.getPrefix(KAFKA_CONFIG_PREFIX)); return new JsonTableSource(topic, conf, schema); } }
@Override public AppendStreamTableSink<Row> getAppendStreamTableSink(ExternalCatalogTable table) throws IOException { DescriptorProperties params = new DescriptorProperties(true); table.addProperties(params); String topic = params.getString(TOPIC_NAME_KEY); Properties conf = new Properties(); conf.putAll(params.getPrefix(KAFKA_CONFIG_PREFIX)); String partitionerClass = params.getOptionalString(PARTITIONER_CLASS_NAME_KEY) .orElse(PARTITIONER_CLASS_NAME_DEFAULT); FlinkKafkaPartitioner<Row> partitioner; try { partitioner = KafkaUtils.instantiatePartitioner(partitionerClass); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { throw new IOException(e); } return new Kafka09JsonTableSink(topic, conf, partitioner); }
@Override public TableSource<Row> create(Map<String, String> properties) { DescriptorProperties params = new DescriptorProperties(true); params.putProperties(properties); TableSchema schema = params.getTableSchema(TABLE_SCHEMA_CONNECTOR_PROPERTY); List<Row> rows = deserializeRows(params.getString(TABLE_DATA_CONNECTOR_PROPERTY)); return new MockTableSource(rows, new RowTypeInfo(schema.getTypes(), schema.getColumnNames())); }
@Override public TaxiRideTableSource createStreamTableSource(Map<String, String> properties) { DescriptorProperties params = getValidatedProperties(properties); return new TaxiRideTableSource( params.getString(CONNECTOR_PATH), params.getOptionalInt(CONNECTOR_MAX_EVENT_DELAY_SECS).orElse(0), params.getOptionalInt(CONNECTOR_SERVING_SPEED_FACTOR).orElse(1) ); }
@Override public TaxiFareTableSource createStreamTableSource(Map<String, String> properties) { DescriptorProperties params = getValidatedProperties(properties); return new TaxiFareTableSource( params.getString(CONNECTOR_PATH), params.getOptionalInt(CONNECTOR_MAX_EVENT_DELAY_SECS).orElse(0), params.getOptionalInt(CONNECTOR_SERVING_SPEED_FACTOR).orElse(1) ); }
@Override public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); // create and configure if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) { return new AvroRowSerializationSchema( descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class)); } else { return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA)); } }