public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
public static Schema removeImplicitRowTimeRowKeyFromSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { String fieldName = field.name(); fieldName = fieldName.substring(fieldName.indexOf('.') + 1); if (!fieldName.equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) && !fieldName.equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { schemaBuilder.field(fieldName, field.schema()); } } return schemaBuilder.build(); }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); delimitedDeserializer = new KsqlDelimitedDeserializer(orderSchema, recordLogger); }
/** * Rename field names to be consistent with the internal column names. */ public static Schema getAvroSerdeKsqlSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name().replace(".", "_"), field.schema()); } return schemaBuilder.build(); }
/** * Remove the alias when reading/writing from outside */ public static Schema getSchemaWithNoAlias(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { final String name = getFieldNameWithNoAlias(field); schemaBuilder.field(name, field.schema()); } return schemaBuilder.build(); }
@Test public void shouldCreateStringTimestampPolicyWhenTimestampFieldIsStringTypeAndFormatProvided() { final String field = "my_string_field"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); final TimestampExtractionPolicy extractionPolicy = TimestampExtractionPolicyFactory.create(schema, field, "yyyy-MM-DD"); assertThat(extractionPolicy, instanceOf(StringTimestampExtractionPolicy.class)); assertThat(extractionPolicy.timestampField(), equalTo(field.toUpperCase())); }
@Test public void shouldSupportFieldsWithQuotedStrings() { final String field = "my_string_field"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); final TimestampExtractionPolicy extractionPolicy = TimestampExtractionPolicyFactory.create(schema, "'"+ field+ "'", "'yyyy-MM-DD'"); assertThat(extractionPolicy, instanceOf(StringTimestampExtractionPolicy.class)); assertThat(extractionPolicy.timestampField(), equalTo(field.toUpperCase())); }
@Test public void shouldCreateLongTimestampPolicyWhenTimestampFieldIsOfTypeLong() { final String timestamp = "timestamp"; final Schema schema = schemaBuilder .field(timestamp.toUpperCase(), Schema.OPTIONAL_INT64_SCHEMA) .build(); final TimestampExtractionPolicy extractionPolicy = TimestampExtractionPolicyFactory.create(schema, timestamp, null); assertThat(extractionPolicy, instanceOf(LongColumnTimestampExtractionPolicy.class)); assertThat(extractionPolicy.timestampField(), equalTo(timestamp.toUpperCase())); }
@Test(expected = KsqlException.class) public void shouldFailIfStringTimestampTypeAndFormatNotSupplied() { final String field = "my_string_field"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); TimestampExtractionPolicyFactory.create(schema, field, null); }
@Test(expected = KsqlException.class) public void shouldThrowIfTimestampFieldTypeIsNotLongOrString() { final String field = "blah"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); TimestampExtractionPolicyFactory.create(schema, "'"+ field+ "'", null); }
private void addTestTopicAndSources() { final Schema schema1 = SchemaBuilder.struct().field("S1_F1", Schema.OPTIONAL_BOOLEAN_SCHEMA); ensureSource( DataSource.DataSourceType.KTABLE, "TEST_TABLE", "KAFKA_TOPIC_1", "KSQL_TOPIC_1", schema1); final Schema schema2 = SchemaBuilder.struct().field("S2_F1", Schema.OPTIONAL_STRING_SCHEMA); ensureSource( DataSource.DataSourceType.KSTREAM, "TEST_STREAM", "KAFKA_TOPIC_2", "KSQL_TOPIC_2", schema2); givenTopicExists("orders-topic"); }
public static Schema buildSchemaWithAlias(final Schema schema, final String alias) { final SchemaBuilder newSchema = SchemaBuilder.struct().name(schema.name()); for (final Field field : schema.fields()) { newSchema.field((alias + "." + field.name()), field.schema()); } return newSchema; }
private static Schema buildStructSchema(final Struct struct) { final SchemaBuilder structSchemaBuilder = SchemaBuilder.struct(); for (final Pair<String, Type> field: struct.getItems()) { structSchemaBuilder.field(field.getLeft(), getTypeSchema(field.getRight())); } return structSchemaBuilder.optional().build(); } }
@Test public void shouldTranslateNullValueCorrectly() { final Schema rowSchema = SchemaBuilder.struct() .field("INT", SchemaBuilder.OPTIONAL_INT32_SCHEMA) .optional() .build(); final Struct connectStruct = new Struct(rowSchema); final ConnectDataTranslator connectToKsqlTranslator = new ConnectDataTranslator(rowSchema); final GenericRow row = connectToKsqlTranslator.toKsqlRow(rowSchema, connectStruct); assertThat(row.getColumns().size(), equalTo(1)); assertThat(row.getColumnValue(0), is(nullValue())); }
@Test public void shouldIgnoreUnsupportedType() { final Schema connectSchema = SchemaBuilder .struct() .field("bytesField", Schema.BYTES_SCHEMA) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.fields().size(), equalTo(0)); } }
@Test public void shouldBuildCorrectMapField() { final Schema schema = SchemaBuilder .struct() .field("field", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final List<FieldInfo> entity = EntityUtil.buildSourceSchemaEntity(schema); assertThat(entity.size(), equalTo(1)); assertThat(entity.get(0).getName(), equalTo("field")); assertThat(entity.get(0).getSchema().getTypeName(), equalTo("MAP")); assertThat(entity.get(0).getSchema().getFields(), equalTo(Optional.empty())); assertThat(entity.get(0).getSchema().getMemberSchema().get().getTypeName(), equalTo("INTEGER")); }
private static List<FieldInfo> buildTestSchema(final int size) { final SchemaBuilder dataSourceBuilder = SchemaBuilder.struct().name("TestSchema"); for (int i = 0; i < size; i++) { dataSourceBuilder.field("f_" + i, SchemaUtil.getTypeSchema("STRING")); } return EntityUtil.buildSourceSchemaEntity(dataSourceBuilder.build()); } }
@Test public void shoudlReplacePrimitivesCorrectly() { final Schema schema = SchemaBuilder.struct() .field("COLUMN_NAME", Schema.OPTIONAL_INT64_SCHEMA) .optional() .build(); final AvroDataTranslator dataTranslator = new AvroDataTranslator(schema, KsqlConstants.DEFAULT_AVRO_SCHEMA_FULL_NAME); final GenericRow ksqlRow = new GenericRow(Collections.singletonList(123L)); final Struct struct = dataTranslator.toConnectRow(ksqlRow); assertThat(struct.get("COLUMN_NAME"), equalTo(123L)); final GenericRow translatedRow = dataTranslator.toKsqlRow(struct.schema(), struct); assertThat(translatedRow, equalTo(ksqlRow)); }
private StructuredDataSource buildDataSource(final String kafkaTopicName) { final Schema schema = SchemaBuilder.struct() .field("field0", Schema.OPTIONAL_INT32_SCHEMA) .build(); final KsqlTopic topic = new KsqlTopic("internal", kafkaTopicName, new KsqlJsonTopicSerDe(), true); return new KsqlStream<>( "query", "stream", schema, schema.fields().get(0), new MetadataTimestampExtractionPolicy(), topic, Serdes.String()); }