public static Schema removeImplicitRowTimeRowKeyFromSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { String fieldName = field.name(); fieldName = fieldName.substring(fieldName.indexOf('.') + 1); if (!fieldName.equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) && !fieldName.equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { schemaBuilder.field(fieldName, field.schema()); } } return schemaBuilder.build(); }
private Schema toKsqlArraySchema(final Schema schema) { return SchemaBuilder.array( toKsqlFieldSchema(schema.valueSchema()) ).optional().build(); }
public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
@Test public void shouldSupportFieldsWithQuotedStrings() { final String field = "my_string_field"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); final TimestampExtractionPolicy extractionPolicy = TimestampExtractionPolicyFactory.create(schema, "'"+ field+ "'", "'yyyy-MM-DD'"); assertThat(extractionPolicy, instanceOf(StringTimestampExtractionPolicy.class)); assertThat(extractionPolicy.timestampField(), equalTo(field.toUpperCase())); }
@Test public void shouldCreateStringTimestampPolicyWhenTimestampFieldIsStringTypeAndFormatProvided() { final String field = "my_string_field"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); final TimestampExtractionPolicy extractionPolicy = TimestampExtractionPolicyFactory.create(schema, field, "yyyy-MM-DD"); assertThat(extractionPolicy, instanceOf(StringTimestampExtractionPolicy.class)); assertThat(extractionPolicy.timestampField(), equalTo(field.toUpperCase())); }
@Test public void shouldDeserializeConnectMapWithInt64Key() { shouldDeserializeConnectTypeCorrectly( SchemaBuilder.map(Schema.INT64_SCHEMA, Schema.INT32_SCHEMA).optional().build(), ImmutableMap.of( 1L, 10, 2L, 20, 3L, 30), SchemaBuilder.map( Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_INT32_SCHEMA ).optional().build(), ImmutableMap.of("1", 10, "2", 20, "3", 30) ); }
@Test(expected = KsqlException.class) public void shouldThrowIfTimestampFieldTypeIsNotLongOrString() { final String field = "blah"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); TimestampExtractionPolicyFactory.create(schema, "'"+ field+ "'", null); }
@Test public void shouldDeserializeConnectMapWithBooleanKey() { shouldDeserializeConnectTypeCorrectly( SchemaBuilder.map(Schema.BOOLEAN_SCHEMA, Schema.INT32_SCHEMA).optional().build(), ImmutableMap.of( true, 10, false, 20), SchemaBuilder.map( Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_INT32_SCHEMA ).optional().build(), ImmutableMap.of("true", 10, "false", 20) ); }
@Test(expected = KsqlException.class) public void shouldFailIfStringTimestampTypeAndFormatNotSupplied() { final String field = "my_string_field"; final Schema schema = schemaBuilder .field(field.toUpperCase(), Schema.OPTIONAL_STRING_SCHEMA) .build(); TimestampExtractionPolicyFactory.create(schema, field, null); }
@Test public void shouldGetCorrectArrayKsqlType() throws Exception { final Schema arraySchema = SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build(); final Type type = TypeUtil.getKsqlType(arraySchema); assertThat(type.getKsqlType(), equalTo(Type.KsqlType.ARRAY)); assertThat(type, instanceOf(Array.class)); assertThat(((Array) type).getItemType().getKsqlType(), equalTo(Type.KsqlType.DOUBLE)); }
@Test public void shouldGetCorrectJavaClassForArray() { final Class arrayClazz = SchemaUtil.getJavaType(SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()); assertThat(arrayClazz, equalTo(List.class)); }
@Test public void shouldGetCorrectJavaClassForMap() { final Class mapClazz = SchemaUtil.getJavaType(SchemaBuilder.map(Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()); assertThat(mapClazz, equalTo(Map.class)); }
@Test public void shouldGetCorrectSqlTypeNameForArray() { assertThat(SchemaUtil.getSqlTypeName(SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()), equalTo("ARRAY<DOUBLE>")); }
@Test public void shouldGetTheCorrectJavaTypeForArray() { final Schema schema = SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build(); final Class javaClass = SchemaUtil.getJavaType(schema); assertThat(javaClass, equalTo(List.class)); }
@Test public void shouldGetTheCorrectJavaTypeForMap() { final Schema schema = SchemaBuilder.map(Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build(); final Class javaClass = SchemaUtil.getJavaType(schema); assertThat(javaClass, equalTo(Map.class)); }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); }
@Test public void shouldDeserializeMapToMap() { shouldDeserializeTypeCorrectly( org.apache.avro.SchemaBuilder.map().values().intType(), ImmutableMap.of("one", 1, "two", 2, "three", 3), SchemaBuilder.map( Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_INT32_SCHEMA ).optional().build() ); }
private static List<FieldInfo> buildTestSchema(final int size) { final SchemaBuilder dataSourceBuilder = SchemaBuilder.struct().name("TestSchema"); for (int i = 0; i < size; i++) { dataSourceBuilder.field("f_" + i, SchemaUtil.getTypeSchema("STRING")); } return EntityUtil.buildSourceSchemaEntity(dataSourceBuilder.build()); } }
private StructuredDataSource buildDataSource(final String kafkaTopicName) { final Schema schema = SchemaBuilder.struct() .field("field0", Schema.OPTIONAL_INT32_SCHEMA) .build(); final KsqlTopic topic = new KsqlTopic("internal", kafkaTopicName, new KsqlJsonTopicSerDe(), true); return new KsqlStream<>( "query", "stream", schema, schema.fields().get(0), new MetadataTimestampExtractionPolicy(), topic, Serdes.String()); }
@Test public void shouldSerializeArray() { shouldSerializeTypeCorrectly( SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build(), ImmutableList.of(1, 2, 3), org.apache.avro.SchemaBuilder.array().items( org.apache.avro.SchemaBuilder.builder() .unionOf().nullType().and().intType().endUnion()) ); }