private static SchemaBuilder handleParametrizedType(final Type type) { if (type instanceof ParameterizedType) { final ParameterizedType parameterizedType = (ParameterizedType) type; if (parameterizedType.getRawType() == Map.class) { return SchemaBuilder.map(getSchemaFromType( parameterizedType.getActualTypeArguments()[0]), getSchemaFromType(parameterizedType.getActualTypeArguments()[1])); } else if (parameterizedType.getRawType() == List.class) { return SchemaBuilder.array(getSchemaFromType( parameterizedType.getActualTypeArguments()[0])); } } throw new KsqlException("Type is not supported: " + type); } }
private Schema toKsqlArraySchema(final Schema schema) { return SchemaBuilder.array( toKsqlFieldSchema(schema.valueSchema()) ).optional().build(); }
@Test public void shouldGetCorrectArrayKsqlType() throws Exception { final Schema arraySchema = SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build(); final Type type = TypeUtil.getKsqlType(arraySchema); assertThat(type.getKsqlType(), equalTo(Type.KsqlType.ARRAY)); assertThat(type, instanceOf(Array.class)); assertThat(((Array) type).getItemType().getKsqlType(), equalTo(Type.KsqlType.DOUBLE)); }
@Test public void shouldGetCorrectJavaClassForArray() { final Class arrayClazz = SchemaUtil.getJavaType(SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()); assertThat(arrayClazz, equalTo(List.class)); }
@Test public void shouldGetCorrectSqlTypeNameForArray() { assertThat(SchemaUtil.getSqlTypeName(SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()), equalTo("ARRAY<DOUBLE>")); }
@Test public void shouldGetTheCorrectJavaTypeForArray() { final Schema schema = SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build(); final Class javaClass = SchemaUtil.getJavaType(schema); assertThat(javaClass, equalTo(List.class)); }
@Test public void shouldGetCorrectSqlType() { final String sqlType1 = SchemaUtil.getSqlTypeName(Schema.OPTIONAL_BOOLEAN_SCHEMA); final String sqlType2 = SchemaUtil.getSqlTypeName(Schema.OPTIONAL_INT32_SCHEMA); final String sqlType3 = SchemaUtil.getSqlTypeName(Schema.OPTIONAL_INT64_SCHEMA); final String sqlType4 = SchemaUtil.getSqlTypeName(Schema.OPTIONAL_FLOAT64_SCHEMA); final String sqlType5 = SchemaUtil.getSqlTypeName(Schema.OPTIONAL_STRING_SCHEMA); final String sqlType6 = SchemaUtil.getSqlTypeName(SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()); final String sqlType7 = SchemaUtil.getSqlTypeName(SchemaBuilder.map(Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()); assertThat("Invalid SQL type.", sqlType1, equalTo("BOOLEAN")); assertThat("Invalid SQL type.", sqlType2, equalTo("INT")); assertThat("Invalid SQL type.", sqlType3, equalTo("BIGINT")); assertThat("Invalid SQL type.", sqlType4, equalTo("DOUBLE")); assertThat("Invalid SQL type.", sqlType5, equalTo("VARCHAR")); assertThat("Invalid SQL type.", sqlType6, equalTo("ARRAY<DOUBLE>")); assertThat("Invalid SQL type.", sqlType7, equalTo("MAP<VARCHAR,DOUBLE>")); }
@Test public void shouldBuildCorrectArrayField() { final Schema schema = SchemaBuilder .struct() .field("field", SchemaBuilder.array(SchemaBuilder.INT64_SCHEMA)) .build(); final List<FieldInfo> entity = EntityUtil.buildSourceSchemaEntity(schema); assertThat(entity.size(), equalTo(1)); assertThat(entity.get(0).getName(), equalTo("field")); assertThat(entity.get(0).getSchema().getTypeName(), equalTo("ARRAY")); assertThat(entity.get(0).getSchema().getFields(), equalTo(Optional.empty())); assertThat(entity.get(0).getSchema().getMemberSchema().get().getTypeName(), equalTo("BIGINT")); }
protected List<SchemaAndValueField> schemasAndValuesForArrayTypesWithNullValues() { return Arrays.asList( new SchemaAndValueField("int_array", SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build(), null), new SchemaAndValueField("bigint_array", SchemaBuilder.array(Schema.OPTIONAL_INT64_SCHEMA).optional().build(), null), new SchemaAndValueField("text_array", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build(), null), new SchemaAndValueField("char_array", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build(), null), new SchemaAndValueField("varchar_array", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build(), null), new SchemaAndValueField("date_array", SchemaBuilder.array(Date.builder().optional().schema()).optional().build(), null), new SchemaAndValueField("numeric_array", SchemaBuilder.array(Decimal.builder(2).parameter(TestHelper.PRECISION_PARAMETER_KEY, "10").optional().build()).optional().build(), null), new SchemaAndValueField("citext_array", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build(), null) ); }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .field("arraycol".toUpperCase(), SchemaBuilder.array(org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .field("mapcol".toUpperCase(), SchemaBuilder .map(org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA, org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .build(); }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .field("arraycol".toUpperCase(), SchemaBuilder.array(org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .field("mapcol".toUpperCase(), SchemaBuilder.map(org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA, org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .build(); ksqlJsonDeserializer = new KsqlJsonDeserializer( orderSchema, false, recordLogger); }
@Test public void shouldDeserializeArrayToArray() { shouldDeserializeTypeCorrectly( org.apache.avro.SchemaBuilder.array().items().intType(), ImmutableList.of(1, 2, 3, 4, 5, 6), SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build() ); }
protected List<SchemaAndValueField> schemaAndValuesForPostgisArrayTypes() { Schema geomSchema = Geometry.builder() .optional() .build(); List<Struct> values = Arrays.asList( // 'GEOMETRYCOLLECTION EMPTY'::postgis.geometry Geometry.createValue(geomSchema, DatatypeConverter.parseHexBinary("010700000000000000"), null), // 'POLYGON((166.51 -46.64, 178.52 -46.64, 178.52 -34.45, 166.51 -34.45, 166.51 -46.64))'::postgis.geometry Geometry.createValue(geomSchema, DatatypeConverter.parseHexBinary("01030000000100000005000000B81E85EB51D0644052B81E85EB5147C0713D0AD7A350664052B81E85EB5147C0713D0AD7A35066409A999999993941C0B81E85EB51D064409A999999993941C0B81E85EB51D0644052B81E85EB5147C0"), null) ); return Arrays.asList( // geometries are encoded here as HexEWKB new SchemaAndValueField("ga", SchemaBuilder.array(geomSchema).optional().build(), values ), new SchemaAndValueField("gann", SchemaBuilder.array(geomSchema).build(), values ) ); }
@Test public void shouldTranslateArray() { final Schema connectSchema = SchemaBuilder .struct() .field("arrayField", SchemaBuilder.array(Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.isOptional(), is(true)); assertThat(arraySchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldReplaceNullWithNull() { final Schema schema = SchemaBuilder.struct() .field( "COLUMN_NAME", SchemaBuilder.array(Schema.OPTIONAL_INT64_SCHEMA).optional().build()) .optional() .build(); final AvroDataTranslator dataTranslator = new AvroDataTranslator(schema, KsqlConstants.DEFAULT_AVRO_SCHEMA_FULL_NAME); final GenericRow ksqlRow = new GenericRow(Collections.singletonList(null)); final Struct struct = dataTranslator.toConnectRow(ksqlRow); assertThat(struct.get("COLUMN_NAME"), nullValue()); final GenericRow translatedRow = dataTranslator.toKsqlRow(struct.schema(), struct); assertThat(translatedRow, equalTo(ksqlRow)); }
@Test public void shouldGetCorrectSqlTypeNameForStruct() { final Schema structSchema = SchemaBuilder.struct() .field("COL1", Schema.OPTIONAL_STRING_SCHEMA) .field("COL2", Schema.OPTIONAL_INT32_SCHEMA) .field("COL3", Schema.OPTIONAL_FLOAT64_SCHEMA) .field("COL4", SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .field("COL5", SchemaBuilder.map(Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .build(); assertThat(SchemaUtil.getSqlTypeName(structSchema), equalTo( "STRUCT<COL1 VARCHAR, COL2 INT, COL3 DOUBLE, COL4 ARRAY<DOUBLE>, COL5 MAP<VARCHAR,DOUBLE>>")); }
@Test public void shouldSerializeArray() { shouldSerializeTypeCorrectly( SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build(), ImmutableList.of(1, 2, 3), org.apache.avro.SchemaBuilder.array().items( org.apache.avro.SchemaBuilder.builder() .unionOf().nullType().and().intType().endUnion()) ); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/pop/#up._S_pop">MongoDB operator array update $pop</a> */ @Test public void shouldTransformOperationPop() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$pop': {dataArrayOfStr: -1}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("c", "e")); }
/** * @see <a href="https://docs.mongodb.com/v3.6/reference/operator/update/pull/#pull">MongoDB operator array update $pull</a> */ @Test public void shouldTransformOperationPull() throws InterruptedException { SourceRecord updateRecord = executeSimpleUpdateOperation( "{'$pull': {dataArrayOfStr: {$in: ['c']}}}" ); final SourceRecord transformedUpdate = transformation.apply(updateRecord); final Struct transformedUpdateValue = (Struct) transformedUpdate.value(); final Schema valueSchema = transformedUpdate.valueSchema(); VerifyRecord.assertConnectSchemasAreEqual("id", valueSchema.field("id").schema(), Schema.OPTIONAL_INT32_SCHEMA); VerifyRecord.assertConnectSchemasAreEqual("dataArrayOfStr", valueSchema.field("dataArrayOfStr").schema(), SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()); assertThat(transformedUpdateValue.get("id")).isEqualTo(1); assertThat(transformedUpdateValue.get("dataArrayOfStr")).isEqualTo(Arrays.asList("a", "e")); }
@Test public void shouldCreateSchemaForEmptyArrayEncodingArray() throws Exception { final BsonDocument val = BsonDocument.parse(EMPTY_ARRAY); final MongoDataConverter arrayConverter = new MongoDataConverter(ArrayEncoding.ARRAY); for (Entry<String, BsonValue> entry : val.entrySet()) { arrayConverter.addFieldSchema(entry, builder); } final Schema arraySchema = builder.build(); assertThat(arraySchema) .isEqualTo( SchemaBuilder.struct().name("array") .field("_id", Schema.OPTIONAL_INT32_SCHEMA) .field("f", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()) .build()); }