public static String getSchemaDefinitionString(final Schema schema) { return schema.fields().stream() .map(field -> field.name() + " : " + getSqlTypeName(field.schema())) .collect(Collectors.joining(", ", "[", "]")); }
private static String getStructString(final Schema schema) { return schema.fields().stream() .map(field -> field.name() + " " + getSqlTypeName(field.schema())) .collect(Collectors.joining(", ", "STRUCT<", ">")); }
protected Object convertTimestampToLocalDateTime(Column column, Field fieldDefn, Object data) { if (data == null && !fieldDefn.schema().isOptional()) { return null; } if (!(data instanceof Timestamp)) { return data; } return ((Timestamp)data).toLocalDateTime(); } }
public static Schema removeImplicitRowTimeRowKeyFromSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { String fieldName = field.name(); fieldName = fieldName.substring(fieldName.indexOf('.') + 1); if (!fieldName.equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) && !fieldName.equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { schemaBuilder.field(fieldName, field.schema()); } } return schemaBuilder.build(); }
private boolean compareStructSchema(final Schema schema1, final Schema schema2) { if (schema1.fields().size() != schema2.fields().size()) { return false; } for (int i = 0; i < schema1.fields().size(); i++) { if (!schema1.fields().get(i).name().equalsIgnoreCase(schema2.fields().get(i).name()) || !compareSchemas(schema1.fields().get(i).schema(), schema2.fields().get(i).schema())) { return false; } } return true; }
@Override public Struct toConnectRow(final GenericRow genericRow) { final List<Object> columns = new ArrayList<>(genericRow.getColumns().size()); for (int i = 0; i < genericRow.getColumns().size(); i++) { columns.add( replaceSchema( avroCompatibleSchema.fields().get(i).schema(), genericRow.getColumns().get(i))); } return innerTranslator.toConnectRow(new GenericRow(columns)); }
private static List<Pair<String, Type>> getStructItems(final Schema struct) { if (struct.type() != Schema.Type.STRUCT) { return null; } final List<Pair<String, Type>> itemList = new ArrayList<>(); for (final Field field: struct.schema().fields()) { itemList.add(new Pair<>(field.name(), getKsqlType(field.schema()))); } return itemList; }
/** * Rename field names to be consistent with the internal column names. */ public static Schema getAvroSerdeKsqlSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name().replace(".", "_"), field.schema()); } return schemaBuilder.build(); }
public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
/** * Remove the alias when reading/writing from outside */ public static Schema getSchemaWithNoAlias(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { final String name = getFieldNameWithNoAlias(field); schemaBuilder.field(name, field.schema()); } return schemaBuilder.build(); }
@Test public void shouldGetTheCorrectFieldName() { final Optional<Field> field = SchemaUtil.getFieldByName(schema, "orderid".toUpperCase()); Assert.assertTrue(field.isPresent()); assertThat(field.get().schema(), sameInstance(Schema.OPTIONAL_INT64_SCHEMA)); assertThat("", field.get().name().toLowerCase(), equalTo("orderid")); final Optional<Field> field1 = SchemaUtil.getFieldByName(schema, "orderid"); Assert.assertFalse(field1.isPresent()); }
private Schema toKsqlStructSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { try { final Schema fieldSchema = toKsqlFieldSchema(field.schema()); schemaBuilder.field(field.name().toUpperCase(), fieldSchema); } catch (final UnsupportedTypeException e) { log.error("Error inferring schema at field {}: {}", field.name(), e.getMessage()); } } return schemaBuilder.optional().build(); } }
private void assertSchemaContent(String[] fields, Schema[] types, Schema keySchema) { IntStream.range(0, fields.length).forEach(i -> { String fieldName = fields[i].trim(); Field field = keySchema.field(Strings.unquoteIdentifierPart(fieldName)); assertNotNull(fieldName + " not found in schema", field); VerifyRecord.assertConnectSchemasAreEqual(fieldName, types[i], field.schema()); }); }
public static Schema buildSchemaWithAlias(final Schema schema, final String alias) { final SchemaBuilder newSchema = SchemaBuilder.struct().name(schema.name()); for (final Field field : schema.fields()) { newSchema.field((alias + "." + field.name()), field.schema()); } return newSchema; }
private void assertSchema(Struct valueA, Schema expected) { Assertions.assertThat(valueA.schema().field("after").schema().field("cola").schema()).isEqualTo(expected); Assertions.assertThat(valueA.schema().field("after").schema().field("colb").schema()).isEqualTo(expected); Assertions.assertThat(valueA.schema().field("after").schema().field("colc").schema()).isEqualTo(expected); Assertions.assertThat(valueA.schema().field("after").schema().field("cold").schema()).isEqualTo(expected); } }
public SourceRecordAssert valueAfterFieldSchemaIsEqualTo(Schema expectedSchema) { Schema valueSchema = record.valueSchema(); Schema afterFieldSchema = valueSchema.field("after").schema(); VerifyRecord.assertConnectSchemasAreEqual(null, afterFieldSchema, expectedSchema); return this; } }
@Test public void shouldTranslateArray() { final Schema connectSchema = SchemaBuilder .struct() .field("arrayField", SchemaBuilder.array(Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.isOptional(), is(true)); assertThat(arraySchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMapWithNonStringKey() { final Schema connectSchema = SchemaBuilder .struct() .field("mapfield", SchemaBuilder.map(Schema.INT32_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMaps() { final Schema connectSchema = SchemaBuilder .struct() .field("mapField", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
private void assertSchema(Struct content) { if (schema == null) { return; } Schema schema = content.schema(); Field field = schema.field(fieldName); assertNotNull(fieldName + " not found in schema " + SchemaUtil.asString(schema), field); VerifyRecord.assertConnectSchemasAreEqual(field.name(), field.schema(), this.schema); } }