private String avroCompatibleFieldName(final Field field) { // Currently the only incompatible field names expected are fully qualified // column identifiers. Once quoted identifier support is introduced we will // need to implement something more generic here. return field.name().replace(".", "_"); }
public static String getFieldNameWithNoAlias(final Field field) { final String name = field.name(); if (name.contains(".")) { return name.substring(name.indexOf(".") + 1); } else { return name; } }
public static Set<Integer> getRowTimeRowKeyIndexes(final Schema schema) { final Set<Integer> indexSet = new HashSet<>(); for (int i = 0; i < schema.fields().size(); i++) { final Field field = schema.fields().get(i); if (field.name().equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) || field.name().equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { indexSet.add(i); } } return indexSet; }
private Map<String, Field> getCaseInsensitiveFieldMap(final Schema schema) { final Map<String, Field> fieldsByName = new HashMap<>(); schema.fields().forEach( field -> fieldsByName.put(field.name().toUpperCase(), field) ); return fieldsByName; }
public static String getSchemaDefinitionString(final Schema schema) { return schema.fields().stream() .map(field -> field.name() + " : " + getSqlTypeName(field.schema())) .collect(Collectors.joining(", ", "[", "]")); }
private static String getStructString(final Schema schema) { return schema.fields().stream() .map(field -> field.name() + " " + getSqlTypeName(field.schema())) .collect(Collectors.joining(", ", "STRUCT<", ">")); }
public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
public static Schema removeImplicitRowTimeRowKeyFromSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { String fieldName = field.name(); fieldName = fieldName.substring(fieldName.indexOf('.') + 1); if (!fieldName.equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) && !fieldName.equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { schemaBuilder.field(fieldName, field.schema()); } } return schemaBuilder.build(); }
private boolean compareStructSchema(final Schema schema1, final Schema schema2) { if (schema1.fields().size() != schema2.fields().size()) { return false; } for (int i = 0; i < schema1.fields().size(); i++) { if (!schema1.fields().get(i).name().equalsIgnoreCase(schema2.fields().get(i).name()) || !compareSchemas(schema1.fields().get(i).schema(), schema2.fields().get(i).schema())) { return false; } } return true; }
private Schema toKsqlStructSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { try { final Schema fieldSchema = toKsqlFieldSchema(field.schema()); schemaBuilder.field(field.name().toUpperCase(), fieldSchema); } catch (final UnsupportedTypeException e) { log.error("Error inferring schema at field {}: {}", field.name(), e.getMessage()); } } return schemaBuilder.optional().build(); } }
private static List<Pair<String, Type>> getStructItems(final Schema struct) { if (struct.type() != Schema.Type.STRUCT) { return null; } final List<Pair<String, Type>> itemList = new ArrayList<>(); for (final Field field: struct.schema().fields()) { itemList.add(new Pair<>(field.name(), getKsqlType(field.schema()))); } return itemList; }
/** * Rename field names to be consistent with the internal column names. */ public static Schema getAvroSerdeKsqlSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name().replace(".", "_"), field.schema()); } return schemaBuilder.build(); }
@Test public void shouldReturnFieldNameWithoutAliasAsIs() { assertThat("Invalid field name", SchemaUtil.getFieldNameWithNoAlias(schema.fields().get(0)), equalTo(schema.fields().get(0).name())); }
@Test public void shouldGetTheCorrectFieldName() { final Optional<Field> field = SchemaUtil.getFieldByName(schema, "orderid".toUpperCase()); Assert.assertTrue(field.isPresent()); assertThat(field.get().schema(), sameInstance(Schema.OPTIONAL_INT64_SCHEMA)); assertThat("", field.get().name().toLowerCase(), equalTo("orderid")); final Optional<Field> field1 = SchemaUtil.getFieldByName(schema, "orderid"); Assert.assertFalse(field1.isPresent()); }
private Struct updateValue(Schema newValueSchema, Struct oldValue) { final Struct newValue = new Struct(newValueSchema); for (org.apache.kafka.connect.data.Field field : oldValue.schema().fields()) { newValue.put(field.name(), oldValue.get(field)); } return newValue; }
@Test public void shouldStripAliasFromFieldName() { final Schema schemaWithAlias = SchemaUtil.buildSchemaWithAlias(schema, "alias"); assertThat("Invalid field name", SchemaUtil.getFieldNameWithNoAlias(schemaWithAlias.fields().get(0)), equalTo(schema.fields().get(0).name())); }
public static Schema buildSchemaWithAlias(final Schema schema, final String alias) { final SchemaBuilder newSchema = SchemaBuilder.struct().name(schema.name()); for (final Field field : schema.fields()) { newSchema.field((alias + "." + field.name()), field.schema()); } return newSchema; }
private void assertSchema(Struct content) { if (schema == null) { return; } Schema schema = content.schema(); Field field = schema.field(fieldName); assertNotNull(fieldName + " not found in schema " + SchemaUtil.asString(schema), field); VerifyRecord.assertConnectSchemasAreEqual(field.name(), field.schema(), this.schema); } }
private void assertSchema(Struct content) { if (schema == null) { return; } Schema schema = content.schema(); Field field = schema.field(fieldName); Assertions.assertThat(field).as(fieldName + " not found in schema " + schema).isNotNull(); VerifyRecord.assertConnectSchemasAreEqual(field.name(), field.schema(), this.schema); } }