/** * Handle the unexpected value from a row with a column type of {@link Types#BLOB}, {@link Types#BINARY}, * {@link Types#VARBINARY}, {@link Types#LONGVARBINARY}. * * @param value the binary value for which no conversion was found; never null * @param fieldDefn the field definition in the Kafka Connect schema; never null * @return the converted value, or null if the conversion could not be made and the column allows nulls * @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls * @see #convertBinary(Column, Field, Object) */ protected byte[] unexpectedBinary(Object value, Field fieldDefn) { logger.warn("Unexpected JDBC BINARY value for field {} with schema {}: class={}, value={}", fieldDefn.name(), fieldDefn.schema(), value.getClass(), value); return null; }
@Test public void shouldMatchNameWithAlias() { final Field field = new Field("foo", 0, Schema.INT32_SCHEMA); assertThat(SchemaUtil.matchFieldName(field, "bar.foo"), is(true)); }
private static boolean areFieldListsEqual(List<Field> fields1, List<Field> fields2) { if (fields1 == null && fields2 != null || fields1 != null && fields2 == null) { return false; } if (fields1.size() != fields2.size()) { return false; } for(int i = 0; i < fields1.size(); i++) { Field field1 = fields1.get(i); Field field2 = fields2.get(i); boolean equal = Objects.equals(field1.index(), field2.index()) && Objects.equals(field1.name(), field2.name()) && areConnectSchemasEqual(field1.schema(), field2.schema()); if (!equal) { return false; } } return true; } }
protected Object convertTimestampToLocalDateTime(Column column, Field fieldDefn, Object data) { if (data == null && !fieldDefn.schema().isOptional()) { return null; } if (!(data instanceof Timestamp)) { return data; } return ((Timestamp)data).toLocalDateTime(); } }
private String avroCompatibleFieldName(final Field field) { // Currently the only incompatible field names expected are fully qualified // column identifiers. Once quoted identifier support is introduced we will // need to implement something more generic here. return field.name().replace(".", "_"); }
public Struct toConnectRow(final GenericRow row) { final Struct struct = new Struct(schema); schema.fields().forEach( field -> struct.put(field, row.getColumns().get(field.index())) ); return struct; } }
@Override public Struct toConnectRow(final GenericRow genericRow) { final List<Object> columns = new ArrayList<>(genericRow.getColumns().size()); for (int i = 0; i < genericRow.getColumns().size(); i++) { columns.add( replaceSchema( avroCompatibleSchema.fields().get(i).schema(), genericRow.getColumns().get(i))); } return innerTranslator.toConnectRow(new GenericRow(columns)); }
/** * Convert an unknown data value. * * @param column the column definition describing the {@code data} value; never null * @param fieldDefn the field definition; never null * @param data the data object to be converted into a {@link Date Kafka Connect date} type; never null * @return the converted value, or null if the conversion could not be made and the column allows nulls * @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls */ protected Object handleUnknownData(Column column, Field fieldDefn, Object data) { if (column.isOptional() || fieldDefn.schema().isOptional()) { Class<?> dataClass = data.getClass(); logger.warn("Unexpected value for JDBC type {} and column {}: class={}", column.jdbcType(), column, dataClass.isArray() ? dataClass.getSimpleName() : dataClass.getName()); // don't include value in case its // sensitive return null; } throw new IllegalArgumentException("Unexpected value for JDBC type " + column.jdbcType() + " and column " + column + ": class=" + data.getClass()); // don't include value in case its sensitive }
public static String getFieldNameWithNoAlias(final Field field) { final String name = field.name(); if (name.contains(".")) { return name.substring(name.indexOf(".") + 1); } else { return name; } }
@Test public void shouldNotMatchDifferentName() { final Field field = new Field("foo", 0, Schema.INT32_SCHEMA); assertThat(SchemaUtil.matchFieldName(field, "bar"), is(false)); }
@Override public GenericRow toKsqlRow(final Schema connectSchema, final Object connectObject) { final GenericRow avroCompatibleRow = innerTranslator.toKsqlRow(connectSchema, connectObject); if (avroCompatibleRow == null) { return null; } final List<Object> columns = new ArrayList<>(avroCompatibleRow.getColumns().size()); for (int i = 0; i < avroCompatibleRow.getColumns().size(); i++) { columns.add( replaceSchema( ksqlSchema.fields().get(i).schema(), avroCompatibleRow.getColumns().get(i))); } return new GenericRow(columns); }
assertThat(values.field("C1").name()).isEqualTo("C1"); assertThat(values.field("C1").index()).isEqualTo(0); assertThat(values.field("C1").schema()).isEqualTo(SchemaBuilder.string().build()); assertThat(values.field("C2").name()).isEqualTo("C2"); assertThat(values.field("C2").index()).isEqualTo(1); assertThat(values.field("C2").schema()).isEqualTo(Decimal.builder(3).parameter("connect.decimal.precision", "5").optional().build()); // scale of 3 assertThat(values.field("C3").name()).isEqualTo("C3"); assertThat(values.field("C3").index()).isEqualTo(2); assertThat(values.field("C3").schema()).isEqualTo(Date.builder().optional().build()); // optional date assertThat(values.field("C4").name()).isEqualTo("C4"); assertThat(values.field("C4").index()).isEqualTo(3); assertThat(values.field("C4").schema()).isEqualTo(SchemaBuilder.int32().optional().build()); // JDBC INTEGER = 32 bits assertThat(values.field("C5").index()).isEqualTo(4); assertThat(values.field("C5").schema()).isEqualTo(SchemaBuilder.bytes().build()); // JDBC BINARY = bytes assertThat(values.field("C6").index()).isEqualTo(5); assertThat(values.field("C6").schema()).isEqualTo(SchemaBuilder.int16().build());
public static Set<Integer> getRowTimeRowKeyIndexes(final Schema schema) { final Set<Integer> indexSet = new HashSet<>(); for (int i = 0; i < schema.fields().size(); i++) { final Field field = schema.fields().get(i); if (field.name().equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) || field.name().equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { indexSet.add(i); } } return indexSet; }
@Test public void shouldMatchName() { final Field field = new Field("foo", 0, Schema.INT32_SCHEMA); assertThat(SchemaUtil.matchFieldName(field, "foo"), is(true)); }
public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
/** * Remove the alias when reading/writing from outside */ public static Schema getSchemaWithNoAlias(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { final String name = getFieldNameWithNoAlias(field); schemaBuilder.field(name, field.schema()); } return schemaBuilder.build(); }