public static boolean areEqualSchemas(final Schema schema1, final Schema schema2) { if (schema1.fields().size() != schema2.fields().size()) { return false; } for (int i = 0; i < schema1.fields().size(); i++) { if (!schema1.fields().get(i).equals(schema2.fields().get(i))) { return false; } } return true; }
public static Set<Integer> getRowTimeRowKeyIndexes(final Schema schema) { final Set<Integer> indexSet = new HashSet<>(); for (int i = 0; i < schema.fields().size(); i++) { final Field field = schema.fields().get(i); if (field.name().equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) || field.name().equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { indexSet.add(i); } } return indexSet; }
public static Optional<Field> getFieldByName(final Schema schema, final String fieldName) { return schema.fields() .stream() .filter(f -> matchFieldName(f, fieldName)) .findFirst(); }
private Map<String, Field> getCaseInsensitiveFieldMap(final Schema schema) { final Map<String, Field> fieldsByName = new HashMap<>(); schema.fields().forEach( field -> fieldsByName.put(field.name().toUpperCase(), field) ); return fieldsByName; }
public static String getSchemaDefinitionString(final Schema schema) { return schema.fields().stream() .map(field -> field.name() + " : " + getSqlTypeName(field.schema())) .collect(Collectors.joining(", ", "[", "]")); }
private static String getStructString(final Schema schema) { return schema.fields().stream() .map(field -> field.name() + " " + getSqlTypeName(field.schema())) .collect(Collectors.joining(", ", "STRUCT<", ">")); }
public static Schema removeImplicitRowTimeRowKeyFromSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { String fieldName = field.name(); fieldName = fieldName.substring(fieldName.indexOf('.') + 1); if (!fieldName.equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) && !fieldName.equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { schemaBuilder.field(fieldName, field.schema()); } } return schemaBuilder.build(); }
public Struct toConnectRow(final GenericRow row) { final Struct struct = new Struct(schema); schema.fields().forEach( field -> struct.put(field, row.getColumns().get(field.index())) ); return struct; } }
@Override public Struct toConnectRow(final GenericRow genericRow) { final List<Object> columns = new ArrayList<>(genericRow.getColumns().size()); for (int i = 0; i < genericRow.getColumns().size(); i++) { columns.add( replaceSchema( avroCompatibleSchema.fields().get(i).schema(), genericRow.getColumns().get(i))); } return innerTranslator.toConnectRow(new GenericRow(columns)); }
private static List<Pair<String, Type>> getStructItems(final Schema struct) { if (struct.type() != Schema.Type.STRUCT) { return null; } final List<Pair<String, Type>> itemList = new ArrayList<>(); for (final Field field: struct.schema().fields()) { itemList.add(new Pair<>(field.name(), getKsqlType(field.schema()))); } return itemList; }
/** * Rename field names to be consistent with the internal column names. */ public static Schema getAvroSerdeKsqlSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name().replace(".", "_"), field.schema()); } return schemaBuilder.build(); }
/** * Remove the alias when reading/writing from outside */ public static Schema getSchemaWithNoAlias(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { final String name = getFieldNameWithNoAlias(field); schemaBuilder.field(name, field.schema()); } return schemaBuilder.build(); }
public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
private Schema toKsqlStructSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { try { final Schema fieldSchema = toKsqlFieldSchema(field.schema()); schemaBuilder.field(field.name().toUpperCase(), fieldSchema); } catch (final UnsupportedTypeException e) { log.error("Error inferring schema at field {}: {}", field.name(), e.getMessage()); } } return schemaBuilder.optional().build(); } }
public static Schema buildSchemaWithAlias(final Schema schema, final String alias) { final SchemaBuilder newSchema = SchemaBuilder.struct().name(schema.name()); for (final Field field : schema.fields()) { newSchema.field((alias + "." + field.name()), field.schema()); } return newSchema; }
@Test public void shouldIgnoreUnsupportedType() { final Schema connectSchema = SchemaBuilder .struct() .field("bytesField", Schema.BYTES_SCHEMA) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.fields().size(), equalTo(0)); } }
private StructuredDataSource buildDataSource(final String kafkaTopicName) { final Schema schema = SchemaBuilder.struct() .field("field0", Schema.OPTIONAL_INT32_SCHEMA) .build(); final KsqlTopic topic = new KsqlTopic("internal", kafkaTopicName, new KsqlJsonTopicSerDe(), true); return new KsqlStream<>( "query", "stream", schema, schema.fields().get(0), new MetadataTimestampExtractionPolicy(), topic, Serdes.String()); }
FakeSourceNode(final String name) { super( new PlanNodeId("fake"), new KsqlStream<>( STATEMENT, name, SCHEMA, SCHEMA.fields().get(0), new MetadataTimestampExtractionPolicy(), new KsqlTopic(name, name, new KsqlJsonTopicSerDe(), false), Serdes.String()), SCHEMA); } }