generator, new AvroData(1), generator.schema(), ordersSchema, new SessionManager(), "orderid");
@Test public void shouldCreateStructWithNestedObject() { for (Entry<String, BsonValue> entry : val.entrySet()) { converter.addFieldSchema(entry, builder); } Schema finalSchema = builder.build(); Struct struct = new Struct(finalSchema); for (Entry<String, BsonValue> entry : val.entrySet()) { converter.convertRecord(entry, finalSchema, struct); } final GenericData.Record avro = (GenericData.Record)avroData.fromConnectData(finalSchema, struct); assertThat(avro.toString()).isEqualTo( "{\"_id\": 1, " + "\"s1\": {\"s1f1\": \"field1s1\", \"s1f2\": \"field2s1\"}, " + "\"s2\": {\"s2f1\": \"field1s2\", \"s2f2\": {\"in1\": 1}}}"); }
/** * When we want to consume SinkRecord which generated by debezium-connector-mysql, it should not * throw error "org.apache.avro.SchemaParseException: Illegal character in: server-id" */ @Test public void shouldValidateSourceInfoSchema() { org.apache.kafka.connect.data.Schema kafkaSchema = SourceInfo.SCHEMA; Schema avroSchema = avroData.fromConnectSchema(kafkaSchema); assertTrue(avroSchema != null); }
final AvroData avroData = new AvroData(1); final org.apache.kafka.connect.data.Schema ksqlSchema = SchemaUtil.getOptionalSchema(avroData.toConnectSchema(avroSchema));
if (value instanceof Record) { final Record record = (Record) value; final Object ksqlValue = avroData.toConnectData(record.getSchema(), record).value(); genericRowValues.add( SchemaUtil.getOptionalValue(ksqlSchema.field(field.name()).schema(), ksqlValue)); final String keyString = avroData.toConnectData( randomAvroMessage.getSchema().getField(key).schema(), randomAvroMessage.get(key)).value().toString();
@Override public Schema retrieveSchema(TableId table, String topic) { String subject = getSubject(topic); try { logger.debug("Retrieving schema information for topic {} with subject {}", topic, subject); SchemaMetadata latestSchemaMetadata = schemaRegistryClient.getLatestSchemaMetadata(subject); org.apache.avro.Schema avroSchema = new Parser().parse(latestSchemaMetadata.getSchema()); return avroData.toConnectSchema(avroSchema); } catch (IOException | RestClientException exception) { throw new ConnectException(String.format( "Exception while fetching latest schema metadata for topic=%s, subject=%s", topic, subject), exception ); } }
@Override public Struct apply(GenericRecord record) { return (Struct) avroData.toConnectData(record.getSchema(), record).value(); } }
@Override public Schema getSchema(Configuration conf, Path path) throws IOException { SeekableInput input = new FsInput(path, conf); DatumReader<Object> reader = new GenericDatumReader<>(); FileReader<Object> fileReader = DataFileReader.openReader(input, reader); org.apache.avro.Schema schema = fileReader.getSchema(); fileReader.close(); return avroData.toConnectSchema(schema); }
@Before public void setup() throws Exception { record = getFile("complex_objects.json"); val = BsonDocument.parse(record); builder = SchemaBuilder.struct().name("complex"); avroData = new AvroData(100); converter = new MongoDataConverter(ArrayEncoding.ARRAY); }
@Test @FixFor("DBZ-650") public void shouldCreateSchemaWithNestedObject() { for (Entry<String, BsonValue> entry : val.entrySet()) { converter.addFieldSchema(entry, builder); } Schema finalSchema = builder.build(); final org.apache.avro.Schema avroSchema = avroData.fromConnectSchema(finalSchema); assertThat(avroSchema.toString()).isEqualTo( "{\"type\":\"record\",\"name\":\"complex\",\"fields\":[" + "{\"name\":\"_id\",\"type\":[\"null\",\"int\"],\"default\":null}," + "{\"name\":\"s1\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"s1\",\"namespace\":\"complex\",\"fields\":[" + "{\"name\":\"s1f1\",\"type\":[\"null\",\"string\"],\"default\":null}," + "{\"name\":\"s1f2\",\"type\":[\"null\",\"string\"],\"default\":null}]," + "\"connect.name\":\"complex.s1\"}],\"default\":null}," + "{\"name\":\"s2\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"s2\",\"namespace\":\"complex\",\"fields\":[" + "{\"name\":\"s2f1\",\"type\":[\"null\",\"string\"],\"default\":null}," + "{\"name\":\"s2f2\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"s2f2\",\"namespace\":\"complex.s2\",\"fields\":[" + "{\"name\":\"in1\",\"type\":[\"null\",\"int\"],\"default\":null}]," + "\"connect.name\":\"complex.s2.s2f2\"}],\"default\":null}]," + "\"connect.name\":\"complex.s2\"}],\"default\":null}]," + "\"connect.name\":\"complex\"}"); }
@Override public void write(SinkRecord record) throws IOException { log.trace("Sink record: {}", record.toString()); Object value = avroData.fromConnectData(schema, record.value()); writer.append(value); }
@Override public Struct apply(GenericRecord record) { return (Struct) avroData.toConnectData(record.getSchema(), record).value(); } }
@Override public Schema getSchema(Configuration conf, Path path) throws IOException { AvroReadSupport<GenericRecord> readSupport = new AvroReadSupport<>(); ParquetReader.Builder<GenericRecord> builder = ParquetReader.builder(readSupport, path); ParquetReader<GenericRecord> parquetReader = builder.withConf(conf).build(); GenericRecord record; Schema schema = null; while ((record = parquetReader.read()) != null) { schema = avroData.toConnectSchema(record.getSchema()); } parquetReader.close(); return schema; }
public GenericRecordToStruct() { this.avroData = new AvroData(CACHE_SIZE); }
@Override public void alterSchema(String database, String tableName, Schema schema) throws HiveMetaStoreException { Table table = hiveMetaStore.getTable(database, tableName); table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString()); hiveMetaStore.alterTable(table); }
@Override public void write(SinkRecord record) throws IOException { Object value = avroData.fromConnectData(record.valueSchema(), record.value()); writer.write((GenericRecord) value); }
@Override public R apply(R record) { try { Object updatedValue = mapper.readValue((String) operatingValue(record), clazz); SchemaAndValue s = avroData.toConnectData(Greeting.getClassSchema(), updatedValue); return newRecord(record, s.schema(), s.value()); } catch (IOException e) { throw new DataException("", e); } }
public GenericRecordToStruct() { this.avroData = new AvroData(CACHE_SIZE); }
@Override public RecordWriter<SinkRecord> getRecordWriter( Configuration conf, final String fileName, SinkRecord record, final AvroData avroData) throws IOException { final Schema avroSchema = avroData.fromConnectSchema(record.valueSchema()); CompressionCodecName compressionCodecName = CompressionCodecName.SNAPPY; int blockSize = 256 * 1024 * 1024; int pageSize = 64 * 1024; Path path = new Path(fileName); final ParquetWriter<GenericRecord> writer = new AvroParquetWriter<>(path, avroSchema, compressionCodecName, blockSize, pageSize, true, conf); return new RecordWriter<SinkRecord>() { @Override public void write(SinkRecord record) throws IOException { Object value = avroData.fromConnectData(record.valueSchema(), record.value()); writer.write((GenericRecord) value); } @Override public void close() throws IOException { writer.close(); } }; } }
/** * parses the value. * @param schema * @param value * @return */ private Map<String, byte[]> parse(final Schema schema, final Object value) { final Map<String, byte[]> values = new LinkedHashMap<>(); try { Object data = avroData.fromConnectData(schema, value); if (data == null || !(data instanceof GenericRecord)) { return EMPTY_MAP; } final GenericRecord record = (GenericRecord) data; final List<Field> fields = schema.fields(); for (Field field : fields) { final byte[] fieldValue = toValue(record, field); if (fieldValue == null) { continue; } values.put(field.name(), fieldValue); } return values; } catch (Exception ex) { final String errorMsg = String.format("Failed to parse the schema [%s] , value [%s] with ex [%s]" , schema, value, ex.getMessage()); throw new EventParsingException(errorMsg, ex); } }