Refine search
private void checkConvertedAvroData(Schema schema, GenericRecord record) { Assert.assertEquals(schema.getNamespace(), "example.avro"); Assert.assertEquals(schema.getType(), Schema.Type.RECORD); Assert.assertEquals(schema.getName(), "User"); Assert.assertEquals(schema.getFields().size(), 3); Schema.Field nameField = schema.getField("name"); Assert.assertEquals(nameField.name(), "name"); Assert.assertEquals(nameField.schema().getType(), Schema.Type.STRING); Schema.Field favNumberField = schema.getField("favorite_number"); Assert.assertEquals(favNumberField.name(), "favorite_number"); Assert.assertEquals(favNumberField.schema().getType(), Schema.Type.INT); Schema.Field favColorField = schema.getField("favorite_color"); Assert.assertEquals(favColorField.name(), "favorite_color"); Assert.assertEquals(favColorField.schema().getType(), Schema.Type.STRING); Assert.assertEquals(record.get("name"), "Alyssa"); Assert.assertEquals(record.get("favorite_number"), 256d); Assert.assertEquals(record.get("favorite_color"), "yellow"); }
private List<Object> workerBase(List<Object> objectRow, Schema fileSchema, List<String> columnNames, List<TypeInfo> columnTypes, GenericRecord record) throws AvroSerdeException { for(int i = 0; i < columnNames.size(); i++) { TypeInfo columnType = columnTypes.get(i); String columnName = columnNames.get(i); Object datum = record.get(columnName); Schema datumSchema = record.getSchema().getField(columnName).schema(); Schema.Field field = AvroSerdeUtils.isNullableType(fileSchema)?AvroSerdeUtils.getOtherTypeFromNullableType(fileSchema).getField(columnName):fileSchema.getField(columnName); objectRow.add(worker(datum, field == null ? null : field.schema(), datumSchema, columnType)); } return objectRow; }
/** * Gets the array schema. * * @param delta the delta * @param field the field * @return the array schema */ private static Schema getArraySchema(GenericRecord delta, String field) { List<Schema> fieldTypes = delta.getSchema().getField(field).schema().getTypes(); for (Schema type : fieldTypes) { if (type.getType() == Schema.Type.ARRAY) { return type; } } return null; }
/** * Gets the delta schema by full name. * * @param fullName the full name * @return the delta schema by full name */ private Schema getDeltaSchemaByFullName(String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField(DELTA).schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
public Schema getSchema() { final String KEY_SCHEMA = "{ \"type\" : \"record\", \"name\" : \"etl\",\"namespace\" : \"reducerTest\", \"fields\" : [ { \"name\" : " + "\"key\", \"type\" : {\"type\" : \"record\", \"name\" : \"key_name\", \"namespace\" : \"key_namespace\", " + "\"fields\" : [ {\"name\" : \"partitionKey\", \"type\" : \"long\", \"doc\" : \"\"}, { \"name\" : \"environment" + "\", \"type\" : \"string\",\"doc\" : \"\"}, {\"name\" : \"subKey\",\"type\" : \"string\", \"doc\" : \"\"} ]}, " + "\"doc\" : \"\", \"attributes_json\" : \"{\\\"delta\\\":false,\\\"pk\\\":true}\" }]}"; Schema keySchema = new Schema.Parser().parse(KEY_SCHEMA); return keySchema.getField("key").schema(); }
/** * Gets the schema by full name. * * @param delta the delta * @param field the field * @param fullName the full name * @return the schema by full name */ private static Schema getSchemaByFullName(GenericRecord delta, String field, String fullName) { Schema fieldSchema = delta.getSchema().getField(field).schema(); if (fieldSchema.getType() == Schema.Type.UNION) { List<Schema> fieldTypes = fieldSchema.getTypes(); return getSchemaByFullName(fieldTypes, fullName); } else { return fieldSchema.getFullName().equals(fullName) ? fieldSchema : null; } }
public static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
private static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
@Test public void shouldSupportAvroStructs() { // When: final org.apache.avro.Schema avroSchema = SchemaUtil.buildAvroSchema(schema, "bob"); // Then: final org.apache.avro.Schema.Field rawStruct = avroSchema.getField("RAW_STRUCT"); assertThat(rawStruct, is(notNullValue())); assertThat(rawStruct.schema().getType(), is(org.apache.avro.Schema.Type.UNION)); assertThat(rawStruct.schema().getTypes().get(0).getType(), is(org.apache.avro.Schema.Type.NULL)); assertThat(rawStruct.schema().getTypes().get(1).toString(), is( "{" + "\"type\":\"record\"," + "\"name\":\"RAW_STRUCT\"," + "\"namespace\":\"ksql.bob\"," + "\"fields\":[" + "{\"name\":\"f0\",\"type\":[\"null\",\"long\"],\"default\":null}," + "{\"name\":\"f1\",\"type\":[\"null\",\"boolean\"],\"default\":null}" + "]}" )); }
@Override void genSourceData() { Random r = newRandom(); Schema uSchema = writeSchema.getField("f").schema(); sourceData = new GenericRecord[count]; for (int i = 0; i < sourceData.length; i++) { GenericRecord rec = new GenericData.Record(writeSchema); int val = r.nextInt(1000000); Integer v = (val < 750000 ? new Integer(val) : null); rec.put("f", v); sourceData[i] = rec; } } }
@Test public void testNamespaceScope() { String z = "{\"type\":\"record\",\"name\":\"Z\",\"fields\":[]}"; String y = "{\"type\":\"record\",\"name\":\"q.Y\",\"fields\":[" +"{\"name\":\"f\",\"type\":"+z+"}]}"; String x = "{\"type\":\"record\",\"name\":\"p.X\",\"fields\":[" +"{\"name\":\"f\",\"type\":"+y+"}," +"{\"name\":\"g\",\"type\":"+z+"}" +"]}"; Schema xs = Schema.parse(x); Schema ys = xs.getField("f").schema(); assertEquals("p.Z", xs.getField("g").schema().getFullName()); assertEquals("q.Z", ys.getField("f").schema().getFullName()); }