@Test public void shouldDeserializeArrayToArray() { shouldDeserializeTypeCorrectly( org.apache.avro.SchemaBuilder.array().items().intType(), ImmutableList.of(1, 2, 3, 4, 5, 6), SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build() ); }
@Test public void testArray() { Schema longSchema = Schema.create(Schema.Type.LONG); Schema expected = Schema.createArray(longSchema); Schema schema1 = SchemaBuilder.array().items().longType(); Assert.assertEquals(expected, schema1); Schema schema2 = SchemaBuilder.array().items(longSchema); Assert.assertEquals(expected, schema2); Schema schema3 = SchemaBuilder.array().prop("p", "v") .items().type("long"); expected.addProp("p", "v"); Assert.assertEquals(expected, schema3); }
@Test public void shouldSerializeArray() { shouldSerializeTypeCorrectly( SchemaBuilder.array(Schema.OPTIONAL_INT32_SCHEMA).optional().build(), ImmutableList.of(1, 2, 3), org.apache.avro.SchemaBuilder.array().items( org.apache.avro.SchemaBuilder.builder() .unionOf().nullType().and().intType().endUnion()) ); }
.endRecord(); final org.apache.avro.Schema avroSchema = org.apache.avro.SchemaBuilder.array().items(entrySchema); final Map<String, Long> value = ImmutableMap.of("foo", 123L); final List<GenericRecord> avroValue = new LinkedList<>();
/** * Look are avPathContexts to generate the schema as a {@code Field} for the current element. * * @param avPathContexts the extracted elements with AVPath * @param aVPath the AVPath of the element to generate * @param field the name of the output field * @return a {@code Field} that describe elements stored in the avPathContexts */ private Field retrieveSchema(List<Evaluator.Ctx> avPathContexts, String aVPath, String field) { // If there is at least one field retrieve, it will be placed on a list. // We are inferring from the the number of element and the TCOMP path in order to know // if we are on a list or not. Evaluator.Ctx firstAvPathcontext = avPathContexts.get(0); if (avPathContexts.size() > 1 || FieldSelectorUtil.canRetrieveMultipleElements(aVPath)) { // create an list for this field try { return new Field(field, SchemaBuilder.array().items(firstAvPathcontext.schema()), "", ""); } catch (SchemaParseException e) { throw ProcessingErrorCode.createInvalidFieldNameErrorException(e, field); } } else { try { return new Field(field, firstAvPathcontext.schema(), "", ""); } catch (SchemaParseException e) { throw ProcessingErrorCode.createInvalidFieldNameErrorException(e, field); } } }
/** * Look are avPathContexts to generate the schema as a {@code Field} for the current element. * * @param avPathContexts the extracted elements with AVPath * @param aVPath the AVPath of the element to generate * @param field the name of the output field * @return a {@code Field} that describe elements stored in the avPathContexts */ private Field retrieveSchema(List<Evaluator.Ctx> avPathContexts, String aVPath, String field) { // If there is at least one field retrieve, it will be placed on a list. // We are inferring from the the number of element and the TCOMP path in order to know // if we are on a list or not. Evaluator.Ctx firstAvPathcontext = avPathContexts.get(0); if (avPathContexts.size() > 1 || FieldSelectorUtil.canRetrieveMultipleElements(aVPath)) { // create an list for this field try { return new Field(field, SchemaBuilder.array().items(firstAvPathcontext.schema()), "", ""); } catch (SchemaParseException e) { throw ProcessingErrorCode.createInvalidFieldNameErrorException(e, field); } } else { try { return new Field(field, firstAvPathcontext.schema(), "", ""); } catch (SchemaParseException e) { throw ProcessingErrorCode.createInvalidFieldNameErrorException(e, field); } } }
private org.apache.avro.Schema inferSchemaField(Field field) { String name = field.getName(); Field.Type sqlType = field.getType(); Field.Mode mode = field.getMode(); // Get the "basic" type of the field. org.apache.avro.Schema fieldSchema = inferSchemaFieldWithoutMode(field); // BigQuery fields are NULLABLE by default. if (Field.Mode.NULLABLE == mode || mode == null) { fieldSchema = AvroUtils.wrapAsNullable(fieldSchema); } else if (Field.Mode.REPEATED == mode) { // Determine if the field is an array. // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type fieldSchema = SchemaBuilder.array().items(fieldSchema); } return fieldSchema; }
private org.apache.avro.Schema inferSchemaField(Field field) { String name = field.getName(); Field.Type sqlType = field.getType(); Field.Mode mode = field.getMode(); // Get the "basic" type of the field. org.apache.avro.Schema fieldSchema = inferSchemaFieldWithoutMode(field); // BigQuery fields are NULLABLE by default. if (Field.Mode.NULLABLE == mode || mode == null) { fieldSchema = AvroUtils.wrapAsNullable(fieldSchema); } else if (Field.Mode.REPEATED == mode) { // Determine if the field is an array. // https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type fieldSchema = SchemaBuilder.array().items(fieldSchema); } return fieldSchema; }
public void run(RegressionEnvironment env) { Schema intArraySchema = SchemaBuilder.array().items(SchemaBuilder.builder().intType()); Schema mixedArraySchema = SchemaBuilder.array().items(SchemaBuilder.unionOf().intType().and().stringType().and().doubleType().endUnion()); Schema nullArraySchema = SchemaBuilder.array().items(SchemaBuilder.builder().nullType());
public void run(RegressionEnvironment env) { String epl = "@name('s0') " + EventRepresentationChoice.AVRO.getAnnotationText() + " select 1 as myInt," + "{1L, 2L} as myLongArray," + EPLInsertIntoPopulateCreateStreamAvro.class.getName() + ".makeByteArray() as myByteArray, " + EPLInsertIntoPopulateCreateStreamAvro.class.getName() + ".makeMapStringString() as myMap " + "from SupportBean"; env.compileDeploy(epl).addListener("s0"); env.sendEventBean(new SupportBean()); EventBean event = env.listener("s0").assertOneGetNewAndReset(); String json = SupportAvroUtil.avroToJson(event); System.out.println(json); assertEquals(1, event.get("myInt")); EPAssertionUtil.assertEqualsExactOrder(new Long[]{1L, 2L}, ((Collection) event.get("myLongArray")).toArray()); assertTrue(Arrays.equals(new byte[]{1, 2, 3}, ((ByteBuffer) event.get("myByteArray")).array())); assertEquals("{k1=v1}", ((Map) event.get("myMap")).toString()); Schema designSchema = record("name").fields() .requiredInt("myInt") .name("myLongArray").type(array().items(unionOf().nullType().and().longType().endUnion())).noDefault() .name("myByteArray").type("bytes").noDefault() .name("myMap").type(map().values().stringBuilder().prop(AvroConstant.PROP_JAVA_STRING_KEY, AvroConstant.PROP_JAVA_STRING_VALUE).endString()).noDefault() .endRecord(); Schema assembledSchema = ((AvroEventType) event.getEventType()).getSchemaAvro(); String compareMsg = SupportAvroUtil.compareSchemas(designSchema, assembledSchema); assertNull(compareMsg, compareMsg); env.undeployAll(); } }
GenericData.Record event = new GenericData.Record(record("name").fields() .optionalString("simple") .name("myarray").type(array().items().longType()).noDefault() .name("mymap").type(map().values().stringType()).noDefault() .endRecord());
@Test(expected = IllegalArgumentException.class) public void testRejectsNonRecordSchemas() { final DatasetDescriptor desc = new DatasetDescriptor.Builder() .schema(SchemaBuilder.array().items().stringType()) .build(); new CSVFileReader<GenericData.Record>(localfs, csvFile, desc, DataModelUtil.accessor(GenericData.Record.class, desc.getSchema())); }
@Test public void testList() { Type list = Types.ListType.ofRequired(34, Types.UUIDType.get()); Schema schema = addElementId(34, SchemaBuilder.array().items( LogicalTypes.uuid().addToSchema(Schema.createFixed("uuid_fixed", null, null, 16)))); Assert.assertEquals("Avro schema to list", list, AvroSchemaUtil.convert(schema)); Assert.assertEquals("List to Avro schema", schema, AvroSchemaUtil.convert(list)); }
assembler.name(propertyName).type(schema).noDefault(); } else { assembler.name(propertyName).type(array().items(schema)).noDefault(); if (eventType instanceof AvroEventType) { schema = ((AvroEventType) eventType).getSchemaAvro(); assembler.name(propertyName).type(array().items(schema)).noDefault(); } else if (eventType instanceof MapEventType) { MapEventType mapEventType = (MapEventType) eventType; Schema nestedSchema = assembleNestedSchema(mapEventType, avroSettings, annotations, eventTypeNameResolver, statementName, optionalMapper); assembler.name(propertyName).type(array().items(nestedSchema)).noDefault(); } else { throw new IllegalStateException("Unrecognized event type " + eventType); Schema array; if (avroSettings.isEnableNativeString()) { array = array().items(builder().stringBuilder().prop(PROP_JAVA_STRING_KEY, PROP_JAVA_STRING_VALUE).endString()); } else { array = array().items(builder().stringBuilder().endString());
@Test public void testListOfStructs() { Type list = Types.ListType.ofRequired(34, Types.StructType.of( required(35, "lat", Types.FloatType.get()), required(36, "long", Types.FloatType.get()) )); Schema schema = addElementId(34, SchemaBuilder.array().items( record("r34", requiredField(35, "lat", Schema.create(Schema.Type.FLOAT)), requiredField(36, "long", Schema.create(Schema.Type.FLOAT))) )); Assert.assertEquals("Avro schema to list", list, AvroSchemaUtil.convert(schema)); Assert.assertEquals("List to Avro schema", schema, AvroSchemaUtil.convert(list)); }