public static File generateEnumTypes(String filename, int nrows, String[][] categories) throws IOException { assert categories.length == 2 : "Needs only 2 columns"; File parentDir = Files.createTempDir(); File f = new File(parentDir, filename); DatumWriter<GenericRecord> w = new GenericDatumWriter<GenericRecord>(); DataFileWriter<GenericRecord> dw = new DataFileWriter<GenericRecord>(w); Schema enumSchema1 = SchemaBuilder.enumeration("CEnum1").symbols(categories[0]); Schema enumSchema2 = SchemaBuilder.enumeration("CEnum2").symbols(categories[1]); Schema schema = SchemaBuilder.builder() .record("test_enum_types").fields() .name("CEnum").type(enumSchema1).noDefault() .name("CUEnum").type().optional().type(enumSchema2) .endRecord(); System.out.println(schema); int numOfCategories1 = categories[0].length; int numOfCategories2 = categories[1].length; try { dw.create(schema, f); for (int i = 0; i < nrows; i++) { GenericRecord gr = new GenericData.Record(schema); gr.put("CEnum", new GenericData.EnumSymbol(enumSchema1, categories[0][i % numOfCategories1])); gr.put("CUEnum", i % (numOfCategories2+1) == 0 ? null : new GenericData.EnumSymbol(enumSchema2, categories[1][i % numOfCategories2])); dw.append(gr); } return f; } finally { dw.close();; } }
@Test public void testEnum() { List<String> symbols = Arrays.asList("a", "b"); Schema expected = Schema.createEnum("myenum", null, null, symbols); expected.addProp("p", "v"); Schema schema = SchemaBuilder.enumeration("myenum") .prop("p", "v").symbols("a", "b"); Assert.assertEquals(expected, schema); }
@Test public void testEnumWithDefault() { List<String> symbols = Arrays.asList("a", "b"); String enumDefault = "a"; Schema expected = Schema.createEnum("myenum", null, null, symbols, enumDefault); expected.addProp("p", "v"); Schema schema = SchemaBuilder.enumeration("myenum") .prop("p", "v").defaultSymbol(enumDefault).symbols("a", "b"); Assert.assertEquals(expected, schema); }
.type() .enumeration("anEnum") .symbols("s1", "s2") .enumDefault("s1") .endRecord()));
public static AvroTestRecordWriter generateSimpleEnumSchema_NoNullValues() throws Exception { final String[] symbols = { "E_SYM_A", "E_SYM_B", "E_SYM_C", "E_SYM_D" }; final Schema schema = SchemaBuilder.record("AvroRecordReaderTest") .namespace("com.dremio.exec.store.avro") .fields() .name("a_string").type().stringType().noDefault() .name("b_enum").type().enumeration("my_enum").symbols(symbols).noDefault() .endRecord(); final File file = File.createTempFile("avro-primitive-test", ".avro"); file.deleteOnExit(); final Schema enumSchema = schema.getField("b_enum").schema(); final AvroTestRecordWriter record = new AvroTestRecordWriter(schema, file); try { for (int i = 0; i < RECORD_COUNT; i++) { record.startRecord(); record.put("a_string", "a_" + i); final GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(enumSchema, symbols[(i + symbols.length) % symbols.length]); record.put("b_enum", symbol); record.endRecord(); } } finally { record.close(); } return record; }
@Test public void testAllAvroTypes() { Schema r = SchemaBuilder.record("r").fields() .requiredBoolean("boolF") .requiredInt("intF") .requiredLong("longF") .requiredFloat("floatF") .requiredDouble("doubleF") .requiredString("stringF") .requiredBytes("bytesF") .name("fixedF1").type().fixed("F1").size(1).noDefault() .name("enumF").type().enumeration("E1").symbols("S").noDefault() .name("mapF").type().map().values().stringType().noDefault() .name("arrayF").type().array().items().stringType().noDefault() .name("recordF").type().record("inner").fields() .name("f").type().intType().noDefault() .endRecord().noDefault() .optionalBoolean("boolO") .endRecord(); assertTrue(SchemaValidationUtil.canRead(r, r)); } }
.name("fixedF2").type().fixed("F2").size(1).fixedDefault(bufdef) .name("fixedF3").type().fixed("F3").size(1).fixedDefault(strdef) .name("enumF").type().enumeration("E1").symbols("S").enumDefault("S") .name("mapF").type().map().values().stringType() .mapDefault(mapdef) .name("fixedU").type().unionOf().fixed("F4").size(1).and() .intType().endUnion().fixedDefault(bytedef) .name("enumU").type().unionOf().enumeration("E2").symbols("SS").and() .intType().endUnion().enumDefault("SS") .name("mapU").type().unionOf().map().values().stringType().and()
.type().enumeration("check_enum_symbols_type").symbols("A", "C").noDefault() .type().enumeration("check_enum_name_type").symbols("A", "B", "C", "D").noDefault() .type().enumeration("check_enum_symbols_type").symbols("A", "B", "C", "D").noDefault() .type().enumeration("check_enum_name_type_ERR").symbols("A", "B", "C", "D").noDefault()
@Test public void testToObject() { assertEquals(null, toObject(null)); assertEquals(JsonProperties.NULL_VALUE, toObject(NullNode.getInstance())); assertEquals(true, toObject(BooleanNode.TRUE)); assertEquals(1, toObject(IntNode.valueOf(1))); assertEquals(2L, toObject(IntNode.valueOf(2), Schema.create(Schema.Type.LONG))); assertEquals(1.0f, toObject(DoubleNode.valueOf(1.0), Schema.create(Schema.Type.FLOAT))); assertEquals(2.0, toObject(DoubleNode.valueOf(2.0))); assertEquals(TextNode.valueOf("\u0001\u0002"), toJsonNode(new byte[]{1, 2})); assertArrayEquals(new byte[]{1, 2}, (byte[]) toObject(TextNode.valueOf("\u0001\u0002"), Schema.create(Schema.Type.BYTES))); assertEquals("a", toObject(TextNode.valueOf("a"))); assertEquals("UP", toObject(TextNode.valueOf("UP"), SchemaBuilder.enumeration("Direction").symbols("UP", "DOWN"))); ArrayNode an = JsonNodeFactory.instance.arrayNode(); an.add(1); assertEquals(Collections.singletonList(1), toObject(an)); ObjectNode on = JsonNodeFactory.instance.objectNode(); on.put("a", 1); assertEquals(Collections.singletonMap("a", 1), toObject(on)); assertEquals(Collections.singletonMap("a", 1L), toObject(on, SchemaBuilder.record("r").fields().requiredLong("a").endRecord())); assertEquals(JsonProperties.NULL_VALUE, toObject(NullNode.getInstance(), SchemaBuilder.unionOf().nullType().and().intType().endUnion())); assertEquals("a", toObject(TextNode.valueOf("a"), SchemaBuilder.unionOf().stringType().and().intType().endUnion())); }
@Test public void testAllTypes() throws SchemaValidationException { Schema s = SchemaBuilder.record("r").fields() .requiredBoolean("boolF") .requiredInt("intF") .requiredLong("longF") .requiredFloat("floatF") .requiredDouble("doubleF") .requiredString("stringF") .requiredBytes("bytesF") .name("fixedF1").type().fixed("F1").size(1).noDefault() .name("enumF").type().enumeration("E1").symbols("S").noDefault() .name("mapF").type().map().values().stringType().noDefault() .name("arrayF").type().array().items().stringType().noDefault() .name("recordF").type().record("inner").fields() .name("f").type().intType().noDefault() .endRecord().noDefault() .optionalBoolean("boolO") .endRecord(); testValidatorPasses(builder.mutualReadStrategy().validateLatest(), s, s); }
@Test public void testUnionFieldBuild() { SchemaBuilder.record("r").fields() .name("allUnion").type().unionOf() .booleanType().and() .intType().and() .longType().and() .floatType().and() .doubleType().and() .stringType().and() .bytesType().and() .nullType().and() .fixed("Fix").size(1).and() .enumeration("Enu").symbols("Q").and() .array().items().intType().and() .map().values().longType().and() .record("Rec").fields() .name("one").type("Fix").noDefault() .endRecord() .endUnion().booleanDefault(false) .endRecord(); }