private ColumnMetaData addColumn(String path, ValueType type, ColumnMetaData parent, boolean isArray) { ColumnMetaData column = new ColumnMetaData(path, type); if (parent != null) column.setParent(parent); column.isArray(isArray); columns.add(column); arrayWidths.add(1); // placeholder return column; }
@Test public void testNamedRecord() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"R1\",\"type\":"+SIMPLE_RECORD+"}," +"{\"name\":\"R2\",\"type\":\"R\"}" +"]}"; check(Schema.parse(s), new ColumnMetaData("R1#x", ValueType.INT), new ColumnMetaData("R1#y", ValueType.STRING), new ColumnMetaData("R2#x", ValueType.INT), new ColumnMetaData("R2#y", ValueType.STRING)); }
@Test public void testNestedRecord() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"R\",\"type\":"+SIMPLE_RECORD+"}," +"{\"name\":\"y\",\"type\":\"string\"}" +"]}"; check(Schema.parse(s), new ColumnMetaData("x", ValueType.INT), new ColumnMetaData("R#x", ValueType.INT), new ColumnMetaData("R#y", ValueType.STRING), new ColumnMetaData("y", ValueType.STRING)); }
@Test public void testNestedArray() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"A\",\"type\":"+RECORD_ARRAY+"}," +"{\"name\":\"y\",\"type\":\"string\"}" +"]}"; ColumnMetaData p = new ColumnMetaData("A[]", ValueType.NULL).isArray(true); check(Schema.parse(s), new ColumnMetaData("x", ValueType.INT), p, new ColumnMetaData("A[]#x", ValueType.INT).setParent(p), new ColumnMetaData("A[]#y", ValueType.STRING).setParent(p), new ColumnMetaData("y", ValueType.STRING)); }
@Test public void testSimpleRecord() throws Exception { check(Schema.parse(SIMPLE_RECORD), new ColumnMetaData("x", ValueType.INT), new ColumnMetaData("y", ValueType.STRING)); }
@Test public void testNestedUnion() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"u\",\"type\":"+UNION+"}," +"{\"name\":\"y\",\"type\":\"string\"}" +"]}"; ColumnMetaData p = new ColumnMetaData("u/R", ValueType.NULL).isArray(true); check(Schema.parse(s), new ColumnMetaData("x", ValueType.INT), new ColumnMetaData("u/int", ValueType.INT).isArray(true), p, new ColumnMetaData("u/R#x", ValueType.INT).setParent(p), new ColumnMetaData("u/R#y", ValueType.STRING).setParent(p), new ColumnMetaData("y", ValueType.STRING)); }
@Test public void testMap() throws Exception { String s = "{\"type\":\"map\",\"values\":"+SIMPLE_RECORD+"}"; ColumnMetaData p = new ColumnMetaData(">", ValueType.NULL).isArray(true); check(Schema.parse(s), p, new ColumnMetaData(">key", ValueType.STRING).setParent(p), new ColumnMetaData(">value#x", ValueType.INT).setParent(p), new ColumnMetaData(">value#y", ValueType.STRING).setParent(p)); }
@Test public void testUnion() throws Exception { ColumnMetaData p = new ColumnMetaData("R", ValueType.NULL).isArray(true); check(Schema.parse(UNION), new ColumnMetaData("int", ValueType.INT).isArray(true), p, new ColumnMetaData("R#x", ValueType.INT).setParent(p), new ColumnMetaData("R#y", ValueType.STRING).setParent(p)); }
@Test public void testArray() throws Exception { ColumnMetaData p = new ColumnMetaData("[]", ValueType.NULL).isArray(true); check(Schema.parse(RECORD_ARRAY), p, new ColumnMetaData("[]#x", ValueType.INT).setParent(p), new ColumnMetaData("[]#y", ValueType.STRING).setParent(p)); }
@Test public void testSimpleMap() throws Exception { String s = "{\"type\":\"map\",\"values\":\"long\"}"; ColumnMetaData p = new ColumnMetaData(">", ValueType.NULL).isArray(true); check(Schema.parse(s), p, new ColumnMetaData(">key", ValueType.STRING).setParent(p), new ColumnMetaData(">value", ValueType.LONG).setParent(p)); }
@Test public void testArrayInUnion() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"a\",\"type\":[\"int\","+RECORD_ARRAY+"]}]}"; ColumnMetaData q = new ColumnMetaData("a/array",ValueType.NULL) .isArray(true); ColumnMetaData r = new ColumnMetaData("a/array[]", ValueType.NULL) .setParent(q) .isArray(true); check(Schema.parse(s), new ColumnMetaData("a/int", ValueType.INT).isArray(true), q, r, new ColumnMetaData("a/array[]#x", ValueType.INT).setParent(r), new ColumnMetaData("a/array[]#y", ValueType.STRING).setParent(r)); }
@Test public void testSimpleUnion() throws Exception { String s = "[\"int\",\"string\"]"; check(Schema.parse(s), new ColumnMetaData("int", ValueType.INT).isArray(true), new ColumnMetaData("string", ValueType.STRING).isArray(true)); }
@Test public void testUnionInArray() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"a\",\"type\":{\"type\":\"array\",\"items\":"+UNION+"}}" +"]}"; ColumnMetaData p = new ColumnMetaData("a[]",ValueType.NULL).isArray(true); ColumnMetaData r = new ColumnMetaData("a[]/R", ValueType.NULL) .setParent(p) .isArray(true); check(Schema.parse(s), p, new ColumnMetaData("a[]/int", ValueType.INT) .setParent(p) .isArray(true), r, new ColumnMetaData("a[]/R#x", ValueType.INT).setParent(r), new ColumnMetaData("a[]/R#y", ValueType.STRING).setParent(r)); }
@Test public void testPrimitives() throws Exception { check(Schema.create(Schema.Type.NULL), new ColumnMetaData("null", ValueType.NULL)); check(Schema.create(Schema.Type.BOOLEAN), new ColumnMetaData("boolean", ValueType.BOOLEAN)); check(Schema.create(Schema.Type.INT), new ColumnMetaData("int", ValueType.INT)); check(Schema.create(Schema.Type.LONG), new ColumnMetaData("long", ValueType.LONG)); check(Schema.create(Schema.Type.FLOAT), new ColumnMetaData("float", ValueType.FLOAT)); check(Schema.create(Schema.Type.DOUBLE), new ColumnMetaData("double", ValueType.DOUBLE)); check(Schema.create(Schema.Type.BYTES), new ColumnMetaData("bytes", ValueType.BYTES)); check(Schema.create(Schema.Type.STRING), new ColumnMetaData("string", ValueType.STRING)); check(Schema.createEnum("E", null, null, Arrays.asList("X","Y","Z")), new ColumnMetaData("E", ValueType.INT)); check(Schema.createFixed("F", null, null, 5), new ColumnMetaData("F", ValueType.BYTES)); }
@Test public void testSimpleOptional() throws Exception { String s = "[\"null\",\"string\"]"; check(Schema.parse(s), new ColumnMetaData("string", ValueType.STRING).isArray(true)); }
static ColumnMetaData read(InputBuffer in, ColumnFileReader file) throws IOException { ColumnMetaData result = new ColumnMetaData(); MetaData.read(in, result); result.name = result.getString(NAME_KEY); result.type = ValueType.forName(result.getString(TYPE_KEY)); result.values = result.getBoolean(VALUES_KEY); result.isArray = result.getBoolean(ARRAY_KEY); String parentName = result.getString(PARENT_KEY); if (parentName != null) result.setParent(file.getColumnMetaData(parentName)); return result; }
@Test public void testSimpleArray() throws Exception { String s = "{\"type\":\"array\",\"items\":\"long\"}"; check(Schema.parse(s), new ColumnMetaData("[]", ValueType.LONG).isArray(true)); }
@Test public void testEmptyColumn() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.INT)); out.writeTo(FILE); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(0, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); ColumnValues<Integer> values = in.getValues("test"); for (int i : values) throw new Exception("no value should be found"); in.close(); }
@Test public void testLongs() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.LONG)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(random.nextLong()); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<Long> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(random.nextLong(), (long)i.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testInts() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.INT)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(TestUtil.randomLength(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<Integer> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(TestUtil.randomLength(random), (int)i.next()); count++; } Assert.assertEquals(COUNT, count); }