@Test public void testCreateMapWithRowValue() { Schema nestedType = Stream.of(Schema.Field.of("f1_str", FieldType.STRING)).collect(toSchema()); Map<Integer, Row> data = ImmutableMap.<Integer, Row>builder() .put(1, Row.withSchema(nestedType).addValues("one").build()) .put(2, Row.withSchema(nestedType).addValues("two").build()) .build(); Schema type = Stream.of(Schema.Field.of("map", FieldType.map(FieldType.INT32, FieldType.row(nestedType)))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
@Test public void testCreatesRowArray() { Schema nestedType = Stream.of(Schema.Field.of("f1_str", FieldType.STRING)).collect(toSchema()); List<Row> data = Lists.newArrayList( Row.withSchema(nestedType).addValues("one").build(), Row.withSchema(nestedType).addValues("two").build(), Row.withSchema(nestedType).addValues("three").build()); Schema type = Stream.of(Schema.Field.of("array", FieldType.array(FieldType.row(nestedType)))) .collect(toSchema()); Row row = Row.withSchema(type).addArray(data).build(); assertEquals(data, row.getArray("array")); }
static Row decodeDelegate(Schema schema, Coder[] coders, InputStream inputStream) throws IOException { BitSet nullFields = NULL_LIST_CODER.decode(inputStream); List<Object> fieldValues = Lists.newArrayListWithCapacity(coders.length); for (int i = 0; i < coders.length; ++i) { if (nullFields.get(i)) { fieldValues.add(null); } else { fieldValues.add(coders[i].decode(inputStream)); } } // We call attachValues instead of setValues. setValues validates every element in the list // is of the proper type, potentially converts to the internal type Row stores, and copies // all values. Since we assume that decode is always being called on a previously-encoded // Row, the values should already be validated and of the correct type. So, we can save // some processing by simply transferring ownership of the list to the Row. return Row.withSchema(schema).attachValues(fieldValues).build(); } }
Create.of( Row.withSchema(inputType) .addValues( 1, Arrays.asList( Row.withSchema(elementSchema).addValues("AA", 11).build(), Row.withSchema(elementSchema).addValues("BB", 22).build())) .build(), Row.withSchema(inputType) .addValues( 2, Arrays.asList( Row.withSchema(elementSchema).addValues("CC", 33).build(), Row.withSchema(elementSchema).addValues("DD", 44).build())) .build()) .withSchema( inputType, SerializableFunctions.identity(), SerializableFunctions.identity())); .containsInAnyOrder( Row.withSchema(resultSchema) .addArray( Arrays.asList( Row.withSchema(elementSchema).addValues("AA", 11).build(), Row.withSchema(elementSchema).addValues("BB", 22).build())) .build(), Row.withSchema(resultSchema) .addArray( Arrays.asList( Row.withSchema(elementSchema).addValues("CC", 33).build(),
Create.of( Row.withSchema(inputType) .addValues( 1, Row.withSchema(nestedSchema) .addValues(312, "CC", 313, Arrays.asList("one", "two")) .build()) .build(), Row.withSchema(inputType) .addValues( 2, Row.withSchema(nestedSchema) .addValues(412, "DD", 413, Arrays.asList("three", "four")) .build()) .build()) .withSchema( inputType, SerializableFunctions.identity(), SerializableFunctions.identity())); Row.withSchema(resultSchema).addArray(Arrays.asList("one", "two")).build(), Row.withSchema(resultSchema).addArray(Arrays.asList("three", "four")).build());
Row row1 = Row.withSchema(INPUT_SCHEMA) .addValues(42) .addArray(Arrays.asList("111", "222", "333")) .build(); Row.withSchema(INPUT_SCHEMA).addValues(13).addArray(Arrays.asList("444", "555")).build(); Row.withSchema(resultType).addValues(42, "111").build(), Row.withSchema(resultType).addValues(42, "222").build(), Row.withSchema(resultType).addValues(42, "333").build(), Row.withSchema(resultType).addValues(13, "444").build(), Row.withSchema(resultType).addValues(13, "555").build());
KV.of( Row.withSchema(keySchema) .addValue(Row.withSchema(innerKeySchema).addValue(1L).build()) .build(), Row.withSchema(valueSchema).addValue(3L).addValue(5).addArray(2L).build()), KV.of( Row.withSchema(keySchema) .addValue(Row.withSchema(innerKeySchema).addValue(2L).build()) .build(), Row.withSchema(valueSchema).addValue(7L).addValue(9).addArray(4L).build())); PAssert.that(aggregations).satisfies(actual -> containsKvs(expected, actual));
ImmutableList.of( KV.of( Row.withSchema(keySchema).addValue(1L).build(), Row.withSchema(valueSchema).addValue(3L).addValue(5).addArray(2L).build()), KV.of( Row.withSchema(keySchema).addValue(2L).build(), Row.withSchema(valueSchema).addValue(7L).addValue(9).addArray(4L).build())); PAssert.that(aggregations).satisfies(actual -> containsKvs(expected, actual));
@Test @Category(NeedsRunner.class) public void testGloballyWithSchemaAggregateFn() { Collection<AggregatePojos> elements = ImmutableList.of( new AggregatePojos(1, 1, 2), new AggregatePojos(2, 1, 3), new AggregatePojos(3, 2, 4), new AggregatePojos(4, 2, 5)); PCollection<Row> aggregate = pipeline .apply(Create.of(elements)) .apply( Group.<AggregatePojos>globally() .aggregateField("field1", Sum.ofLongs(), "field1_sum") .aggregateField("field3", Sum.ofIntegers(), "field3_sum") .aggregateField("field1", Top.largestLongsFn(1), "field1_top")); Schema aggregateSchema = Schema.builder() .addInt64Field("field1_sum") .addInt32Field("field3_sum") .addArrayField("field1_top", FieldType.INT64) .build(); Row expectedRow = Row.withSchema(aggregateSchema).addValues(10L, 14).addArray(4L).build(); PAssert.that(aggregate).containsInAnyOrder(expectedRow); pipeline.run(); }
@Test public void testMapFieldSetters() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); Row row1 = createSimpleRow("string1"); Row row2 = createSimpleRow("string2"); Row row3 = createSimpleRow("string3"); Row row = Row.withSchema(NESTED_MAP_BEAN_SCHEMA) .addValue( ImmutableMap.of( "simple1", row1, "simple2", row2, "simple3", row3)) .build(); NestedMapBean bean = registry.getFromRowFunction(NestedMapBean.class).apply(row); assertEquals(3, bean.getMap().size()); assertEquals("string1", bean.getMap().get("simple1").getStr()); assertEquals("string2", bean.getMap().get("simple2").getStr()); assertEquals("string3", bean.getMap().get("simple3").getStr()); } }
@Test public void testTimestampWithZeroTimezone() throws Exception { Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); TestTableProvider tableProvider = new TestTableProvider(); Connection connection = JdbcDriver.connect(tableProvider); // A table with one TIMESTAMP column Schema schema = Schema.builder().addDateTimeField("ts").build(); connection .createStatement() .executeUpdate("CREATE EXTERNAL TABLE test (ts TIMESTAMP) TYPE 'test'"); ReadableInstant july1 = ISODateTimeFormat.dateTimeParser().parseDateTime("2018-07-01T01:02:03Z"); tableProvider.addRows("test", Row.withSchema(schema).addValue(july1).build()); ResultSet selectResult = connection.createStatement().executeQuery(String.format("SELECT ts FROM test")); selectResult.next(); Timestamp ts = selectResult.getTimestamp(1, cal); assertThat( String.format( "Wrote %s to a table, but got back %s", ISODateTimeFormat.basicDateTime().print(july1), ISODateTimeFormat.basicDateTime().print(ts.getTime())), ts.getTime(), equalTo(july1.getMillis())); }
@Test public void testRecursiveSetters() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); Row nestedRow = createSimpleRow("string"); Row row = Row.withSchema(NESTED_BEAN_SCHEMA).addValue(nestedRow).build(); NestedBean bean = registry.getFromRowFunction(NestedBean.class).apply(row); assertEquals("string", bean.getNested().getStr()); assertEquals((byte) 1, bean.getNested().getaByte()); assertEquals((short) 2, bean.getNested().getaShort()); assertEquals((int) 3, bean.getNested().getAnInt()); assertEquals((long) 4, bean.getNested().getaLong()); assertEquals(true, bean.getNested().isaBoolean()); assertEquals(DATE, bean.getNested().getDateTime()); assertEquals(DATE.toInstant(), bean.getNested().getInstant()); assertArrayEquals("not equal", BYTE_ARRAY, bean.getNested().getBytes()); assertArrayEquals("not equal", BYTE_ARRAY, bean.getNested().getByteBuffer().array()); assertEquals(BigDecimal.ONE, bean.getNested().getBigDecimal()); assertEquals("stringbuilder", bean.getNested().getStringBuilder().toString()); }
@Test public void testProjectArrayField() { PCollection<Row> input = pCollectionOf2Elements(); Schema resultType = Schema.builder() .addInt32Field("f_int") .addArrayField("f_stringArr", Schema.FieldType.STRING) .build(); PCollection<Row> result = input.apply("sqlQuery", SqlTransform.query("SELECT f_int, f_stringArr FROM PCOLLECTION")); PAssert.that(result) .containsInAnyOrder( Row.withSchema(resultType).addValues(1).addArray(Arrays.asList("111", "222")).build(), Row.withSchema(resultType) .addValues(2) .addArray(Arrays.asList("33", "44", "55")) .build()); pipeline.run(); }
@Test public void testCreateMapWithMapValueWithNull() { Map<Integer, Map<Integer, String>> data = new HashMap(); Map<Integer, String> innerData = new HashMap(); innerData.put(11, null); innerData.put(12, "value3"); data.put(1, ImmutableMap.of(1, "value1")); data.put(2, ImmutableMap.of(2, "value2")); data.put(3, null); data.put(4, innerData); Schema type = Stream.of( Schema.Field.of( "map", FieldType.map( FieldType.INT32, FieldType.map(FieldType.INT32, FieldType.STRING, true), true))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
private PCollection<Row> pCollectionOf2Elements() { return pipeline.apply( "boundedInput1", Create.of( Row.withSchema(INPUT_ROW_TYPE) .addValues(1) .addValue(ImmutableMap.of("key11", 11, "key22", 22)) .build(), Row.withSchema(INPUT_ROW_TYPE) .addValues(2) .addValue(ImmutableMap.of("key33", 33, "key44", 44, "key55", 55)) .build()) .withSchema( INPUT_ROW_TYPE, SerializableFunctions.identity(), SerializableFunctions.identity())); } }
@Test public void testSingleElement() throws Exception { Row inputRow = Row.withSchema(INPUT_SCHEMA).addValues(1).addArray(Arrays.asList("111")).build(); PCollection<Row> input = pipeline.apply( "boundedInput1", Create.of(inputRow) .withSchema( INPUT_SCHEMA, SerializableFunctions.identity(), SerializableFunctions.identity())); Schema resultType = Schema.builder().addStringField("f_arrElem").build(); PCollection<Row> result = input.apply("sqlQuery", SqlTransform.query("SELECT ELEMENT(f_stringArr) FROM PCOLLECTION")); PAssert.that(result).containsInAnyOrder(Row.withSchema(resultType).addValues("111").build()); pipeline.run(); }
private void tryGetters(SchemaRegistry registry) throws NoSuchSchemaException { assertEquals(STRING_SCHEMA, registry.getSchema(String.class)); assertEquals(STRING_SCHEMA, registry.getSchema(TypeDescriptors.strings())); assertEquals( Row.withSchema(STRING_SCHEMA).addValue("foobar").build(), registry.getToRowFunction(String.class).apply("foobar")); assertEquals( Row.withSchema(STRING_SCHEMA).addValue("foobar").build(), registry.getToRowFunction(TypeDescriptors.strings()).apply("foobar")); assertEquals(INTEGER_SCHEMA, registry.getSchema(Integer.class)); assertEquals(INTEGER_SCHEMA, registry.getSchema(TypeDescriptors.integers())); assertEquals( Row.withSchema(INTEGER_SCHEMA).addValue(42).build(), registry.getToRowFunction(Integer.class).apply(42)); assertEquals( Row.withSchema(INTEGER_SCHEMA).addValue(42).build(), registry.getToRowFunction(TypeDescriptors.integers()).apply(42)); thrown.expect(NoSuchSchemaException.class); registry.getSchema(Double.class); }
@Test public void testMapFieldSetters() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); Row row1 = createSimpleRow("string1"); Row row2 = createSimpleRow("string2"); Row row3 = createSimpleRow("string3"); Row row = Row.withSchema(NESTED_MAP_POJO_SCHEMA) .addValue( ImmutableMap.of( "simple1", row1, "simple2", row2, "simple3", row3)) .build(); NestedMapPOJO pojo = registry.getFromRowFunction(NestedMapPOJO.class).apply(row); assertEquals(3, pojo.map.size()); assertEquals("string1", pojo.map.get("simple1").str); assertEquals("string2", pojo.map.get("simple2").str); assertEquals("string3", pojo.map.get("simple3").str); }
@Test public void testRecursiveSetters() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); Row nestedRow = createSimpleRow("string"); Row row = Row.withSchema(NESTED_POJO_SCHEMA).addValue(nestedRow).build(); NestedPOJO pojo = registry.getFromRowFunction(NestedPOJO.class).apply(row); assertEquals("string", pojo.nested.str); assertEquals((byte) 1, pojo.nested.aByte); assertEquals((short) 2, pojo.nested.aShort); assertEquals((int) 3, pojo.nested.anInt); assertEquals((long) 4, pojo.nested.aLong); assertEquals(true, pojo.nested.aBoolean); assertEquals(DATE, pojo.nested.dateTime); assertEquals(INSTANT, pojo.nested.instant); assertArrayEquals("not equal", BYTE_ARRAY, pojo.nested.bytes); assertEquals(BYTE_BUFFER, pojo.nested.byteBuffer); assertEquals(BigDecimal.ONE, pojo.nested.bigDecimal); assertEquals("stringbuilder", pojo.nested.stringBuilder.toString()); }