@ProcessElement public void process(@Element Row inputRow, OutputReceiver<Row> output) { for (Object element : inputRow.getArray(0)) { output.output(Row.withSchema(schema).addValue(element).build()); } } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof Row)) { return false; } Row other = (Row) o; if (!Objects.equals(getSchema(), other.getSchema())) { return false; } for (int i = 0; i < getFieldCount(); i++) { if (!Equals.deepEquals(getValue(i), other.getValue(i), getSchema().getField(i).getType())) { return false; } } return true; }
@Override public KV<Row, Row> apply(Row input) { Row row = joinColumns.stream().map(input::getValue).collect(toRow(schema)); return KV.of(row, input); }
private static BitSet scanNullFields(Row row, boolean hasNullableFields) { BitSet nullFields = new BitSet(row.getFieldCount()); if (hasNullableFields) { for (int idx = 0; idx < row.getFieldCount(); ++idx) { if (row.getValue(idx) == null) { nullFields.set(idx); } } } return nullFields; } }
@Test public void testToRow() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); SimplePOJO pojo = createSimple("string"); Row row = registry.getToRowFunction(SimplePOJO.class).apply(pojo); assertEquals(12, row.getFieldCount()); assertEquals("string", row.getString("str")); assertEquals((byte) 1, (Object) row.getByte("aByte")); assertEquals((short) 2, (Object) row.getInt16("aShort")); assertEquals((int) 3, (Object) row.getInt32("anInt")); assertEquals((long) 4, (Object) row.getInt64("aLong")); assertEquals(true, row.getBoolean("aBoolean")); assertEquals(DATE.toInstant(), row.getDateTime("dateTime")); assertEquals(INSTANT, row.getDateTime("instant").toInstant()); assertArrayEquals(BYTE_ARRAY, row.getBytes("bytes")); assertArrayEquals(BYTE_BUFFER.array(), row.getBytes("byteBuffer")); assertEquals(BigDecimal.ONE, row.getDecimal("bigDecimal")); assertEquals("stringbuilder", row.getString("stringBuilder")); }
new DateTime().withDate(1979, 03, 14).withTime(1, 2, 3, 4).withZone(DateTimeZone.UTC); Row row = Row.withSchema(schema) .addValues( (byte) 0, (short) 1, 2, 3L, new BigDecimal(2.3), 1.2f, 3.0d, "str", dateTime, false) .build(); assertEquals((byte) 0, (Object) row.getByte("f_byte")); assertEquals((byte) 0, (Object) row.getByte(0)); assertEquals((short) 1, (Object) row.getInt16("f_int16")); assertEquals((short) 1, (Object) row.getInt16(1)); assertEquals((int) 2, (Object) row.getInt32("f_int32")); assertEquals((int) 2, (Object) row.getInt32(2)); assertEquals((long) 3, (Object) row.getInt64("f_int64")); assertEquals((long) 3, (Object) row.getInt64(3)); assertEquals(new BigDecimal(2.3), row.getDecimal("f_decimal")); assertEquals(new BigDecimal(2.3), row.getDecimal(4)); assertEquals(1.2f, row.getFloat("f_float"), 0); assertEquals(1.2f, row.getFloat(5), 0); assertEquals(3.0d, row.getDouble("f_double"), 0); assertEquals(3.0d, row.getDouble(6), 0); assertEquals("str", row.getString("f_string")); assertEquals("str", row.getString(7)); assertEquals(dateTime, row.getDateTime("f_datetime")); assertEquals(dateTime, row.getDateTime(8)); assertEquals(false, row.getBoolean("f_boolean")); assertEquals(false, row.getBoolean(9));
@ProcessElement public void process(@Element Row row, OutputReceiver<Row> r) { r.output( Row.withSchema(schema2) .addValues(row.getString(0), row.getInt32(1)) .build()); } }))
public static Row toBeamRow(GenericRecord record, Schema schema) { List<Object> values = new ArrayList(); for (int i = 0; i < record.getSchema().getFields().size(); i++) { org.apache.avro.Schema.Field avroField = record.getSchema().getFields().get(i); values.add(AvroUtils.convertAvroFormat(schema.getField(i), record.get(avroField.name()))); } return Row.withSchema(schema).addValues(values).build(); }
@Test public void testCreatesNestedRow() { Schema nestedType = Stream.of(Schema.Field.of("f1_str", Schema.FieldType.STRING)).collect(toSchema()); Schema type = Stream.of( Schema.Field.of("f_int", Schema.FieldType.INT32), Schema.Field.of("nested", Schema.FieldType.row(nestedType))) .collect(toSchema()); Row nestedRow = Row.withSchema(nestedType).addValues("foobar").build(); Row row = Row.withSchema(type).addValues(42, nestedRow).build(); assertEquals((int) 42, (Object) row.getInt32("f_int")); assertEquals("foobar", row.getRow("nested").getString("f1_str")); }
/** * Get a {@link TypeName#INT32} value by field index, {@link ClassCastException} is thrown if * schema doesn't match. */ @Nullable public Integer getInt32(int idx) { return getValue(idx); }
@ProcessElement public void process(@Element Row row, OutputReceiver<Row> out) { @Nullable List<Object> rawValues = row.getArray(unnestIndex); if (rawValues == null) { return; } for (Object uncollectedValue : rawValues) { out.output( Row.withSchema(outputSchema) .addValues(row.getValues()) .addValue(uncollectedValue) .build()); } } }
@Test public void testCreateExplicitSchema() { PCollection<String> out = p.apply( Create.of("a", "b", "c", "d") .withSchema( STRING_SCHEMA, s -> Row.withSchema(STRING_SCHEMA).addValue(s).build(), r -> r.getString("field"))); assertThat(out.getCoder(), instanceOf(SchemaCoder.class)); }
private static void unnestRow(Row input, Row.Builder output) { for (int i = 0; i < input.getSchema().getFieldCount(); ++i) { Field field = input.getSchema().getField(i); if (field.getType().getTypeName().isCompositeType()) { unnestRow(input.getRow(i), output); } else { output.addValue(input.getValue(i)); } } } /** A {@link PTransform} that unnests nested row. */
/** * Get a {@link TypeName#DATETIME} value by field name, {@link IllegalStateException} is thrown if * schema doesn't match. */ @Nullable public ReadableDateTime getDateTime(String fieldName) { return getDateTime(getSchema().indexOf(fieldName)); }