/** * Get a {@link TypeName#DATETIME} value by field name, {@link IllegalStateException} is thrown if * schema doesn't match. */ @Nullable public ReadableDateTime getDateTime(String fieldName) { return getDateTime(getSchema().indexOf(fieldName)); }
@Override public Void apply(Iterable<Row> input) { Iterator<Row> iter = input.iterator(); assertTrue(iter.hasNext()); Row row = iter.next(); // LOCALTIME DateTime date = DateTime.now(); assertTrue(date.getMillis() - row.getDateTime(0).getMillis() < 1000); assertTrue(date.getMillis() - row.getDateTime(1).getMillis() < 1000); assertTrue(date.getMillis() - row.getDateTime(2).getMillis() < 1000); assertTrue(date.getMillis() - row.getDateTime(3).getMillis() < 1000); assertTrue(date.getMillis() - row.getDateTime(4).getMillis() < 1000); assertFalse(iter.hasNext()); return null; } }
/** Extract timestamps from the windowFieldIndex, then window into windowFns. */ private PCollection<Row> assignTimestampsAndWindow(PCollection<Row> upstream) { PCollection<Row> windowedStream; windowedStream = upstream .apply( "assignEventTimestamp", WithTimestamps.<Row>of(row -> row.getDateTime(windowFieldIndex).toInstant()) .withAllowedTimestampSkew(new Duration(Long.MAX_VALUE))) .setCoder(upstream.getCoder()) .apply(Window.into(windowFn)); return windowedStream; }
@Override public PCollection<Row> buildIOReader(PBegin begin) { TestStream.Builder<Row> values = TestStream.create( schema, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Pair<Duration, List<Row>> pair : timestampedRows) { values = values.advanceWatermarkTo(new Instant(0).plus(pair.getKey())); for (int i = 0; i < pair.getValue().size(); i++) { values = values.addElements( TimestampedValue.of( pair.getValue().get(i), new Instant(pair.getValue().get(i).getDateTime(timestampField)))); } } return begin .apply( "MockedUnboundedTable_" + COUNTER.incrementAndGet(), values.advanceWatermarkToInfinity()) .setRowSchema(getSchema()); } }
assertEquals("2018-11-01", row.getDateTime("f_date").toString("yyyy-MM-dd")); assertEquals("15:23:59.000", row.getDateTime("f_time").toString("HH:mm:ss.SSS")); assertEquals(parseTimestampWithUTCTimeZone("2018-07-01 21:26:07.123"), row.getDateTime("f_ts"));
/** * Builds an unbounded {@link PCollection} in {@link Pipeline} set by {@link * #inPipeline(Pipeline)}. * * <p>If timestamp field was set with {@link #withTimestampField(String)} then watermark will be * advanced to the values from that field. */ public PCollection<Row> buildUnbounded() { checkArgument(pipeline != null); checkArgument(rows.size() > 0); if (type == null) { type = rows.get(0).getSchema(); } TestStream.Builder<Row> values = TestStream.create( type, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rows) { if (timestampField != null) { values = values.advanceWatermarkTo(new Instant(row.getDateTime(timestampField))); } values = values.addElements(row); } return PBegin.in(pipeline).apply("unboundedPCollection", values.advanceWatermarkToInfinity()); } }
private PCollection<Row> prepareUnboundedPCollection2() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); Row row = rowsInTableA.get(0); values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); return PBegin.in(pipeline) .apply("unboundedInput2", values.advanceWatermarkToInfinity()) .apply( "unboundedInput2.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); } }
private PCollection<Row> prepareUnboundedPCollection1() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rowsInTableA) { values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); } return PBegin.in(pipeline) .apply("unboundedInput1", values.advanceWatermarkToInfinity()) .apply( "unboundedInput1.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); }
@Test public void testToRow() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); SimplePOJO pojo = createSimple("string"); Row row = registry.getToRowFunction(SimplePOJO.class).apply(pojo); assertEquals(12, row.getFieldCount()); assertEquals("string", row.getString("str")); assertEquals((byte) 1, (Object) row.getByte("aByte")); assertEquals((short) 2, (Object) row.getInt16("aShort")); assertEquals((int) 3, (Object) row.getInt32("anInt")); assertEquals((long) 4, (Object) row.getInt64("aLong")); assertEquals(true, row.getBoolean("aBoolean")); assertEquals(DATE.toInstant(), row.getDateTime("dateTime")); assertEquals(INSTANT, row.getDateTime("instant").toInstant()); assertArrayEquals(BYTE_ARRAY, row.getBytes("bytes")); assertArrayEquals(BYTE_BUFFER.array(), row.getBytes("byteBuffer")); assertEquals(BigDecimal.ONE, row.getDecimal("bigDecimal")); assertEquals("stringbuilder", row.getString("stringBuilder")); }
@Test public void testRecursiveGetters() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); SchemaTestUtils.assertSchemaEquivalent( NESTED_POJO_SCHEMA, registry.getSchema(NestedPOJO.class)); NestedPOJO pojo = new NestedPOJO(createSimple("string")); Row row = registry.getToRowFunction(NestedPOJO.class).apply(pojo); Row nestedRow = row.getRow("nested"); assertEquals("string", nestedRow.getString("str")); assertEquals((byte) 1, (Object) nestedRow.getByte("aByte")); assertEquals((short) 2, (Object) nestedRow.getInt16("aShort")); assertEquals((int) 3, (Object) nestedRow.getInt32("anInt")); assertEquals((long) 4, (Object) nestedRow.getInt64("aLong")); assertEquals(true, nestedRow.getBoolean("aBoolean")); assertEquals(DATE.toInstant(), nestedRow.getDateTime("dateTime")); assertEquals(INSTANT, nestedRow.getDateTime("instant").toInstant()); assertArrayEquals("not equal", BYTE_ARRAY, nestedRow.getBytes("bytes")); assertArrayEquals("not equal", BYTE_BUFFER.array(), nestedRow.getBytes("byteBuffer")); assertEquals(BigDecimal.ONE, nestedRow.getDecimal("bigDecimal")); assertEquals("stringbuilder", nestedRow.getString("stringBuilder")); }
@Test public void testToRow() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); SimpleBean bean = createSimple("string"); Row row = registry.getToRowFunction(SimpleBean.class).apply(bean); assertEquals(12, row.getFieldCount()); assertEquals("string", row.getString("str")); assertEquals((byte) 1, (Object) row.getByte("aByte")); assertEquals((short) 2, (Object) row.getInt16("aShort")); assertEquals((int) 3, (Object) row.getInt32("anInt")); assertEquals((long) 4, (Object) row.getInt64("aLong")); assertEquals(true, (Object) row.getBoolean("aBoolean")); assertEquals(DATE.toInstant(), row.getDateTime("dateTime")); assertEquals(DATE.toInstant(), row.getDateTime("instant")); assertArrayEquals(BYTE_ARRAY, row.getBytes("bytes")); assertArrayEquals(BYTE_ARRAY, row.getBytes("byteBuffer")); assertEquals(BigDecimal.ONE, row.getDecimal("bigDecimal")); assertEquals("stringbuilder", row.getString("stringBuilder")); }
@Test public void testRecursiveGetters() throws NoSuchSchemaException { SchemaRegistry registry = SchemaRegistry.createDefault(); SchemaTestUtils.assertSchemaEquivalent( NESTED_BEAN_SCHEMA, registry.getSchema(NestedBean.class)); NestedBean bean = new NestedBean(createSimple("string")); Row row = registry.getToRowFunction(NestedBean.class).apply(bean); Row nestedRow = row.getRow("nested"); assertEquals("string", nestedRow.getString("str")); assertEquals((byte) 1, (Object) nestedRow.getByte("aByte")); assertEquals((short) 2, (Object) nestedRow.getInt16("aShort")); assertEquals((int) 3, (Object) nestedRow.getInt32("anInt")); assertEquals((long) 4, (Object) nestedRow.getInt64("aLong")); assertEquals(true, nestedRow.getBoolean("aBoolean")); assertEquals(DATE.toInstant(), nestedRow.getDateTime("dateTime")); assertEquals(DATE.toInstant(), nestedRow.getDateTime("instant")); assertArrayEquals("not equal", BYTE_ARRAY, nestedRow.getBytes("bytes")); assertArrayEquals("not equal", BYTE_ARRAY, nestedRow.getBytes("byteBuffer")); assertEquals(BigDecimal.ONE, nestedRow.getDecimal("bigDecimal")); assertEquals("stringbuilder", nestedRow.getString("stringBuilder")); }
assertEquals("str", row.getString("f_string")); assertEquals("str", row.getString(7)); assertEquals(dateTime, row.getDateTime("f_datetime")); assertEquals(dateTime, row.getDateTime(8)); assertEquals(false, row.getBoolean("f_boolean")); assertEquals(false, row.getBoolean(9));
assertNull(row.getString("f_string")); assertNull(row.getString(7)); assertNull(row.getDateTime("f_datetime")); assertNull(row.getDateTime(8)); assertNull(row.getBoolean("f_boolean")); assertNull(row.getBoolean(9));