Refine search
@Test public void encodingAndDecodingWorks() throws Exception { KinesisRecord record = new KinesisRecord( ByteBuffer.wrap("data".getBytes(StandardCharsets.UTF_8)), "sequence", 128L, "partition", Instant.now(), Instant.now(), "stream", "shard"); CoderProperties.coderDecodeEncodeEqual(new KinesisRecordCoder(), record); } }
@Test public void createKeyedBundleKeyed() { StructuralKey<String> key = StructuralKey.of("foo", StringUtf8Coder.of()); CommittedBundle<KV<String, Integer>> keyedBundle = context .<String, KV<String, Integer>>createKeyedBundle(key, downstream) .commit(Instant.now()); assertThat(keyedBundle.getKey(), Matchers.equalTo(key)); }
@Test public void commitAfterCommitShouldThrowException() { UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); bundle.add(WindowedValue.valueInGlobalWindow(1)); CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now()); assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1))); thrown.expect(IllegalStateException.class); thrown.expectMessage("committed"); bundle.commit(Instant.now()); }
@Test public void commitAfterCommitShouldThrowException() { UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); bundle.add(WindowedValue.valueInGlobalWindow(1)); CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now()); assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1))); thrown.expect(IllegalStateException.class); thrown.expectMessage("committed"); bundle.commit(Instant.now()); }
@Test public void testExplodeWindowsInOneWindowEquals() { Instant now = Instant.now(); BoundedWindow window = new IntervalWindow(now.minus(1000L), now.plus(1000L)); WindowedValue<String> value = WindowedValue.of("foo", now, window, PaneInfo.ON_TIME_AND_ONLY_FIRING); assertThat(Iterables.getOnlyElement(value.explodeWindows()), equalTo(value)); }
@Test public void rootBundleSucceeds() { UncommittedBundle<byte[]> root = factory.createRootBundle(); byte[] array = new byte[] {0, 1, 2}; root.add(WindowedValue.valueInGlobalWindow(array)); CommittedBundle<byte[]> committed = root.commit(Instant.now()); assertThat( committed.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(array))); }
@Test public void createKeyedBundleKeyed() { CommittedBundle<KV<String, Integer>> keyedBundle = bundleFactory .<String, KV<String, Integer>>createKeyedBundle( StructuralKey.of("foo", StringUtf8Coder.of()), downstream) .commit(Instant.now()); assertThat(keyedBundle.getKey().getKey(), Matchers.equalTo("foo")); } }
@Test @Category(NeedsRunner.class) public void testUnboundedInputRate() { long numElements = 5000; long elemsPerPeriod = 10L; Duration periodLength = Duration.millis(8); PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withRate(elemsPerPeriod, periodLength)); addCountingAsserts(input, 0, numElements); long expectedRuntimeMillis = (periodLength.getMillis() * numElements) / elemsPerPeriod; Instant startTime = Instant.now(); p.run(); Instant endTime = Instant.now(); assertThat(endTime.isAfter(startTime.plus(expectedRuntimeMillis)), is(true)); }
@Test public void getUncommittedElementsEqualInput() { CommittedBundle<Integer> bundle = bundleFactory .<Integer>createBundle(created) .add(WindowedValue.valueInGlobalWindow(2)) .commit(Instant.now()); CommittedResult<PTransformNode> result = CommittedResult.create( StepTransformResult.withoutHold(transform).build(), Optional.of(bundle), Collections.emptyList(), EnumSet.noneOf(OutputType.class)); assertThat(result.getUnprocessedInputs().get(), Matchers.equalTo(bundle)); }
@Test public void unchangedSucceeds() { WindowedValue<byte[]> element = WindowedValue.valueInGlobalWindow("bar".getBytes(UTF_8)); CommittedBundle<byte[]> elements = bundleFactory.createBundle(pcollection).add(element).commit(Instant.now()); ModelEnforcement<byte[]> enforcement = factory.forBundle(elements, consumer); enforcement.beforeElement(element); enforcement.afterElement(element); enforcement.afterFinish( elements, StepTransformResult.<byte[]>withoutHold(consumer).build(), Collections.emptyList()); }
@Test public void getUncommittedElementsEqualInput() { CommittedBundle<Integer> bundle = bundleFactory .createBundle(created) .add(WindowedValue.valueInGlobalWindow(2)) .commit(Instant.now()); CommittedResult<AppliedPTransform<?, ?, ?>> result = CommittedResult.create( StepTransformResult.withoutHold(transform).build(), Optional.of(bundle), Collections.emptyList(), EnumSet.noneOf(OutputType.class)); assertThat(result.getUnprocessedInputs().get(), Matchers.equalTo(bundle)); }
@Test public void testInvokesLifecycleMethods() throws Exception { DoFn<Integer, String> fn = new LifecycleVerifyingFn(); try (ProcessFnTester<Integer, String, SomeRestriction, Void> tester = new ProcessFnTester<>( Instant.now(), fn, BigEndianIntegerCoder.of(), SerializableCoder.of(SomeRestriction.class), MAX_OUTPUTS_PER_BUNDLE, MAX_BUNDLE_DURATION)) { tester.startElement(42, new SomeRestriction()); } } }
@Test public void testUnboundedDisplayData() { Duration maxReadTime = Duration.standardHours(5); SerializableFunction<Long, Instant> timestampFn = input -> Instant.now(); PTransform<?, ?> input = GenerateSequence.from(0).to(1234).withMaxReadTime(maxReadTime).withTimestampFn(timestampFn); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("maxReadTime", maxReadTime)); assertThat(displayData, hasDisplayItem("timestampFn", timestampFn.getClass())); }
@Test public void rootBundleSucceedsIgnoresCoder() { WindowedValue<Record> one = WindowedValue.valueInGlobalWindow(new Record()); WindowedValue<Record> two = WindowedValue.valueInGlobalWindow(new Record()); CommittedBundle<Record> root = factory.<Record>createRootBundle().add(one).add(two).commit(Instant.now()); assertThat(root.getElements(), containsInAnyOrder(one, two)); }