@Test public void testViewUnboundedAsIterableDirect() { testViewUnbounded(pipeline, View.asIterable()); }
@Test public void testViewNonmergingAsIterableDirect() { testViewNonmerging(pipeline, View.asIterable()); }
@Test public void testViewNonmergingAsIterableBatch() { testViewNonmerging(createTestBatchRunner(), View.asIterable()); }
@Test public void testViewUnboundedAsIterableStreaming() { testViewUnbounded(createTestStreamingRunner(), View.asIterable()); }
@Test public void testViewUnboundedAsIterableBatch() { testViewUnbounded(createTestBatchRunner(), View.asIterable()); }
@Test public void testViewNonmergingAsIterableStreaming() { testViewNonmerging(createTestStreamingRunner(), View.asIterable()); }
@Test public void createViewWithViewFnNotCreatePCollectionView() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(view.getViewFn().getClass()); assertThat(matcher.matches(getAppliedTransform(View.asIterable())), is(false)); }
@Before public void setUp() { PCollection<String> pc = Pipeline.create().apply(Create.of("1")); view1 = pc.apply(Window.into(FixedWindows.of(new Duration(WINDOW_MSECS_1)))) .apply(View.asIterable()); view2 = pc.apply(Window.into(FixedWindows.of(new Duration(WINDOW_MSECS_2)))) .apply(View.asIterable()); }
@Before public void setUp() { PCollection<String> pc = Pipeline.create().apply(Create.of("1")); view1 = pc.apply(Window.into(FixedWindows.of(new Duration(WINDOW_MSECS_1)))) .apply(View.asIterable()); view2 = pc.apply(Window.into(FixedWindows.of(new Duration(WINDOW_MSECS_2)))) .apply(View.asIterable()); }
@Before public void setUp() { PCollection<String> pc = Pipeline.create().apply(Create.of("1")); view1 = pc.apply(Window.into(FixedWindows.of(new Duration(WINDOW_MSECS_1)))) .apply(View.asIterable()); view2 = pc.apply(Window.into(FixedWindows.of(new Duration(WINDOW_MSECS_2)))) .apply(View.asIterable()); }
@Override public PCollection<Iterable<T>> expand(PCollection<T> input) { final PCollectionView<Iterable<T>> view = input.apply(View.asIterable()); return input .getPipeline() .apply(Create.of((Void) null).withCoder(VoidCoder.of())) .apply( ParDo.of( new DoFn<Void, Iterable<T>>() { @ProcessElement public void processElement(ProcessContext c) { c.output(c.sideInput(view)); } }) .withSideInputs(view)); } }
@Test public void createViewWithViewFnDifferentViewFn() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); // Purposely create a subclass to get a different class then what was expected. ViewFn<?, ?> viewFn = new PCollectionViews.IterableViewFn() {}; CreatePCollectionView<?, ?> createView = CreatePCollectionView.of(view); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(viewFn.getClass()); assertThat(matcher.matches(getAppliedTransform(createView)), is(false)); }
@Test public void createViewWithViewFn() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); ViewFn<?, ?> viewFn = view.getViewFn(); CreatePCollectionView<?, ?> createView = CreatePCollectionView.of(view); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(viewFn.getClass()); assertThat(matcher.matches(getAppliedTransform(createView)), is(true)); }
@Test public void testToIterableTranslationWithIsmSideInput() throws Exception { // A "change detector" test that makes sure the translation // of getting a PCollectionView<Iterable<T>> does not change // in bad ways during refactor DataflowPipelineOptions options = buildPipelineOptions(); DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options); Pipeline pipeline = Pipeline.create(options); pipeline.apply(Create.of(1, 2, 3)).apply(View.asIterable()); DataflowRunner runner = DataflowRunner.fromOptions(options); runner.replaceTransforms(pipeline); Job job = translator.translate(pipeline, runner, Collections.emptyList()).getJob(); assertAllStepOutputsHaveUniqueIds(job); List<Step> steps = job.getSteps(); assertEquals(3, steps.size()); @SuppressWarnings("unchecked") List<Map<String, Object>> toIsmRecordOutputs = (List<Map<String, Object>>) steps.get(1).getProperties().get(PropertyNames.OUTPUT_INFO); assertTrue( Structs.getBoolean(Iterables.getOnlyElement(toIsmRecordOutputs), "use_indexed_format")); Step collectionToSingletonStep = steps.get(2); assertEquals("CollectionToSingleton", collectionToSingletonStep.getKind()); }
@Before public void setup() { MockitoAnnotations.initMocks(this); PCollection<Integer> create = pipeline.apply("forBaseCollection", Create.of(1, 2, 3, 4)); mapView = create.apply("forKeyTypes", WithKeys.of("foo")).apply("asMapView", View.asMap()); singletonView = create.apply("forCombinedTypes", Mean.<Integer>globally().asSingletonView()); iterableView = create.apply("asIterableView", View.asIterable()); container = SideInputContainer.create(context, ImmutableList.of(iterableView, mapView, singletonView)); }
@Test public void testNewInputReplacesPreviousInput() { // new input should completely replace old input // the creation of the Iterable that has the side input // contents happens upstream. this is also where // accumulation/discarding is decided. SideInputHandler sideInputHandler = new SideInputHandler(ImmutableList.of(view1), InMemoryStateInternals.<Void>forKey(null)); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(WINDOW_MSECS_1)); // add a first value for view1 sideInputHandler.addSideInputValue( view1, valuesInWindow(materializeValuesFor(View.asIterable(), "Hello"), new Instant(0), window)); assertThat(sideInputHandler.get(view1, window), contains("Hello")); // subsequent values should replace existing values sideInputHandler.addSideInputValue( view1, valuesInWindow( materializeValuesFor(View.asIterable(), "Ciao", "Buongiorno"), new Instant(0), window)); assertThat(sideInputHandler.get(view1, window), contains("Ciao", "Buongiorno")); }
@Test public void writeForMultipleIdenticalElementsInSameWindowSucceeds() throws Exception { ImmutableList.Builder<WindowedValue<?>> valuesBuilder = ImmutableList.builder(); for (Object materializedValue : materializeValuesFor(View.asIterable(), 44, 44)) { valuesBuilder.add( WindowedValue.of( materializedValue, FIRST_WINDOW.maxTimestamp().minus(200L), FIRST_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING)); } container.write(iterableView, valuesBuilder.build()); assertThat( container .createReaderForViews(ImmutableList.of(iterableView)) .get(iterableView, FIRST_WINDOW), contains(44, 44)); }
@Test public void testViewGetName() { assertEquals("View.AsSingleton", View.<Integer>asSingleton().getName()); assertEquals("View.AsIterable", View.<Integer>asIterable().getName()); assertEquals("View.AsMap", View.<String, Integer>asMap().getName()); assertEquals("View.AsMultimap", View.<String, Integer>asMultimap().getName()); }
@Test @Category(ValidatesRunner.class) public void testIterableSideInput() { final PCollectionView<Iterable<Integer>> view = pipeline.apply("CreateSideInput", Create.of(11, 13, 17, 23)).apply(View.asIterable()); PCollection<Integer> output = pipeline .apply("CreateMainInput", Create.of(29, 31)) .apply( "OutputSideInputs", ParDo.of( new DoFn<Integer, Integer>() { @ProcessElement public void processElement(ProcessContext c) { for (Integer i : c.sideInput(view)) { c.output(i); } } }) .withSideInputs(view)); PAssert.that(output).containsInAnyOrder(11, 13, 17, 23, 11, 13, 17, 23); pipeline.run(); }
@Test public void testIsReady() { SideInputHandler sideInputHandler = new SideInputHandler( ImmutableList.of(view1, view2), InMemoryStateInternals.<Void>forKey(null)); IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(WINDOW_MSECS_1)); IntervalWindow secondWindow = new IntervalWindow(new Instant(0), new Instant(WINDOW_MSECS_2)); // side input should not yet be ready assertFalse(sideInputHandler.isReady(view1, firstWindow)); // add a value for view1 sideInputHandler.addSideInputValue( view1, valuesInWindow( materializeValuesFor(View.asIterable(), "Hello"), new Instant(0), firstWindow)); // now side input should be ready assertTrue(sideInputHandler.isReady(view1, firstWindow)); // second window input should still not be ready assertFalse(sideInputHandler.isReady(view1, secondWindow)); }