@Override public Dataset<OUT> output(OutputHint... outputHints) { Flow flow = input.getFlow(); FlatMap<IN, OUT> map = new FlatMap<>(name, flow, input, functor, evtTimeFn, Sets.newHashSet(outputHints)); flow.add(map); return map.output(); } }
@Override public Dataset<OUT> output(OutputHint... outputHints) { Flow flow = input.getFlow(); FlatMap<IN, OUT> map = new FlatMap<>(name, flow, input, functor, evtTimeFn, Sets.newHashSet(outputHints)); flow.add(map); return map.output(); } }
@Test public void testBuild() { Flow flow = Flow.create("TEST"); Dataset<String> dataset = Util.createMockDataset(flow, 1); Dataset<String> mapped = FlatMap.named("FlatMap1") .of(dataset) .using((String s, Collector<String> c) -> c.collect(s)) .output(); assertEquals(flow, mapped.getFlow()); assertEquals(1, flow.size()); FlatMap map = (FlatMap) flow.operators().iterator().next(); assertEquals(flow, map.getFlow()); assertEquals("FlatMap1", map.getName()); assertNotNull(map.getFunctor()); assertEquals(mapped, map.output()); assertNull(map.getEventTimeExtractor()); }
@Test public void testBuild_EventTimeExtractor() { Flow flow = Flow.create("TEST"); Dataset<String> dataset = Util.createMockDataset(flow, 1); Dataset<BigDecimal> mapped = FlatMap.named("FlatMap2") .of(dataset) .using((String s, Collector<BigDecimal> c) -> c.collect(null)) .eventTimeBy(Long::parseLong) // ~ consuming the original input elements .output(); assertEquals(flow, mapped.getFlow()); assertEquals(1, flow.size()); FlatMap map = (FlatMap) flow.operators().iterator().next(); assertEquals(flow, map.getFlow()); assertEquals("FlatMap2", map.getName()); assertNotNull(map.getFunctor()); assertEquals(mapped, map.output()); assertNotNull(map.getEventTimeExtractor()); }
@Test public void testBuild_WithCounters() { Flow flow = Flow.create("TEST"); Dataset<String> dataset = Util.createMockDataset(flow, 1); Dataset<String> mapped = FlatMap.named("FlatMap1") .of(dataset) .using((String s, Collector<String> c) -> { c.getCounter("my-counter").increment(); c.collect(s); }) .output(); assertEquals(flow, mapped.getFlow()); assertEquals(1, flow.size()); FlatMap map = (FlatMap) flow.operators().iterator().next(); assertEquals(flow, map.getFlow()); assertEquals("FlatMap1", map.getName()); assertNotNull(map.getFunctor()); assertEquals(mapped, map.output()); }
null); rbk = new ReduceByKey<>( getName() + "::ReduceByKey::attached", getFlow(), map.output(), Pair::getFirst, null, p -> valueExtractor.apply(p.getSecond()), null, reducer, valueComparator, getHints());
null); rbk = new ReduceByKey<>( getName() + "::ReduceByKey::attached", getFlow(), map.output(), Pair::getFirst, null, p -> valueExtractor.apply(p.getSecond()), null, reducer, valueComparator, getHints());