@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return FullJoin.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Optional<Integer> l, Optional<Long> r, Collector<String> c) -> c.collect(l.orElse(null) + "+" + r.orElse(null))) .output(); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return FullJoin.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Optional<Integer> l, Optional<Long> r, Collector<String> c) -> c.collect(l.orElse(null) + "+" + r.orElse(null))) .output(); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return LeftJoin.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Integer l, Optional<Long> r, Collector<String> c) -> c.collect(l + "+" + r.orElse(null))) .output(); }
.using((Integer l, Pair<Integer, Long> r, Collector<Long> c) -> c.collect(r.getSecond())) .output() .persist(new VoidSink<>());
@Test public void testBuild() { Flow flow = Flow.create("TEST"); Dataset<String> left = Util.createMockDataset(flow, 2); Dataset<String> right = Util.createMockDataset(flow, 3); Dataset<Pair<Integer, String>> joined = Join.named("Join1") .of(left, right) .by(String::length, String::length) .using((String l, String r, Collector<String> c) -> { // no-op }) .output(); assertEquals(flow, joined.getFlow()); assertEquals(1, flow.size()); Join join = (Join) flow.operators().iterator().next(); assertEquals(flow, join.getFlow()); assertEquals("Join1", join.getName()); assertNotNull(join.leftKeyExtractor); assertNotNull(join.rightKeyExtractor); assertEquals(joined, join.output()); assertNull(join.getWindowing()); assertEquals(Join.Type.INNER, join.getType()); }
@Test public void testBuild_WithCounters() { Flow flow = Flow.create("TEST"); Dataset<String> left = Util.createMockDataset(flow, 2); Dataset<String> right = Util.createMockDataset(flow, 3); Dataset<Pair<Integer, String>> joined = Join.named("Join1") .of(left, right) .by(String::length, String::length) .using((String l, String r, Collector<String> c) -> { c.getCounter("my-counter").increment(); c.collect(l + r); }) .output(); assertEquals(flow, joined.getFlow()); assertEquals(1, flow.size()); Join join = (Join) flow.operators().iterator().next(); assertEquals(flow, join.getFlow()); assertEquals("Join1", join.getName()); assertNotNull(join.leftKeyExtractor); assertNotNull(join.rightKeyExtractor); assertEquals(joined, join.output()); assertNull(join.getWindowing()); assertEquals(Join.Type.INNER, join.getType()); }
@Test public void testBuild_RightJoin() { Flow flow = Flow.create("TEST"); Dataset<String> left = Util.createMockDataset(flow, 1); Dataset<String> right = Util.createMockDataset(flow, 1); RightJoin.named("Join1") .of(left, right) .by(String::length, String::length) .using((Optional<String> l, String r, Collector<String> c) -> { // no-op }) .output(); Join join = (Join) flow.operators().iterator().next(); assertEquals(Join.Type.RIGHT, join.getType()); }
@Test public void testBuild_LeftJoin() { Flow flow = Flow.create("TEST"); Dataset<String> left = Util.createMockDataset(flow, 1); Dataset<String> right = Util.createMockDataset(flow, 1); LeftJoin.named("Join1") .of(left, right) .by(String::length, String::length) .using((String l, Optional<String> r, Collector<String> c) -> { // no-op }) .output(); Join join = (Join) flow.operators().iterator().next(); assertEquals(Join.Type.LEFT, join.getType()); }
@Test public void testBuild_ImplicitName() { Flow flow = Flow.create("TEST"); Dataset<String> left = Util.createMockDataset(flow, 1); Dataset<String> right = Util.createMockDataset(flow, 1); Join.of(left, right) .by(String::length, String::length) .using((String l, String r, Collector<String> c) -> { // no-op }) .output(); Join join = (Join) flow.operators().iterator().next(); assertEquals("Join", join.getName()); }
@Test public void testBuild_FullJoin() { Flow flow = Flow.create("TEST"); Dataset<String> left = Util.createMockDataset(flow, 1); Dataset<String> right = Util.createMockDataset(flow, 1); FullJoin.named("Join1") .of(left, right) .by(String::length, String::length) .using((Optional<String> l, Optional<String> r, Collector<String> c) -> c.collect(l.orElse(null) + r.orElse(null))) .output(); Join join = (Join) flow.operators().iterator().next(); assertEquals(Join.Type.FULL, join.getType()); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return RightJoin.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Optional<Integer> l, Long r, Collector<String> c) -> c.collect(l.orElse(null) + "+" + r)) .output(); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return Join.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Integer l, Long r, Collector<String> c) -> { c.collect(l + "+" + r); }) .output(); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return RightJoin.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Optional<Integer> l, Long r, Collector<String> c) -> c.collect(l.orElse(null) + "+" + r)) .output(); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return LeftJoin.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Integer l, Optional<Long> r, Collector<String> c) -> c.collect(l + "+" + r.orElse(null))) .output(); }
@Override protected Dataset<Pair<Integer, String>> getOutput( Dataset<Integer> left, Dataset<Long> right) { return Join.of(left, right) .by(e -> e, e -> (int) (e % 10), Integer.class) .using((Integer l, Long r, Collector<String> c) -> { c.collect(l + "+" + r); }) .output(); }