.projectSecond(); } catch (Exception e) { System.out.println("FAILED: " + e);
.projectSecond(); } catch (Exception e) { System.out.println("FAILED: " + e);
.where(0).equalTo(0).projectFirst(1).projectSecond(0);
.projectFirst(0).projectSecond(0);
.equalTo(0) .projectFirst(0, 1) .<Tuple3<K, K, FloatValue>>projectSecond(2) .setParallelism(parallelism) .name("Edge score");
.projectSecond(1, 3) .output(new DiscardingOutputFormat<Tuple>());
.where(new DummyTestKeySelector()).equalTo(new DummyTestKeySelector()) .projectFirst(2) .projectSecond(0, 0, 3) .projectFirst(0, 4) .projectSecond(2);
.projectFirst(1).projectSecond(1); .projectFirst(1).projectSecond(1);
.where(1).equalTo(0) .projectFirst(0, 1) .<Tuple3<Long, Long, Long>>projectSecond(2) .withPartitioner(partitioner);
invariantInput .map(new IdentityMapper<Tuple2<Long, Long>>()).withBroadcastSet(iter.getWorkset(), "bc data") .join(iter.getSolutionSet()).where(0).equalTo(1).projectFirst(1).projectSecond(1);
.projectFirst(0).projectSecond(0);
@Test public void testProjectOnATuple1Input() throws Exception { /* * Project join on a tuple input 1 */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env); DataSet<Tuple6<String, Long, String, Integer, Long, Long>> joinDs = ds1.join(ds2) .where(1) .equalTo(1) .projectFirst(2, 1) .projectSecond(3) .projectFirst(0) .projectSecond(4, 1); List<Tuple6<String, Long, String, Integer, Long, Long>> result = joinDs.collect(); String expected = "Hi,1,Hallo,1,1,1\n" + "Hello,2,Hallo Welt,2,2,2\n" + "Hello world,2,Hallo Welt,3,2,2\n"; compareResultAsTuples(result, expected); }
@Test public void testJoinProjectionSemProps1() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo); tupleDs.join(tupleDs).where(0).equalTo(0) .projectFirst(2, 3) .projectSecond(1, 4) .output(new DiscardingOutputFormat<Tuple>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> projectJoinOperator = ((InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput()); DualInputSemanticProperties props = projectJoinOperator.getSemanticProperties(); assertEquals(1, props.getForwardingTargetFields(0, 2).size()); assertEquals(1, props.getForwardingTargetFields(0, 3).size()); assertEquals(1, props.getForwardingTargetFields(1, 1).size()); assertEquals(1, props.getForwardingTargetFields(1, 4).size()); assertTrue(props.getForwardingTargetFields(0, 2).contains(0)); assertTrue(props.getForwardingTargetFields(0, 3).contains(1)); assertTrue(props.getForwardingTargetFields(1, 1).contains(2)); assertTrue(props.getForwardingTargetFields(1, 4).contains(3)); }
@Test(expected = InvalidProgramException.class) public void testRangePartitionInIteration() throws Exception { // does not apply for collection execution if (super.mode == TestExecutionMode.COLLECTION) { throw new InvalidProgramException("Does not apply for collection execution"); } final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSource<Long> source = env.generateSequence(0, 10000); DataSet<Tuple2<Long, String>> tuples = source.map(new MapFunction<Long, Tuple2<Long, String>>() { @Override public Tuple2<Long, String> map(Long v) throws Exception { return new Tuple2<>(v, Long.toString(v)); } }); DeltaIteration<Tuple2<Long, String>, Tuple2<Long, String>> it = tuples.iterateDelta(tuples, 10, 0); DataSet<Tuple2<Long, String>> body = it.getWorkset() .partitionByRange(1) // Verify that range partition is not allowed in iteration .join(it.getSolutionSet()) .where(0).equalTo(0).projectFirst(0).projectSecond(1); DataSet<Tuple2<Long, String>> result = it.closeWith(body, body); result.collect(); // should fail }
@Test public void testProjectJoinOnATuple2Input() throws Exception { /* * Project join on a tuple input 2 */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env); DataSet<Tuple6<String, String, Long, Long, Long, Integer>> joinDs = ds1.join(ds2) .where(1) .equalTo(1) .projectSecond(3) .projectFirst(2, 1) .projectSecond(4, 1) .projectFirst(0); List<Tuple6<String, String, Long, Long, Long, Integer>> result = joinDs.collect(); String expected = "Hallo,Hi,1,1,1,1\n" + "Hallo Welt,Hello,2,2,2,2\n" + "Hallo Welt,Hello world,2,2,2,3\n"; compareResultAsTuples(result, expected); }
private DataSet<Tuple5<KT, KB, EV, VVT, VVB>> joinEdgeWithVertices() { return edges .join(topVertices, JoinHint.REPARTITION_HASH_SECOND) .where(0) .equalTo(0) .projectFirst(0, 1, 2) .<Tuple4<KT, KB, EV, VVT>>projectSecond(1) .name("Edge with vertex") .join(bottomVertices, JoinHint.REPARTITION_HASH_SECOND) .where(1) .equalTo(0) .projectFirst(0, 1, 2, 3) .<Tuple5<KT, KB, EV, VVT, VVB>>projectSecond(1) .name("Edge with vertices"); }
@Test public void testJoinProjection4() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { ds1.join(ds2).where(0).equalTo(0) .projectFirst(0, 2) .projectSecond(1, 4) .projectFirst(1); } catch (Exception e) { Assert.fail(); } }
@Test public void testJoinProjection3() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { ds1.join(ds2).where(0).equalTo(0) .projectFirst(0) .projectSecond(3); } catch (Exception e) { Assert.fail(); } }
@Test(expected = IndexOutOfBoundsException.class) public void testJoinProjection14() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should not work, index out of range ds1.join(ds2).where(0).equalTo(0) .projectFirst(0) .projectSecond(5); }
@Test(expected = IndexOutOfBoundsException.class) public void testJoinProjection34() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should not work, index out of range ds1.join(ds2).where(0).equalTo(0) .projectFirst(0) .projectSecond(-1); }