UnsortedGrouping<Tuple3<Integer, Long, String>> partitionedDS = ds.partitionByHash(0).groupBy(1);
.groupBy(1) .reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long,Long>>()) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
@Test public void testPartitionCustomOperatorPreservesFields() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<>(0L, 0L))); data.partitionCustom(new Partitioner<Long>() { public int partition(Long key, int numPartitions) { return key.intValue(); } }, 1) .groupBy(1) .reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long, Long>>()) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); SinkPlanNode sink = op.getDataSinks().iterator().next(); SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource(); SingleInputPlanNode partitioner = (SingleInputPlanNode) reducer.getInput().getSource(); assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy()); assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
UnsortedGrouping<Tuple3<Integer, Long, String>> partitionedDS = ds.partitionByHash(0).groupBy(1);
@Test public void testRangePartitionByKeyField2() throws Exception { /* * Test range partition by key field */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); AggregateOperator<Tuple3<Integer, Long, String>> sum = ds .map(new PrefixMapper()) .partitionByRange(1, 2) .groupBy(1, 2) .sum(0); List<Tuple3<Integer, Long, String>> result = sum.collect(); String expected = "(1,1,Hi)\n" + "(5,2,Hello)\n" + "(4,3,Hello)\n" + "(5,3,I am )\n" + "(6,3,Luke )\n" + "(34,4,Comme)\n" + "(65,5,Comme)\n" + "(111,6,Comme)"; compareResultAsText(result, expected); }
@Test public void testHashPartitionByKeyField2() throws Exception { /* * Test hash partition by key field */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); AggregateOperator<Tuple3<Integer, Long, String>> sum = ds .map(new PrefixMapper()) .partitionByHash(1, 2) .groupBy(1, 2) .sum(0); List<Tuple3<Integer, Long, String>> result = sum.collect(); String expected = "(1,1,Hi)\n" + "(5,2,Hello)\n" + "(4,3,Hello)\n" + "(5,3,I am )\n" + "(6,3,Luke )\n" + "(34,4,Comme)\n" + "(65,5,Comme)\n" + "(111,6,Comme)"; compareResultAsText(result, expected); }
.groupBy(1) .reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long,Long>>()) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>());