public PlanProjectOperator(int[] fields, String name, TypeInformation<T> inType, TypeInformation<R> outType, ExecutionConfig executionConfig) { super(PlanProjectOperator.<T, R, Tuple>createTypedProjector(fields), new UnaryOperatorInformation<T, R>(inType, outType), name); }
public PlanFilterOperator(FilterFunction<T> udf, String name, TypeInformation<T> type) { super(new FlatMapFilter<T>(udf), new UnaryOperatorInformation<T, T>(type, type), name); }
@Override public UnaryOperatorInformation<OUT, OUT> getOperatorInfo() { TypeInformation<OUT> previousOut = input.getOperatorInfo().getOutputType(); return new UnaryOperatorInformation<OUT, OUT>(previousOut, previousOut); }
public PlanUnwrappingGroupCombineOperator(GroupCombineFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name, TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey) { super(new TupleUnwrappingGroupCombiner<IN, OUT, K>(udf), new UnaryOperatorInformation<Tuple2<K, IN>, OUT>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name); }
public PlanUnwrappingReduceOperator(ReduceFunction<T> udf, Keys.SelectorFunctionKeys<T, K> key, String name, TypeInformation<T> type, TypeInformation<Tuple2<K, T>> typeInfoWithKey) { super(new ReduceWrapper<T, K>(udf), new UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>>(typeInfoWithKey, typeInfoWithKey), key.computeLogicalKeyPositions(), name); }
public PlanUnwrappingSortedGroupCombineOperator(GroupCombineFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K1> groupingKey, Keys.SelectorFunctionKeys<IN, K2> sortingKey, String name, TypeInformation<OUT> outType, TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey) { super(new TupleUnwrappingGroupReducer<IN, OUT, K1, K2>(udf), new UnaryOperatorInformation<Tuple3<K1, K2, IN>, OUT>(typeInfoWithKey, outType), groupingKey.computeLogicalKeyPositions(), name); }
private FlatMapOperatorBase<String, String, FlatMapFunction<String, String>> getTestFlatMapOperator( FlatMapFunction<String, String> udf) { UnaryOperatorInformation<String, String> typeInfo = new UnaryOperatorInformation<String, String>( BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); return new FlatMapOperatorBase<String, String, FlatMapFunction<String, String>>( udf, typeInfo, "flatMap on Collections"); } }
public PlanUnwrappingReduceGroupOperator( GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name, TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey, boolean combinable) { super( combinable ? new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf), new UnaryOperatorInformation<>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover( org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey, SelectorFunctionKeys<T, K> key) { TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper = new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>( new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<>(typeInfoWithKey, inputType), "Key Remover" ); mapper.setInput(inputWithKey); mapper.setParallelism(inputWithKey.getParallelism()); return mapper; }
public PlanUnwrappingSortedReduceGroupOperator( GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K1> groupingKey, Keys.SelectorFunctionKeys<IN, K2> sortingKey, String name, TypeInformation<OUT> outType, TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey, boolean combinable) { super( combinable ? new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K1, K2>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K1, K2>(udf), new UnaryOperatorInformation<>(typeInfoWithKey, outType), groupingKey.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
/** * @param criterion */ public <X> void setTerminationCriterion(Operator<X> criterion) { TypeInformation<X> type = criterion.getOperatorInfo().getOutputType(); FlatMapOperatorBase<X, X, TerminationCriterionMapper<X>> mapper = new FlatMapOperatorBase<X, X, TerminationCriterionMapper<X>>( new TerminationCriterionMapper<X>(), new UnaryOperatorInformation<X, X>(type, type), "Termination Criterion Aggregation Wrapper"); mapper.setInput(criterion); this.terminationCriterion = mapper; this.getAggregators().registerAggregationConvergenceCriterion(TERMINATION_CRITERION_AGGREGATOR_NAME, new TerminationCriterionAggregator(), new TerminationCriterionAggregationConvergence()); }
private static MapNode getMapNode() { return new MapNode(new MapOperatorBase<String, String, MapFunction<String,String>>(new IdentityMapper<String>(), new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), "map op")); }
@SuppressWarnings("unchecked") private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner( SelectorFunctionKeys<T, ?> rawKeys, PartitionMethod pMethod, String name, Operator<T> input, int partitionDop, Partitioner<?> customPartitioner, Order[] orders) { final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys; TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput = new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name); keyedPartitionedInput.setInput(keyedInput); keyedPartitionedInput.setCustomPartitioner(customPartitioner); keyedPartitionedInput.setParallelism(partitionDop); keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING)); return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys); }
@Override protected MapPartitionOperatorBase<IN, OUT, MapPartitionFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "MapPartition at " + defaultName; // create operator MapPartitionOperatorBase<IN, OUT, MapPartitionFunction<IN, OUT>> po = new MapPartitionOperatorBase<IN, OUT, MapPartitionFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; } }
@Override protected MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "Map at " + defaultName; // create operator MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> po = new MapOperatorBase<IN, OUT, MapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
@Override protected FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "FlatMap at " + defaultName; // create operator FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; } }
@Test public void testMapPlain() { try { final MapFunction<String, Integer> parser = new MapFunction<String, Integer>() { @Override public Integer map(String value) { return Integer.parseInt(value); } }; MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>( parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), "TestMapper"); List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6")); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); List<Integer> resultMutableSafe = op.executeOnCollections(input, null, executionConfig); executionConfig.enableObjectReuse(); List<Integer> resultRegular = op.executeOnCollections(input, null, executionConfig); assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe); assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
private <T> BulkIterationBase<T> translateBulkIteration(BulkIterationResultSet<?> untypedIterationEnd) { @SuppressWarnings("unchecked") BulkIterationResultSet<T> iterationEnd = (BulkIterationResultSet<T>) untypedIterationEnd; IterativeDataSet<T> iterationHead = iterationEnd.getIterationHead(); BulkIterationBase<T> iterationOperator = new BulkIterationBase<>(new UnaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getType()), "Bulk Iteration"); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } translated.put(iterationHead, iterationOperator.getPartialSolution()); Operator<T> translatedBody = translate(iterationEnd.getNextPartialSolution()); iterationOperator.setNextPartialSolution(translatedBody); iterationOperator.setMaximumNumberOfIterations(iterationHead.getMaxIterations()); iterationOperator.setInput(translate(iterationHead.getInput())); iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); if (iterationEnd.getTerminationCriterion() != null) { iterationOperator.setTerminationCriterion(translate(iterationEnd.getTerminationCriterion())); } return iterationOperator; }
@Test public void testDataSourcePlain() { try { TestNonRichOutputFormat out = new TestNonRichOutputFormat(); GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>( out, new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)), "test_sink"); sink.setInput(source); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig); assertEquals(out.output, asList(TestIOData.NAMES)); executionConfig.enableObjectReuse(); out.clear(); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig); assertEquals(out.output, asList(TestIOData.NAMES)); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }