public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover( org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey, SelectorFunctionKeys<T, K> key) { TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper = new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>( new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<>(typeInfoWithKey, inputType), "Key Remover" ); mapper.setInput(inputWithKey); mapper.setParallelism(inputWithKey.getParallelism()); return mapper; }
@Override protected MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "Map at " + defaultName; // create operator MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> po = new MapOperatorBase<IN, OUT, MapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
@SuppressWarnings("unchecked") public static <T, K> org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> appendKeyExtractor( org.apache.flink.api.common.operators.Operator<T> input, SelectorFunctionKeys<T, K> key) { if (input instanceof Union) { // if input is a union, we apply the key extractors recursively to all inputs org.apache.flink.api.common.operators.Operator<T> firstInput = ((Union) input).getFirstInput(); org.apache.flink.api.common.operators.Operator<T> secondInput = ((Union) input).getSecondInput(); org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> firstInputWithKey = appendKeyExtractor(firstInput, key); org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> secondInputWithKey = appendKeyExtractor(secondInput, key); return new Union(firstInputWithKey, secondInputWithKey, input.getName()); } TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); KeyExtractingMapper<T, K> extractor = new KeyExtractingMapper(key.getKeyExtractor()); MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>> mapper = new MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>>( extractor, new UnaryOperatorInformation(inputType, typeInfoWithKey), "Key Extractor" ); mapper.setInput(input); mapper.setParallelism(input.getParallelism()); return mapper; }
@SuppressWarnings("unchecked") public static <T, K1, K2> org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> appendKeyExtractor( org.apache.flink.api.common.operators.Operator<T> input, SelectorFunctionKeys<T, K1> key1, SelectorFunctionKeys<T, K2> key2) { if (input instanceof Union) { // if input is a union, we apply the key extractors recursively to all inputs org.apache.flink.api.common.operators.Operator<T> firstInput = ((Union) input).getFirstInput(); org.apache.flink.api.common.operators.Operator<T> secondInput = ((Union) input).getSecondInput(); org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> firstInputWithKey = appendKeyExtractor(firstInput, key1, key2); org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> secondInputWithKey = appendKeyExtractor(secondInput, key1, key2); return new Union(firstInputWithKey, secondInputWithKey, input.getName()); } TypeInformation<T> inputType = key1.getInputType(); TypeInformation<Tuple3<K1, K2, T>> typeInfoWithKey = createTypeWithKey(key1, key2); TwoKeyExtractingMapper<T, K1, K2> extractor = new TwoKeyExtractingMapper<>(key1.getKeyExtractor(), key2.getKeyExtractor()); MapOperatorBase<T, Tuple3<K1, K2, T>, MapFunction<T, Tuple3<K1, K2, T>>> mapper = new MapOperatorBase<T, Tuple3<K1, K2, T>, MapFunction<T, Tuple3<K1, K2, T>>>( extractor, new UnaryOperatorInformation<>(inputType, typeInfoWithKey), "Key Extractor" ); mapper.setInput(input); mapper.setParallelism(input.getParallelism()); return mapper; }
public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover( org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey, SelectorFunctionKeys<T, K> key) { TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper = new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>( new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<>(typeInfoWithKey, inputType), "Key Remover" ); mapper.setInput(inputWithKey); mapper.setParallelism(inputWithKey.getParallelism()); return mapper; }
public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover( org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey, SelectorFunctionKeys<T, K> key) { TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper = new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>( new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<>(typeInfoWithKey, inputType), "Key Remover" ); mapper.setInput(inputWithKey); mapper.setParallelism(inputWithKey.getParallelism()); return mapper; }
@SuppressWarnings("unchecked") public static <T, K> org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> appendKeyExtractor( org.apache.flink.api.common.operators.Operator<T> input, SelectorFunctionKeys<T, K> key) { TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); KeyExtractingMapper<T, K> extractor = new KeyExtractingMapper(key.getKeyExtractor()); MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>> mapper = new MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>>( extractor, new UnaryOperatorInformation(inputType, typeInfoWithKey), "Key Extractor" ); mapper.setInput(input); mapper.setParallelism(input.getParallelism()); return mapper; }
@SuppressWarnings("unchecked") public static <T, K1, K2> org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> appendKeyExtractor( org.apache.flink.api.common.operators.Operator<T> input, SelectorFunctionKeys<T, K1> key1, SelectorFunctionKeys<T, K2> key2) { TypeInformation<T> inputType = key1.getInputType(); TypeInformation<Tuple3<K1, K2, T>> typeInfoWithKey = createTypeWithKey(key1, key2); TwoKeyExtractingMapper<T, K1, K2> extractor = new TwoKeyExtractingMapper<>(key1.getKeyExtractor(), key2.getKeyExtractor()); MapOperatorBase<T, Tuple3<K1, K2, T>, MapFunction<T, Tuple3<K1, K2, T>>> mapper = new MapOperatorBase<T, Tuple3<K1, K2, T>, MapFunction<T, Tuple3<K1, K2, T>>>( extractor, new UnaryOperatorInformation<>(inputType, typeInfoWithKey), "Key Extractor" ); mapper.setInput(input); mapper.setParallelism(input.getParallelism()); return mapper; }
@Override protected MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "Map at " + defaultName; // create operator MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> po = new MapOperatorBase<IN, OUT, MapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
@Override protected MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) { String name = getName() != null ? getName() : "Map at " + defaultName; // create operator MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> po = new MapOperatorBase<IN, OUT, MapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name); // set input po.setInput(input); // set parallelism if (this.getParallelism() > 0) { // use specified parallelism po.setParallelism(this.getParallelism()); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining po.setParallelism(input.getParallelism()); } return po; }
@SuppressWarnings("unchecked") public static <T, K> org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> appendKeyExtractor( org.apache.flink.api.common.operators.Operator<T> input, SelectorFunctionKeys<T, K> key) { if (input instanceof Union) { // if input is a union, we apply the key extractors recursively to all inputs org.apache.flink.api.common.operators.Operator<T> firstInput = ((Union) input).getFirstInput(); org.apache.flink.api.common.operators.Operator<T> secondInput = ((Union) input).getSecondInput(); org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> firstInputWithKey = appendKeyExtractor(firstInput, key); org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> secondInputWithKey = appendKeyExtractor(secondInput, key); return new Union(firstInputWithKey, secondInputWithKey, input.getName()); } TypeInformation<T> inputType = key.getInputType(); TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key); KeyExtractingMapper<T, K> extractor = new KeyExtractingMapper(key.getKeyExtractor()); MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>> mapper = new MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>>( extractor, new UnaryOperatorInformation(inputType, typeInfoWithKey), "Key Extractor" ); mapper.setInput(input); mapper.setParallelism(input.getParallelism()); return mapper; }
@SuppressWarnings("unchecked") public static <T, K1, K2> org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> appendKeyExtractor( org.apache.flink.api.common.operators.Operator<T> input, SelectorFunctionKeys<T, K1> key1, SelectorFunctionKeys<T, K2> key2) { if (input instanceof Union) { // if input is a union, we apply the key extractors recursively to all inputs org.apache.flink.api.common.operators.Operator<T> firstInput = ((Union) input).getFirstInput(); org.apache.flink.api.common.operators.Operator<T> secondInput = ((Union) input).getSecondInput(); org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> firstInputWithKey = appendKeyExtractor(firstInput, key1, key2); org.apache.flink.api.common.operators.Operator<Tuple3<K1, K2, T>> secondInputWithKey = appendKeyExtractor(secondInput, key1, key2); return new Union(firstInputWithKey, secondInputWithKey, input.getName()); } TypeInformation<T> inputType = key1.getInputType(); TypeInformation<Tuple3<K1, K2, T>> typeInfoWithKey = createTypeWithKey(key1, key2); TwoKeyExtractingMapper<T, K1, K2> extractor = new TwoKeyExtractingMapper<>(key1.getKeyExtractor(), key2.getKeyExtractor()); MapOperatorBase<T, Tuple3<K1, K2, T>, MapFunction<T, Tuple3<K1, K2, T>>> mapper = new MapOperatorBase<T, Tuple3<K1, K2, T>, MapFunction<T, Tuple3<K1, K2, T>>>( extractor, new UnaryOperatorInformation<>(inputType, typeInfoWithKey), "Key Extractor" ); mapper.setInput(input); mapper.setParallelism(input.getParallelism()); return mapper; }