private String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) { try { if (wrapper.hasObject()) { try { return wrapper.getUserCodeObject().toString(); } catch (Throwable t) { return wrapper.getUserCodeClass().getName(); } } else { return wrapper.getUserCodeClass().getName(); } } catch (Throwable t) { return null; } }
@Override protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, ctx); FunctionUtils.openFunction(function, this.parameters); ArrayList<T> result = new ArrayList<T>(inputData.size()); ListCollector<T> collector = new ListCollector<T>(result); for (T element : inputData) { function.flatMap(element, collector); } FunctionUtils.closeFunction(function); return result; } }
@Override protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { MapPartitionFunction<IN, OUT> function = this.userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, ctx); FunctionUtils.openFunction(function, this.parameters); ArrayList<OUT> result = new ArrayList<OUT>(inputData.size() / 4); TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig); TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig); CopyingIterator<IN> source = new CopyingIterator<IN>(inputData.iterator(), inSerializer); CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer); function.mapPartition(source, resultCollector); result.trimToSize(); FunctionUtils.closeFunction(function); return result; } }
format = getOperator().getFormatWrapper().getUserCodeObject(); Configuration config = getOperator().getParameters(); format.configure(config);
@Override protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { MapFunction<IN, OUT> function = this.userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, ctx); FunctionUtils.openFunction(function, this.parameters); ArrayList<OUT> result = new ArrayList<OUT>(inputData.size()); TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig); TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig); for (IN element : inputData) { IN inCopy = inSerializer.copy(element); OUT out = function.map(inCopy); result.add(outSerializer.copy(out)); } FunctionUtils.closeFunction(function); return result; } }
ReduceFunction<T> function = this.userFunction.getUserCodeObject();
@Override protected List<OUT> executeOnCollections(List<IN> input, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { FlatMapFunction<IN, OUT> function = userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, ctx); FunctionUtils.openFunction(function, parameters); ArrayList<OUT> result = new ArrayList<OUT>(input.size()); TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig); TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig); CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer); for (IN element : input) { IN inCopy = inSerializer.copy(element); function.flatMap(inCopy, resultCollector); } FunctionUtils.closeFunction(function); return result; } }
@Override protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { CrossFunction<IN1, IN2, OUT> function = this.userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, ctx); FunctionUtils.openFunction(function, this.parameters); ArrayList<OUT> result = new ArrayList<OUT>(inputData1.size() * inputData2.size()); TypeSerializer<IN1> inSerializer1 = getOperatorInfo().getFirstInputType().createSerializer(executionConfig); TypeSerializer<IN2> inSerializer2 = getOperatorInfo().getSecondInputType().createSerializer(executionConfig); TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig); for (IN1 element1 : inputData1) { for (IN2 element2 : inputData2) { IN1 copy1 = inSerializer1.copy(element1); IN2 copy2 = inSerializer2.copy(element2); OUT o = function.cross(copy1, copy2); result.add(outSerializer.copy(o)); } } FunctionUtils.closeFunction(function); return result; }
protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception { FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();
((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat(); if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) && this.estimatedOutputSize >= 0) {
CoGroupFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();
@Override protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { GroupReduceFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
protected List<OUT> executeOnCollections(RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { @SuppressWarnings("unchecked") InputFormat<OUT, InputSplit> inputFormat = (InputFormat<OUT, InputSplit>) this.formatWrapper.getUserCodeObject();
@Override protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { GroupCombineFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
@SuppressWarnings("unchecked") protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { OutputFormat<IN> format = this.formatWrapper.getUserCodeObject(); TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();
CoGroupFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();
.getFormatWrapper().getUserCodeObject()).getOutputFilePath().toString(); Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
@Override protected List<OUT> executeOnCollections(List<IN1> leftInput, List<IN2> rightInput, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception { TypeInformation<IN1> leftInformation = getOperatorInfo().getFirstInputType(); TypeInformation<IN2> rightInformation = getOperatorInfo().getSecondInputType(); TypeInformation<OUT> outInformation = getOperatorInfo().getOutputType(); TypeComparator<IN1> leftComparator = buildComparatorFor(0, executionConfig, leftInformation); TypeComparator<IN2> rightComparator = buildComparatorFor(1, executionConfig, rightInformation); TypeSerializer<IN1> leftSerializer = leftInformation.createSerializer(executionConfig); TypeSerializer<IN2> rightSerializer = rightInformation.createSerializer(executionConfig); OuterJoinListIterator<IN1, IN2> outerJoinIterator = new OuterJoinListIterator<>(leftInput, leftSerializer, leftComparator, rightInput, rightSerializer, rightComparator, outerJoinType); // -------------------------------------------------------------------- // Run UDF // -------------------------------------------------------------------- FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject(); FunctionUtils.setFunctionRuntimeContext(function, runtimeContext); FunctionUtils.openFunction(function, this.parameters); List<OUT> result = new ArrayList<>(); Collector<OUT> collector = new CopyingListCollector<>(result, outInformation.createSerializer(executionConfig)); while (outerJoinIterator.next()) { IN1 left = outerJoinIterator.getLeft(); IN2 right = outerJoinIterator.getRight(); function.join(left == null ? null : leftSerializer.copy(left), right == null ? null : rightSerializer.copy(right), collector); } FunctionUtils.closeFunction(function); return result; }
if (solutionSetJoin.getUserCodeWrapper().getUserCodeObject() instanceof WrappingFunction) { WrappingFunction<?> wf = (WrappingFunction<?>) solutionSetJoin.getUserCodeWrapper().getUserCodeObject(); assertEquals(SolutionWorksetJoin.class, wf.getWrappedFunction().getClass());
@Override @SuppressWarnings("unchecked") public <F> Iterator<Pair<OperatorID, F>> getFormat(FormatType<F> type) { if (type != FormatType.INPUT) { return EmptyIterator.get(); } SingleElementIterator<Pair<OperatorID, F>> iterator = new SingleElementIterator<>(); iterator.set(new DefaultPair<>(stubKey, (F) wrapper.getUserCodeObject(InputFormat.class, classLoader))); return iterator; }