@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) { Operator<?> children = getOperator().getInput(); final OptimizerNode pred; final DagConnection conn; pred = contractToNode.get(children); conn = new DagConnection(pred, this, defaultExchangeMode); // create the connection and add it this.input = conn; pred.addOutgoingConnection(conn); }
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception { Operator<?> inputOp = sink.getInput(); if (inputOp == null) { throw new InvalidProgramException("The data sink " + sink.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> input = (List<IN>) execute(inputOp); @SuppressWarnings("unchecked") GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); } else { ctx = null; } typedSink.executeOnCollections(input, ctx, executionConfig); }
@Override public void computeInterestingPropertiesForInputs(CostEstimator estimator) { final InterestingProperties iProps = new InterestingProperties(); { final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties(); iProps.addGlobalProperties(partitioningProps); } { final Ordering localOrder = getOperator().getLocalOrder(); final RequestedLocalProperties orderProps = new RequestedLocalProperties(); if (localOrder != null) { orderProps.setOrdering(localOrder); } iProps.addLocalProperties(orderProps); } this.input.setInterestingProperties(iProps); }
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) { // select the name (or create a default one) String name = this.name != null ? this.name : this.format.toString(); GenericDataSinkBase<T> sink = new GenericDataSinkBase<>(this.format, new UnaryOperatorInformation<>(this.type, new NothingTypeInfo()), name); // set input sink.setInput(input); // set parameters if (this.parameters != null) { sink.getParameters().addAll(this.parameters); } // set parallelism if (this.parallelism > 0) { // use specified parallelism sink.setParallelism(this.parallelism); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining sink.setParallelism(input.getParallelism()); } if (this.sortKeyPositions != null) { // configure output sorting Ordering ordering = new Ordering(); for (int i = 0; i < this.sortKeyPositions.length; i++) { ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]); } sink.setLocalOrder(ordering); } return sink; }
@Test public void testDataSourcePlain() { try { TestNonRichOutputFormat out = new TestNonRichOutputFormat(); GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>( out, new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)), "test_sink"); sink.setInput(source); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig); assertEquals(out.output, asList(TestIOData.NAMES)); executionConfig.enableObjectReuse(); out.clear(); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig); assertEquals(out.output, asList(TestIOData.NAMES)); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
/** * Traverses the job depth first from all data sinks on towards the sources. * * @see Visitable#accept(Visitor) */ @Override public void accept(Visitor<Operator<?>> visitor) { for (GenericDataSinkBase<?> sink : this.sinks) { sink.accept(visitor); } }
/** * Adds to the input the union of the given operators. * * @param inputs The operator(s) to be unioned with the input. * @deprecated This method will be removed in future versions. Use the {@link org.apache.flink.api.common.operators.Union} operator instead. */ @SuppressWarnings("unchecked") @Deprecated public void addInputs(List<? extends Operator<IN>> inputs) { checkNotNull(inputs, "The inputs may not be null."); this.input = createUnionCascade(this.input, (Operator<IN>[]) inputs.toArray(new Operator[inputs.size()])); }
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) { // select the name (or create a default one) String name = this.name != null ? this.name : this.format.toString(); GenericDataSinkBase<T> sink = new GenericDataSinkBase<>(this.format, new UnaryOperatorInformation<>(this.type, new NothingTypeInfo()), name); // set input sink.setInput(input); // set parameters if (this.parameters != null) { sink.getParameters().addAll(this.parameters); } // set parallelism if (this.parallelism > 0) { // use specified parallelism sink.setParallelism(this.parallelism); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining sink.setParallelism(input.getParallelism()); } if (this.sortKeyPositions != null) { // configure output sorting Ordering ordering = new Ordering(); for (int i = 0; i < this.sortKeyPositions.length; i++) { ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]); } sink.setLocalOrder(ordering); } return sink; }
outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
try { TestRichOutputFormat out = new TestRichOutputFormat(); GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>( out, new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)), "test_sink"); sink.setInput(source); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext( taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext( taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()), executionConfig);
/** * Traverses the job depth first from all data sinks on towards the sources. * * @see Visitable#accept(Visitor) */ @Override public void accept(Visitor<Operator<?>> visitor) { for (GenericDataSinkBase<?> sink : this.sinks) { sink.accept(visitor); } }
/** * Adds to the input the union of the given operators. * * @param inputs The operator(s) to be unioned with the input. * @deprecated This method will be removed in future versions. Use the {@link org.apache.flink.api.common.operators.Union} operator instead. */ @SuppressWarnings("unchecked") @Deprecated public void addInputs(List<? extends Operator<IN>> inputs) { checkNotNull(inputs, "The inputs may not be null."); this.input = createUnionCascade(this.input, (Operator<IN>[]) inputs.toArray(new Operator[inputs.size()])); }
@SuppressWarnings("unchecked") protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { OutputFormat<IN> format = this.formatWrapper.getUserCodeObject(); TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception { Operator<?> inputOp = sink.getInput(); if (inputOp == null) { throw new InvalidProgramException("The data sink " + sink.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> input = (List<IN>) execute(inputOp); @SuppressWarnings("unchecked") GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); } else { ctx = null; } typedSink.executeOnCollections(input, ctx, executionConfig); }
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) { // select the name (or create a default one) String name = this.name != null ? this.name : this.format.toString(); GenericDataSinkBase<T> sink = new GenericDataSinkBase<>(this.format, new UnaryOperatorInformation<>(this.type, new NothingTypeInfo()), name); // set input sink.setInput(input); // set parameters if (this.parameters != null) { sink.getParameters().addAll(this.parameters); } // set parallelism if (this.parallelism > 0) { // use specified parallelism sink.setParallelism(this.parallelism); } else { // if no parallelism has been specified, use parallelism of input operator to enable chaining sink.setParallelism(input.getParallelism()); } if (this.sortKeyPositions != null) { // configure output sorting Ordering ordering = new Ordering(); for (int i = 0; i < this.sortKeyPositions.length; i++) { ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]); } sink.setLocalOrder(ordering); } return sink; }
outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
@Override public void computeInterestingPropertiesForInputs(CostEstimator estimator) { final InterestingProperties iProps = new InterestingProperties(); { final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties(); iProps.addGlobalProperties(partitioningProps); } { final Ordering localOrder = getOperator().getLocalOrder(); final RequestedLocalProperties orderProps = new RequestedLocalProperties(); if (localOrder != null) { orderProps.setOrdering(localOrder); } iProps.addLocalProperties(orderProps); } this.input.setInterestingProperties(iProps); }
/** * Traverses the job depth first from all data sinks on towards the sources. * * @see Visitable#accept(Visitor) */ @Override public void accept(Visitor<Operator<?>> visitor) { for (GenericDataSinkBase<?> sink : this.sinks) { sink.accept(visitor); } }