/** * Traverses the job depth first from all data sinks on towards the sources. * * @see Visitable#accept(Visitor) */ @Override public void accept(Visitor<Operator<?>> visitor) { for (GenericDataSinkBase<?> sink : this.sinks) { sink.accept(visitor); } }
/** * Adds to the input the union of the given operators. * * @param inputs The operator(s) to be unioned with the input. * @deprecated This method will be removed in future versions. Use the {@link eu.stratosphere.api.common.operators.Union} operator instead. */ @SuppressWarnings("unchecked") @Deprecated public void addInputs(List<? extends Operator<IN>> inputs) { Preconditions.checkNotNull(inputs, "The inputs may not be null."); this.input = createUnionCascade(this.input, (Operator<IN>[]) inputs.toArray(new Operator[inputs.size()])); }
/** * Checks if DataSinkContract is correctly connected. In case that the * contract is incorrectly connected a RuntimeException is thrown. * * @param dataSinkContract * DataSinkContract that is checked. */ private void checkDataSink(GenericDataSinkBase<?> dataSinkContract) { Operator<?> input = dataSinkContract.getInput(); // check if input exists if (input == null) { throw new MissingChildException(); } }
@Override public boolean isMemoryConsumer() { return getPactContract().getPartitionOrdering() != null || getPactContract().getLocalOrder() != null; }
final Ordering partitioning = getPactContract().getPartitionOrdering(); final DataDistribution dataDist = getPactContract().getDataDistribution(); final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties(); if (partitioning != null) { final Ordering localOrder = getPactContract().getLocalOrder(); final RequestedLocalProperties orderProps = new RequestedLocalProperties(); if (localOrder != null) {
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) { // select the name (or create a default one) String name = this.name != null ? this.name : this.format.toString(); GenericDataSinkBase<T> sink = new GenericDataSinkBase<T>(this.format, new UnaryOperatorInformation<T, Nothing>(this.type, new NothingTypeInfo()), name); // set input sink.setInput(input); // set dop if(this.dop > 0) { // use specified dop sink.setDegreeOfParallelism(this.dop); } else { // if no dop has been specified, use dop of input operator to enable chaining sink.setDegreeOfParallelism(input.getDegreeOfParallelism()); } return sink; }
outputPlans.add(new SinkPlanNode(this, "DataSink("+this.getPactContract().getName()+")" ,c));
final Ordering partitioning = getPactContract().getPartitionOrdering(); final DataDistribution dataDist = getPactContract().getDataDistribution(); final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties(); if (partitioning != null) { final Ordering localOrder = getPactContract().getLocalOrder(); final RequestedLocalProperties orderProps = new RequestedLocalProperties(); if (localOrder != null) {
@Override public boolean isMemoryConsumer() { return getPactContract().getPartitionOrdering() != null || getPactContract().getLocalOrder() != null; }
outputPlans.add(new SinkPlanNode(this, "DataSink("+this.getPactContract().getName()+")" ,c));
@Override protected void getSinkSchema(SinkPlanNode sinkPlanNode, SparseKeySchema schema) throws CompilerPostPassException { GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getPactContract(); Ordering partitioning = sink.getPartitionOrdering(); Ordering sorting = sink.getLocalOrder(); try { if (partitioning != null) { addOrderingToSchema(partitioning, schema); } if (sorting != null) { addOrderingToSchema(sorting, schema); } } catch (ConflictingFieldTypeInfoException ex) { throw new CompilerPostPassException("Conflicting information found when adding data sink types. " + "Probable reason is contradicting type infos for partitioning and sorting ordering."); } }
/** * Checks if DataSinkContract is correctly connected. In case that the * contract is incorrectly connected a RuntimeException is thrown. * * @param dataSinkContract * DataSinkContract that is checked. */ private void checkDataSink(GenericDataSinkBase<?> dataSinkContract) { Operator<?> input = dataSinkContract.getInput(); // check if input exists if (input == null) { throw new MissingChildException(); } }
/** * Adds to the input the union of the given operators. * * @param inputs The operator(s) to be unioned with the input. * @deprecated This method will be removed in future versions. Use the {@link eu.stratosphere.api.common.operators.Union} operator instead. */ @SuppressWarnings("unchecked") @Deprecated public void addInputs(List<? extends Operator<IN>> inputs) { Preconditions.checkNotNull(inputs, "The inputs may not be null."); this.input = createUnionCascade(this.input, (Operator<IN>[]) inputs.toArray(new Operator[inputs.size()])); }
/** * Traverses the job depth first from all data sinks on towards the sources. * * @see Visitable#accept(Visitor) */ @Override public void accept(Visitor<Operator<?>> visitor) { for (GenericDataSinkBase<?> sink : this.sinks) { sink.accept(visitor); } }
@Override protected void getSinkSchema(SinkPlanNode sinkPlanNode, SparseKeySchema schema) throws CompilerPostPassException { GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getPactContract(); Ordering partitioning = sink.getPartitionOrdering(); Ordering sorting = sink.getLocalOrder(); try { if (partitioning != null) { addOrderingToSchema(partitioning, schema); } if (sorting != null) { addOrderingToSchema(sorting, schema); } } catch (ConflictingFieldTypeInfoException ex) { throw new CompilerPostPassException("Conflicting information found when adding data sink types. " + "Probable reason is contradicting type infos for partitioning and sorting ordering."); } }
@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) { Operator<?> children = getPactContract().getInput(); final OptimizerNode pred; final PactConnection conn; pred = contractToNode.get(children); conn = new PactConnection(pred, this); // create the connection and add it this.input = conn; pred.addOutgoingConnection(conn); }
@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) { Operator<?> children = getPactContract().getInput(); final OptimizerNode pred; final PactConnection conn; pred = contractToNode.get(children); conn = new PactConnection(pred, this); // create the connection and add it this.input = conn; pred.addOutgoingConnection(conn); }