/** * Checks whether a SingleInputOperator is correctly connected. In case that * the contract is incorrectly connected a RuntimeException is thrown. * * @param singleInputContract * SingleInputOperator that is checked. */ private void checkSingleInputContract(SingleInputOperator<?, ?, ?> singleInputContract) { Operator<?> input = singleInputContract.getInput(); // check if input exists if (input == null) { throw new MissingChildException(); } }
/** * Creates a new node with a single input for the optimizer plan. * * @param pactContract The PACT that the node represents. */ protected SingleInputNode(SingleInputOperator<?, ?, ?> pactContract) { super(pactContract); int[] k = pactContract.getKeyColumns(0); this.keys = k == null || k.length == 0 ? null : new FieldSet(k); }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "FlatMap ("+node.getPactContract().getName()+")", in, DriverStrategy.FLAT_MAP); }
@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) throws CompilerException { final Configuration conf = getPactContract().getParameters(); final String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null); final ShipStrategyType preSet; Operator<?> children = ((SingleInputOperator<?, ?, ?>) getPactContract()).getInput(); throw new CompilerException("Error: Node for '" + getPactContract().getName() + "' has no input."); } else { pred = contractToNode.get(children);
private <I, O> eu.stratosphere.api.common.operators.SingleInputOperator<?, O, ?> translateSingleInputOperator(SingleInputOperator<?, ?, ?> op) { @SuppressWarnings("unchecked") SingleInputOperator<I, O, ?> typedOp = (SingleInputOperator<I, O, ?>) op; @SuppressWarnings("unchecked") DataSet<I> typedInput = (DataSet<I>) op.getInput(); Operator<I> input = translate(typedInput); eu.stratosphere.api.common.operators.SingleInputOperator<?, O, ?> dataFlowOp = typedOp.translateToDataFlow(input); if (op instanceof UdfOperator<?> ) { @SuppressWarnings("unchecked") SingleInputUdfOperator<I, O, ?> udfOp = (SingleInputUdfOperator<I, O, ?>) op; // set configuration parameters Configuration opParams = udfOp.getParameters(); if (opParams != null) { dataFlowOp.getParameters().addAll(opParams); } // set the semantic properties dataFlowOp.setSemanticProperties(udfOp.getSematicProperties()); } return dataFlowOp; }
@Override public boolean isFieldConstant(int input, int fieldNumber) { if (input != 0) { throw new IndexOutOfBoundsException(); } SingleInputOperator<?, ?, ?> c = getPactContract(); SingleInputSemanticProperties semanticProperties = c.getSemanticProperties(); if (semanticProperties != null) { FieldSet fs; if ((fs = semanticProperties.getForwardedField(fieldNumber)) != null) { return fs.contains(fieldNumber); } } return false; }
/** * Sets the inputs of the given {@link Operator}.<br> * Currently, the list can have 0, 1, or 2 elements and the number of elements must match the type of the contract. * * @param contract * the Operator whose inputs should be set * @param inputs * all input contracts to this contract */ @SuppressWarnings({ "deprecation", "rawtypes", "unchecked" }) public static void setInputs(final Operator<?> contract, final List<List<Operator>> inputs) { if (contract instanceof GenericDataSinkBase) { if (inputs.size() != 1) { throw new IllegalArgumentException("wrong number of inputs"); } ((GenericDataSinkBase) contract).setInputs(inputs.get(0)); } else if (contract instanceof SingleInputOperator) { if (inputs.size() != 1) { throw new IllegalArgumentException("wrong number of inputs"); } ((SingleInputOperator) contract).setInputs(inputs.get(0)); } else if (contract instanceof DualInputOperator) { if (inputs.size() != 2) { throw new IllegalArgumentException("wrong number of inputs"); } ((DualInputOperator) contract).setFirstInputs(inputs.get(0)); ((DualInputOperator) contract).setSecondInputs(inputs.get(1)); } } }
sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(), getSortOrders(sn.getKeys(), sn.getSortOrders())));
@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) throws CompilerException { final Configuration conf = getPactContract().getParameters(); final String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null); final ShipStrategyType preSet; Operator<?> children = ((SingleInputOperator<?, ?, ?>) getPactContract()).getInput(); throw new CompilerException("Error: Node for '" + getPactContract().getName() + "' has no input."); } else { pred = contractToNode.get(children);
@Override public boolean isFieldConstant(int input, int fieldNumber) { if (input != 0) { throw new IndexOutOfBoundsException(); } SingleInputOperator<?, ?, ?> c = getPactContract(); SingleInputSemanticProperties semanticProperties = c.getSemanticProperties(); if (semanticProperties != null) { FieldSet fs; if ((fs = semanticProperties.getForwardedField(fieldNumber)) != null) { return fs.contains(fieldNumber); } } return false; }
/** * Sets the inputs of the given {@link Operator}.<br> * Currently, the list can have 0, 1, or 2 elements and the number of elements must match the type of the contract. * * @param contract * the Operator whose inputs should be set * @param inputs * all input contracts to this contract */ @SuppressWarnings({ "deprecation", "rawtypes", "unchecked" }) public static void setInputs(final Operator<?> contract, final List<List<Operator>> inputs) { if (contract instanceof GenericDataSinkBase) { if (inputs.size() != 1) { throw new IllegalArgumentException("wrong number of inputs"); } ((GenericDataSinkBase) contract).setInputs(inputs.get(0)); } else if (contract instanceof SingleInputOperator) { if (inputs.size() != 1) { throw new IllegalArgumentException("wrong number of inputs"); } ((SingleInputOperator) contract).setInputs(inputs.get(0)); } else if (contract instanceof DualInputOperator) { if (inputs.size() != 2) { throw new IllegalArgumentException("wrong number of inputs"); } ((DualInputOperator) contract).setFirstInputs(inputs.get(0)); ((DualInputOperator) contract).setSecondInputs(inputs.get(1)); } } }
sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(), getSortOrders(sn.getKeys(), sn.getSortOrders())));
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE); }
/** * Creates a new node with a single input for the optimizer plan. * * @param pactContract The PACT that the node represents. */ protected SingleInputNode(SingleInputOperator<?, ?, ?> pactContract) { super(pactContract); int[] k = pactContract.getKeyColumns(0); this.keys = k == null || k.length == 0 ? null : new FieldSet(k); }
/** * Checks whether a SingleInputOperator is correctly connected. In case that * the contract is incorrectly connected a RuntimeException is thrown. * * @param singleInputContract * SingleInputOperator that is checked. */ private void checkSingleInputContract(SingleInputOperator<?, ?, ?> singleInputContract) { Operator<?> input = singleInputContract.getInput(); // check if input exists if (input == null) { throw new MissingChildException(); } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "Map ("+node.getPactContract().getName()+")", in, DriverStrategy.MAP); }
@Override protected void getSingleInputNodeSchema(SingleInputPlanNode node, SparseKeySchema schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException { // check that we got the right types SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getPactContract(); if (! (contract instanceof RecordOperator)) { throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation."); } RecordOperator recContract = (RecordOperator) contract; // add the information to the schema int[] localPositions = contract.getKeyColumns(0); Class<? extends Key<?>>[] types = recContract.getKeyClasses(); for (int i = 0; i < localPositions.length; i++) { schema.addType(localPositions[i], types[i]); } // this is a temporary fix, we should solve this more generic if (contract instanceof GroupReduceOperatorBase) { Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder(); if (groupOrder != null) { addOrderingToSchema(groupOrder, schema); } } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "Filter ("+node.getPactContract().getName()+")", in, DriverStrategy.FLAT_MAP); }
@Override protected void getSingleInputNodeSchema(SingleInputPlanNode node, SparseKeySchema schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException { // check that we got the right types SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getPactContract(); if (! (contract instanceof RecordOperator)) { throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation."); } RecordOperator recContract = (RecordOperator) contract; // add the information to the schema int[] localPositions = contract.getKeyColumns(0); Class<? extends Key<?>>[] types = recContract.getKeyClasses(); for (int i = 0; i < localPositions.length; i++) { schema.addType(localPositions[i], types[i]); } // this is a temporary fix, we should solve this more generic if (contract instanceof GroupReduceOperatorBase) { Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder(); if (groupOrder != null) { addOrderingToSchema(groupOrder, schema); } } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList); }