@Override protected HadoopCoGroupClosure createClosure() { return new HadoopCoGroupClosure( flowProcess, splice.getNumSelfJoins(), keyFields, valuesFields ); }
@Override protected HadoopCoGroupClosure createClosure() { return new TezCoGroupClosure( flowProcess, splice.getNumSelfJoins(), keyFields, valuesFields ); }
@Override protected HadoopCoGroupClosure createClosure() { return new HadoopCoGroupClosure( flowProcess, splice.getNumSelfJoins(), keyFields, valuesFields ); }
@Override protected HadoopCoGroupClosure createClosure() { return new TezCoGroupClosure( flowProcess, splice.getNumSelfJoins(), keyFields, valuesFields ); }
private static int hash( FlowElement flowElement ) { int lhs = flowElement.getClass().getName().hashCode(); int rhs = 0; if( flowElement instanceof Operator && ( (Operator) flowElement ).getOperation() != null ) rhs = ( (Operator) flowElement ).getOperation().getClass().getName().hashCode(); else if( flowElement instanceof Tap && ( (Tap) flowElement ).getScheme() != null ) rhs = ( (Tap) flowElement ).getScheme().getClass().getName().hashCode(); else if( flowElement instanceof Splice ) rhs = ( (Splice) flowElement ).getJoiner().getClass().getName().hashCode() + 31 * ( (Splice) flowElement ).getNumSelfJoins(); return pair( lhs, rhs ); }
@Override public void prepare() { if( role != IORole.source ) { throw new UnsupportedOperationException("Non-source group by not supported in CoGroupBufferInGate"); } if( role != IORole.sink ) { closure = new CoGroupBufferClosure(flowProcess, this.getSplice().getNumSelfJoins(), keyFields, valuesFields); } if( grouping != null && splice.getJoinDeclaredFields() != null && splice.getJoinDeclaredFields().isNone() ) { grouping.joinerClosure = closure; } }
private List<DataSet<Tuple>> computeSpliceInputsFieldsKeys(Splice splice, FlowNode node, List<DataSet<Tuple>> inputs, Fields[] inputFields, Fields[] keyFields, String[][] flinkKeys) { int numJoinInputs = splice.isSelfJoin() ? splice.getNumSelfJoins() + 1 : inputs.size(); List<Scope> inScopes = getInputScopes(node, splice); List<DataSet<Tuple>> inputs2;
@Override public void prepare() { if( role != IORole.source ) { throw new UnsupportedOperationException("Non-source group by not supported in GroupByInGate"); } if( role != IORole.sink ) { Fields[] keyFields; Fields[] valuesFields; if(splice.isSelfJoin()) { keyFields = new Fields[splice.getNumSelfJoins() + 1]; valuesFields = new Fields[splice.getNumSelfJoins() + 1]; for(int i=0; i<keyFields.length; i++) { keyFields[i] = super.keyFields[0]; valuesFields[i] = super.valuesFields[0]; } } else { keyFields = super.keyFields; valuesFields = super.valuesFields; } closure = new JoinClosure(flowProcess, keyFields, valuesFields); } if( grouping != null && splice.getJoinDeclaredFields() != null && splice.getJoinDeclaredFields().isNone() ) { grouping.joinerClosure = closure; } this.resultIterator = new JoinResultIterator(closure, this.splice.getJoiner()); }
@Override public void prepare() { super.prepare(); keyValues = createKeyValuesArray(); closure = new MemoryCoGroupClosure( flowProcess, splice.getNumSelfJoins(), keyFields, valuesFields ); if( grouping != null && splice.getJoinDeclaredFields() != null && splice.getJoinDeclaredFields().isNone() ) grouping.joinerClosure = closure; }
@Override public void prepare() { int numJoinInputs = this.splice.isSelfJoin() ? this.splice.getNumSelfJoins() + 1 : this.splice.getPrevious().length; Fields[] keyFields = new Fields[numJoinInputs]; Fields[] valueFields = new Fields[numJoinInputs]; Scope outgoingScope = outgoingScopes.get( 0 ); if(!this.splice.isSelfJoin()) { for(int i=0; i < numJoinInputs; i++) { Scope incomingScope = incomingScopes.get( i ); int ordinal = incomingScope.getOrdinal(); keyFields[ordinal] = outgoingScope.getKeySelectors().get(incomingScope.getName()); valueFields[ordinal] = incomingScope.getIncomingSpliceFields(); } } else { Scope incomingScope = incomingScopes.get(0); keyFields[0] = outgoingScope.getKeySelectors().get(incomingScope.getName()); valueFields[0] = incomingScope.getIncomingSpliceFields(); for (int i = 1; i < numJoinInputs; i++) { keyFields[i] = keyFields[0]; valueFields[i] = valueFields[0]; } } this.closure = new JoinClosure(this.flowProcess, keyFields, valueFields); this.joiner = this.splice.getJoiner(); this.entryIterator = new TupleEntryChainIterator(outgoingScope.getOutValuesFields()); }