public HadoopGroupGate( FlowProcess flowProcess, Splice splice, IORole role ) { super( flowProcess, splice, role ); isBufferJoin = splice.getJoiner() instanceof BufferJoin; }
public HadoopGroupGate( FlowProcess flowProcess, Splice splice, IORole role ) { super( flowProcess, splice, role ); isBufferJoin = splice.getJoiner() instanceof BufferJoin; }
private static int hash( FlowElement flowElement ) { int lhs = flowElement.getClass().getName().hashCode(); int rhs = 0; if( flowElement instanceof Operator && ( (Operator) flowElement ).getOperation() != null ) rhs = ( (Operator) flowElement ).getOperation().getClass().getName().hashCode(); else if( flowElement instanceof Tap && ( (Tap) flowElement ).getScheme() != null ) rhs = ( (Tap) flowElement ).getScheme().getClass().getName().hashCode(); else if( flowElement instanceof Splice ) rhs = ( (Splice) flowElement ).getJoiner().getClass().getName().hashCode() + 31 * ( (Splice) flowElement ).getNumSelfJoins(); return pair( lhs, rhs ); }
private void performJoinWith( Tuple keyTuple ) { // never replace the first array, pos == 0 for( int i = 1; i < keyValues.length; i++ ) { // if key does not exist, #get will create an empty array list, // and store the key, which is not a copy if( keyValues[ i ].containsKey( keyTuple ) ) collections[ i ] = keyValues[ i ].get( keyTuple ); else collections[ i ] = Collections.EMPTY_LIST; } closure.reset( collections ); keyEntry.setTuple( keyTuple ); tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) ); next.receive( this, 0, grouping ); }
private void push( Collection<Tuple>[] collections, Tuple keysTuple ) { closure.reset( collections ); keyEntry.setTuple( closure.getGroupTuple( keysTuple ) ); // create Closure type here tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) ); next.receive( this, 0, grouping ); } }
@Override public void prepare() { int numJoinInputs = this.splice.isSelfJoin() ? this.splice.getNumSelfJoins() + 1 : this.splice.getPrevious().length; Fields[] keyFields = new Fields[numJoinInputs]; Fields[] valueFields = new Fields[numJoinInputs]; Scope outgoingScope = outgoingScopes.get( 0 ); if(!this.splice.isSelfJoin()) { for(int i=0; i < numJoinInputs; i++) { Scope incomingScope = incomingScopes.get( i ); int ordinal = incomingScope.getOrdinal(); keyFields[ordinal] = outgoingScope.getKeySelectors().get(incomingScope.getName()); valueFields[ordinal] = incomingScope.getIncomingSpliceFields(); } } else { Scope incomingScope = incomingScopes.get(0); keyFields[0] = outgoingScope.getKeySelectors().get(incomingScope.getName()); valueFields[0] = incomingScope.getIncomingSpliceFields(); for (int i = 1; i < numJoinInputs; i++) { keyFields[i] = keyFields[0]; valueFields[i] = valueFields[0]; } } this.closure = new JoinClosure(this.flowProcess, keyFields, valueFields); this.joiner = this.splice.getJoiner(); this.entryIterator = new TupleEntryChainIterator(outgoingScope.getOutValuesFields()); }
public void accept( Tuple key, Iterator<Tuple>[] values ) { key = unwrapGrouping( key ); closure.reset( key, values ); // Buffer is using JoinerClosure directly if( !isBufferJoin ) tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) ); else tupleEntryIterator.reset( values ); keyEntry.setTuple( closure.getGroupTuple( key ) ); next.receive( this, 0, grouping ); }
@SuppressWarnings("unchecked") @Override public void run(Object input) { KeyPeekingIterator keyPeekingIt; try { keyPeekingIt = new KeyPeekingIterator((Iterator<Tuple>)input, keyBuilder[0]); } catch(ClassCastException cce) { throw new RuntimeException("GroupByInGate requires Iterator<Tuple>.", cce); } closure.reset(keyPeekingIt); // Buffer is using JoinerClosure directly if( !isBufferJoin ) { tupleEntryIterator.reset(splice.getJoiner().getIterator(closure)); } else { tupleEntryIterator.reset(keyPeekingIt); } Tuple groupTuple = keyPeekingIt.peekNextKey(); keyEntry.setTuple( groupTuple ); next.receive( this, grouping ); }
@SuppressWarnings("unchecked") @Override public void run(Object input) { KeyPeekingIterator iterator; try { iterator = new KeyPeekingIterator((Iterator<Tuple3<Tuple, Integer, Tuple>>)input); } catch(ClassCastException cce) { throw new RuntimeException("CoGroupBufferInGate requires Iterator<Tuple3<Tuple, Integer, Tuple>", cce); } Tuple key = iterator.peekNextKey(); closure.reset(iterator); // Buffer is using JoinerClosure directly if( !isBufferJoin ) { tupleEntryIterator.reset(splice.getJoiner().getIterator(closure)); } keyEntry.setTuple( this.closure.getGroupTuple(key) ); next.receive( this, grouping ); }
public void accept( Tuple key, Iterator<Tuple>[] values ) { key = unwrapGrouping( key ); closure.reset( key, values ); // Buffer is using JoinerClosure directly if( !isBufferJoin ) tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) ); else tupleEntryIterator.reset( values ); keyEntry.setTuple( closure.getGroupTuple( key ) ); next.receive( this, 0, grouping ); }
@Override public void prepare() { if( role != IORole.source ) { throw new UnsupportedOperationException("Non-source group by not supported in GroupByInGate"); } if( role != IORole.sink ) { Fields[] keyFields; Fields[] valuesFields; if(splice.isSelfJoin()) { keyFields = new Fields[splice.getNumSelfJoins() + 1]; valuesFields = new Fields[splice.getNumSelfJoins() + 1]; for(int i=0; i<keyFields.length; i++) { keyFields[i] = super.keyFields[0]; valuesFields[i] = super.valuesFields[0]; } } else { keyFields = super.keyFields; valuesFields = super.valuesFields; } closure = new JoinClosure(flowProcess, keyFields, valuesFields); } if( grouping != null && splice.getJoinDeclaredFields() != null && splice.getJoinDeclaredFields().isNone() ) { grouping.joinerClosure = closure; } this.resultIterator = new JoinResultIterator(closure, this.splice.getJoiner()); }