Fields resolveOutgoingValuesSelector( Set<Scope> incomingScopes, Fields argumentFields, Fields declaredFields ) { try { return resolveOutgoingSelector( incomingScopes, argumentFields, declaredFields ); } catch( Exception exception ) { if( exception instanceof OperatorException ) throw (OperatorException) exception; throw new OperatorException( this, "could not resolve outgoing values selector in: " + this, exception ); } } }
Fields resolveOutgoingGroupingSelector( Set<Scope> incomingScopes, Fields argumentSelector, Fields declared ) { try { return resolveOutgoingSelector( incomingScopes, argumentSelector, declared ); } catch( Exception exception ) { if( exception instanceof OperatorException ) throw (OperatorException) exception; if( isBuffer() ) throw new OperatorException( this, "could not resolve outgoing values selector in: " + this, exception ); else throw new OperatorException( this, "could not resolve outgoing grouping selector in: " + this, exception ); } } }
Map<String, Fields> resolveSortingSelectors( Set<Scope> incomingScopes ) { try { if( getSortingSelectors().isEmpty() ) return null; return resolveSelectorsAgainstIncoming( incomingScopes, getSortingSelectors(), "sorting" ); } catch( FieldsResolverException exception ) { throw new OperatorException( this, OperatorException.Kind.sorting, exception.getSourceFields(), exception.getSelectorFields(), exception ); } catch( RuntimeException exception ) { throw new OperatorException( this, "could not resolve sorting selector in: " + this, exception ); } }
Map<String, Fields> resolveGroupingSelectors( Set<Scope> incomingScopes ) { try { Map<String, Fields> groupingSelectors = getKeySelectors(); Map<String, Fields> groupingFields = resolveSelectorsAgainstIncoming( incomingScopes, groupingSelectors, "grouping" ); if( !verifySameSize( groupingFields ) ) throw new OperatorException( this, "all grouping fields must be same size: " + toString() ); verifySameTypes( groupingSelectors, groupingFields ); return groupingFields; } catch( FieldsResolverException exception ) { throw new OperatorException( this, OperatorException.Kind.grouping, exception.getSourceFields(), exception.getSelectorFields(), exception ); } catch( RuntimeException exception ) { throw new OperatorException( this, "could not resolve grouping selector in: " + this, exception ); } }
void verifyDeclaredFields( Fields declared ) { if( declared.isDefined() && declared.size() == 0 ) throw new OperatorException( this, "field declaration: " + getFieldDeclaration().printVerbose() + ", resolves to an empty field set, current grouping is on all fields" ); }
void verifyOutputSelector( Fields outputSelector ) { if( outputSelector.isDefined() && outputSelector.size() == 0 ) throw new OperatorException( this, "output selector: " + getOutputSelector().printVerbose() + ", resolves to an empty field set, current grouping is on all fields" ); }
private void verifyAggregatorArguments( Fields argumentFields, Scope incomingScope ) { if( ( !isBuffer() ) && incomingScope.isEvery() && argumentFields.contains( incomingScope.getOperationDeclaredFields() ) ) throw new OperatorException( this, "arguments may not select a declared field from a previous Every" ); }
void verifyArguments( Fields argumentSelector ) { if( argumentSelector.isUnknown() ) return; if( operation.getNumArgs() != Operation.ANY && argumentSelector.size() < operation.getNumArgs() ) throw new OperatorException( this, "resolved wrong number of arguments: " + argumentSelector.printVerbose() + ", expected: " + operation.getNumArgs() ); }
@Override public void receive( Duct previous, int ordinal, TupleEntry incomingEntry ) { this.incomingEntry = incomingEntry; argumentsEntry.setTuple( argumentsBuilder.makeResult( incomingEntry.getTuple(), null ) ); try { function.operate( flowProcess, operationCall ); // adds results to collector } catch( CascadingException exception ) { handleException( exception, argumentsEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( each, "operator Each failed executing operation", throwable ), argumentsEntry ); } } }
@Override public void completeGroup( Duct previous, TupleEntry incomingEntry ) { this.incomingEntry = incomingEntry; operationCall.setArguments( null ); try { groupAssertion.doAssert( flowProcess, operationCall ); // collector calls next reducing.completeGroup( this, incomingEntry ); } catch( CascadingException exception ) { handleException( exception, incomingEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed completing operation: " + every.getOperation(), throwable ), incomingEntry ); } } }
@Override public void completeGroup( Duct previous, TupleEntry incomingEntry ) { this.incomingEntry = incomingEntry; operationCall.setArguments( null ); operationCall.setOutputCollector( outputCollector ); try { aggregator.complete( flowProcess, operationCall ); // collector calls next } catch( CascadingException exception ) { handleException( exception, incomingEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed completing operation: " + every.getOperation(), throwable ), incomingEntry ); } } }
private Map<String, Fields> resolveSelectorsAgainstIncoming( Set<Scope> incomingScopes, Map<String, Fields> selectors, String type ) { Map<String, Fields> resolvedFields = new HashMap<String, Fields>(); for( Scope incomingScope : incomingScopes ) { Fields selector = selectors.get( incomingScope.getName() ); if( selector == null ) throw new OperatorException( this, "no " + type + " selector found for: " + incomingScope.getName() ); Fields incomingFields; if( selector.isNone() ) incomingFields = Fields.NONE; else if( selector.isAll() ) incomingFields = incomingScope.getIncomingSpliceFields(); else if( selector.isGroup() ) incomingFields = incomingScope.getOutGroupingFields(); else if( selector.isValues() ) incomingFields = incomingScope.getOutValuesFields().subtract( incomingScope.getOutGroupingFields() ); else incomingFields = incomingScope.getIncomingSpliceFields().select( selector ); resolvedFields.put( incomingScope.getName(), incomingFields ); } return resolvedFields; }
Fields resolveArgumentSelector( Set<Scope> incomingScopes ) { Fields argumentSelector = getArgumentSelector(); try { Scope incomingScope = getFirst( incomingScopes ); if( argumentSelector.isAll() ) return resolveIncomingOperationArgumentFields( incomingScope ); if( argumentSelector.isGroup() ) return incomingScope.getOutGroupingFields(); if( argumentSelector.isValues() ) return incomingScope.getOutGroupingValueFields(); return resolveIncomingOperationArgumentFields( incomingScope ).select( argumentSelector ); } catch( FieldsResolverException exception ) { throw new OperatorException( this, OperatorException.Kind.argument, exception.getSourceFields(), argumentSelector, exception ); } catch( Exception exception ) { throw new OperatorException( this, "unable to resolve argument selector: " + argumentSelector.printVerbose(), exception ); } }
@Override public void receive( Duct previous, int ordinal, TupleEntry incomingEntry ) { argumentsEntry.setTuple( argumentsBuilder.makeResult( incomingEntry.getTuple(), null ) ); try { valueAssertion.doAssert( flowProcess, operationCall ); next.receive( this, 0, incomingEntry ); } catch( CascadingException exception ) { handleException( exception, argumentsEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( each, "operator Each failed executing operation", throwable ), argumentsEntry ); } } }
@Override public void receive( Duct previous, int ordinal, TupleEntry incomingEntry ) { argumentsEntry.setTuple( argumentsBuilder.makeResult( incomingEntry.getTuple(), null ) ); try { if( filter.isRemove( flowProcess, operationCall ) ) return; next.receive( this, 0, incomingEntry ); } catch( CascadingException exception ) { handleException( exception, argumentsEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( each, "operator Each failed executing operation", throwable ), argumentsEntry ); } } }
@Override public void startGroup( Duct previous, TupleEntry groupEntry ) { operationCall.setGroup( groupEntry ); operationCall.setArguments( null ); // zero it out operationCall.setOutputCollector( null ); // zero it out try { aggregator.start( flowProcess, operationCall ); } catch( CascadingException exception ) { handleException( exception, groupEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed starting operation: " + every.getOperation(), throwable ), groupEntry ); } reducing.startGroup( this, groupEntry ); }
@Override public void startGroup( Duct previous, TupleEntry groupEntry ) { operationCall.setGroup( groupEntry ); operationCall.setArguments( null ); // zero it out operationCall.setOutputCollector( null ); // zero it out try { groupAssertion.start( flowProcess, operationCall ); } catch( CascadingException exception ) { handleException( exception, groupEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed starting operation: " + every.getOperation(), throwable ), groupEntry ); } reducing.startGroup( this, groupEntry ); }
@Override public void receive( Duct previous, int ordinal, TupleEntry tupleEntry ) { try { argumentsEntry.setTuple( argumentsBuilder.makeResult( tupleEntry.getTuple(), null ) ); operationCall.setArguments( argumentsEntry ); groupAssertion.aggregate( flowProcess, operationCall ); } catch( CascadingException exception ) { handleException( exception, argumentsEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed executing operation: " + every.getOperation(), throwable ), argumentsEntry ); } next.receive( this, 0, tupleEntry ); }
@Override public void receive( Duct previous, int ordinal, TupleEntry tupleEntry ) { try { argumentsEntry.setTuple( argumentsBuilder.makeResult( tupleEntry.getTuple(), null ) ); operationCall.setArguments( argumentsEntry ); aggregator.aggregate( flowProcess, operationCall ); } catch( CascadingException exception ) { handleException( exception, argumentsEntry ); } catch( Throwable throwable ) { handleException( new OperatorException( every, "operator Every failed executing operation: " + every.getOperation(), throwable ), argumentsEntry ); } next.receive( this, ordinal, tupleEntry ); }
@Override public Scope outgoingScopeFor( Set<Scope> incomingScopes ) { Scope incomingScope = getFirst( incomingScopes ); if( !isBuffer() && incomingScope.getOutValuesFields().isNone() ) throw new OperatorException( this, "only a Buffer may be preceded by a CoGroup declaring Fields.NONE as the join fields" ); Fields argumentFields = resolveArgumentSelector( incomingScopes ); verifyArguments( argumentFields ); // we currently don't support using result from a previous Every in the current Every verifyAggregatorArguments( argumentFields, incomingScope ); Fields declaredFields = resolveDeclared( incomingScopes, argumentFields ); verifyDeclaredFields( declaredFields ); Fields outgoingGroupingFields = resolveOutgoingGroupingSelector( incomingScopes, argumentFields, declaredFields ); verifyOutputSelector( outgoingGroupingFields ); Fields outgoingValuesFields = incomingScope.getOutValuesFields(); // the incoming fields eligible to be outgoing, for Every only the grouping fields. Fields passThroughFields = resolveIncomingOperationPassThroughFields( incomingScope ); Fields remainderFields = resolveRemainderFields( incomingScopes, argumentFields ); return new Scope( getName(), Scope.Kind.EVERY, passThroughFields, remainderFields, argumentFields, declaredFields, outgoingGroupingFields, outgoingValuesFields ); }