Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes));
@Override public void updateState(MapState map, List<TridentTuple> tuples, TridentCollector collector) { Map<List<Object>, List<TridentTuple>> grouped = new HashMap<>(); for(TridentTuple t: tuples) { List<Object> group = _groupFactory.create(t); List<TridentTuple> groupTuples = grouped.get(group); if(groupTuples==null) { groupTuples = new ArrayList<>(); grouped.put(group, groupTuples); } groupTuples.add(_inputFactory.create(t)); } List<List<Object>> uniqueGroups = new ArrayList<>(grouped.keySet()); List<ValueUpdater> updaters = new ArrayList<>(uniqueGroups.size()); for(List<Object> group: uniqueGroups) { updaters.add(new ReducerValueUpdater(_agg, grouped.get(group))); } List<Object> results = map.multiUpdate(uniqueGroups, updaters); for (int i = 0; i < uniqueGroups.size(); i++) { List<Object> group = uniqueGroups.get(i); Object result = results.get(i); collector.emit(_factory.create(new List[] { group, new Values(result) })); } }
Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes));
Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes));
@Override public void updateState(MapState map, List<TridentTuple> tuples, TridentCollector collector) { Map<List<Object>, List<TridentTuple>> grouped = new HashMap(); List<List<Object>> groups = new ArrayList<List<Object>>(tuples.size()); List<Object> values = new ArrayList<Object>(tuples.size()); for(TridentTuple t: tuples) { List<Object> group = _groupFactory.create(t); List<TridentTuple> groupTuples = grouped.get(group); if(groupTuples==null) { groupTuples = new ArrayList(); grouped.put(group, groupTuples); } groupTuples.add(_inputFactory.create(t)); } List<List<Object>> uniqueGroups = new ArrayList(grouped.keySet()); List<ValueUpdater> updaters = new ArrayList(uniqueGroups.size()); for(List<Object> group: uniqueGroups) { updaters.add(new ReducerValueUpdater(_agg, grouped.get(group))); } List<Object> results = map.multiUpdate(uniqueGroups, updaters); for(int i=0; i<uniqueGroups.size(); i++) { List<Object> group = uniqueGroups.get(i); Object result = results.get(i); collector.emit(_factory.create(new List[] {group, new Values(result) })); } }
@Override public void updateState(MapState map, List<TridentTuple> tuples, TridentCollector collector) { Map<List<Object>, List<TridentTuple>> grouped = new HashMap<>(); for(TridentTuple t: tuples) { List<Object> group = _groupFactory.create(t); List<TridentTuple> groupTuples = grouped.get(group); if(groupTuples==null) { groupTuples = new ArrayList<>(); grouped.put(group, groupTuples); } groupTuples.add(_inputFactory.create(t)); } List<List<Object>> uniqueGroups = new ArrayList<>(grouped.keySet()); List<ValueUpdater> updaters = new ArrayList<>(uniqueGroups.size()); for(List<Object> group: uniqueGroups) { updaters.add(new ReducerValueUpdater(_agg, grouped.get(group))); } List<Object> results = map.multiUpdate(uniqueGroups, updaters); for (int i = 0; i < uniqueGroups.size(); i++) { List<Object> group = uniqueGroups.get(i); Object result = results.get(i); collector.emit(_factory.create(new List[] { group, new Values(result) })); } }
private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, TridentTuple overrideTuple) { List<List>[] sides = state.sides; int[] indices = state.indices; for (int i = 0; i < indices.length; i++) { indices[i] = 0; } boolean keepGoing = true; // emit cross-join of all emitted tuples while (keepGoing) { List[] combined = new List[sides.length + 1]; combined[0] = state.group; for (int i = 0; i < sides.length; i++) { if (i == overrideIndex) { combined[i + 1] = overrideTuple; } else { combined[i + 1] = sides[i].get(indices[i]); } } collector.emit(_factory.create(combined)); keepGoing = increment(sides, indices, indices.length - 1, overrideIndex); } }
public void complete(ChainedResult val, TridentCollector collector) { val.setFollowThroughCollector(collector); for(int i=0; i<_aggs.length; i++) { _aggs[i].complete(val.objs[i], val.collectors[i]); } if(_aggs.length > 1) { // otherwise, tuples were emitted directly int[] indices = new int[val.collectors.length]; for(int i=0; i<indices.length; i++) { indices[i] = 0; } boolean keepGoing = true; //emit cross-join of all emitted tuples while(keepGoing) { List[] combined = new List[_aggs.length]; for(int i=0; i< _aggs.length; i++) { CaptureCollector capturer = (CaptureCollector) val.collectors[i]; combined[i] = capturer.captured.get(indices[i]); } collector.emit(_fact.create(combined)); keepGoing = increment(val.collectors, indices, indices.length - 1); } } }
public void complete(ChainedResult val, TridentCollector collector) { val.setFollowThroughCollector(collector); for (int i = 0; i < _aggs.length; i++) { _aggs[i].complete(val.objs[i], val.collectors[i]); } if (_aggs.length > 1) { // otherwise, tuples were emitted directly int[] indices = new int[val.collectors.length]; for (int i = 0; i < indices.length; i++) { indices[i] = 0; } boolean keepGoing = true; // emit cross-join of all emitted tuples while (keepGoing) { List[] combined = new List[_aggs.length]; for (int i = 0; i < _aggs.length; i++) { CaptureCollector capturer = (CaptureCollector) val.collectors[i]; combined[i] = capturer.captured.get(indices[i]); } collector.emit(_fact.create(combined)); keepGoing = increment(val.collectors, indices, indices.length - 1); } } }
private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, TridentTuple overrideTuple) { List<List>[] sides = state.sides; int[] indices = state.indices; for(int i=0; i<indices.length; i++) { indices[i] = 0; } boolean keepGoing = true; //emit cross-join of all emitted tuples while(keepGoing) { List[] combined = new List[sides.length+1]; combined[0] = state.group; for(int i=0; i<sides.length; i++) { if(i==overrideIndex) { combined[i+1] = overrideTuple; } else { combined[i+1] = sides[i].get(indices[i]); } } collector.emit(_factory.create(combined)); keepGoing = increment(sides, indices, indices.length - 1, overrideIndex); } }
@Override public void updateState(MapState map, List<TridentTuple> tuples, TridentCollector collector) { List<List<Object>> groups = new ArrayList<List<Object>>(tuples.size()); List<ValueUpdater> updaters = new ArrayList<ValueUpdater>(tuples.size()); for (TridentTuple t : tuples) { groups.add(_groupFactory.create(t)); updaters.add(new CombinerValueUpdater(_agg, _inputFactory.create(t).getValue(0))); } List<Object> newVals = map.multiUpdate(groups, updaters); for (int i = 0; i < tuples.size(); i++) { List<Object> key = groups.get(i); Object result = newVals.get(i); collector.emit(_factory.create(new List[] { key, new Values(result) })); } }
@Override public void updateState(MapState map, List<TridentTuple> tuples, TridentCollector collector) { List<List<Object>> groups = new ArrayList<List<Object>>(tuples.size()); List<ValueUpdater> updaters = new ArrayList<ValueUpdater>(tuples.size()); for(TridentTuple t: tuples) { groups.add(_groupFactory.create(t)); updaters.add(new CombinerValueUpdater(_agg,_inputFactory.create(t).getValue(0))); } List<Object> newVals = map.multiUpdate(groups, updaters); for(int i=0; i<tuples.size(); i++) { List<Object> key = groups.get(i); Object result = newVals.get(i); collector.emit(_factory.create(new List[] {key, new Values(result) })); } }
@Override public void prepare(Map conf, TridentMultiReducerContext context) { int[] sizes = new int[_sideFields.size() + 1]; sizes[0] = _numGroupFields; for (int i = 0; i < _sideFields.size(); i++) { sizes[i + 1] = _sideFields.get(i).size(); } _factory = new ComboList.Factory(sizes); }
public GroupedAggregator(Aggregator agg, Fields group, Fields input, int outSize) { _groupFields = group; _inFields = input; _agg = agg; int[] sizes = new int[2]; sizes[0] = _groupFields.size(); sizes[1] = outSize; _fact = new ComboList.Factory(sizes); }
public GroupedAggregator(Aggregator agg, Fields group, Fields input, int outSize) { _groupFields = group; _inFields = input; _agg = agg; int[] sizes = new int[2]; sizes[0] = _groupFields.size(); sizes[1] = outSize; _fact = new ComboList.Factory(sizes); }
public MapCombinerAggStateUpdater(CombinerAggregator agg, Fields groupFields, Fields inputFields) { _agg = agg; _groupFields = groupFields; _inputFields = inputFields; if (inputFields.size() != 1) { throw new IllegalArgumentException("Combiner aggs only take a single field as input. Got this instead: " + inputFields.toString()); } _factory = new ComboList.Factory(groupFields.size(), inputFields.size()); }
@Override public void emit(List<Object> values) { List[] delegates = new List[2]; delegates[0] = currGroup; delegates[1] = values; _collector.emit(_factory.create(delegates)); }
@Override public void emit(List<Object> values) { List[] delegates = new List[2]; delegates[0] = currGroup; delegates[1] = values; _collector.emit(_factory.create(delegates)); }
@Override public void prepare(Map conf, TridentMultiReducerContext context) { int[] sizes = new int[_sideFields.size() + 1]; sizes[0] = _numGroupFields; for(int i=0; i<_sideFields.size(); i++) { sizes[i+1] = _sideFields.get(i).size(); } _factory = new ComboList.Factory(sizes); }
public MapReducerAggStateUpdater(ReducerAggregator agg, Fields groupFields, Fields inputFields) { _agg = agg; _groupFields = groupFields; _inputFields = inputFields; _factory = new ComboList.Factory(groupFields.size(), 1); }