/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. */ public Object evaluate(AggregationBuffer agg) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { return terminatePartial(agg); } else { return terminate(agg); } }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. */ public Object evaluate(AggregationBuffer agg) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { return terminatePartial(agg); } else { return terminate(agg); } }
private List<Object> runPartial2(List<Object> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL2, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.merge(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; } }
private List<Object> runPartial1(List<Object[]> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object[]> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL1, info.getParameterObjectInspectors()); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.iterate(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; }
/** * Calculate the partial result sum + count giving a parition range * @return a 2-element Object array of [count long, sum ResultType] */ private Object[] calcPartialResult(PTFPartitionIterator<Object> pItr, LeadLagInfo leadLagInfo) throws HiveException { // To handle the case like SUM(LAG(f)) over(), aggregation function includes // LAG/LEAD call PTFOperator.connectLeadLagFunctionsToPartition(leadLagInfo, pItr); AggregationBuffer aggBuffer = wrappedEvaluator.getNewAggregationBuffer(); Object[] argValues = new Object[parameters == null ? 0 : parameters.size()]; while(pItr.hasNext()) { Object row = pItr.next(); int i = 0; if ( parameters != null ) { for(PTFExpressionDef param : parameters) { argValues[i++] = param.getExprEvaluator().evaluate(row); } } wrappedEvaluator.aggregate(aggBuffer, argValues); } // The object [count LongWritable, sum ResultType] is reused during evaluating Object[] partial = (Object[])wrappedEvaluator.terminatePartial(aggBuffer); return new Object[] {((LongWritable)partial[0]).get(), ObjectInspectorUtils.copyToStandardObject(partial[1], outputOI)}; }
/** * Calculate the partial result sum + count giving a parition range * @return a 2-element Object array of [count long, sum ResultType] */ private Object[] calcPartialResult(PTFPartitionIterator<Object> pItr, LeadLagInfo leadLagInfo) throws HiveException { // To handle the case like SUM(LAG(f)) over(), aggregation function includes // LAG/LEAD call PTFOperator.connectLeadLagFunctionsToPartition(leadLagInfo, pItr); AggregationBuffer aggBuffer = wrappedEvaluator.getNewAggregationBuffer(); Object[] argValues = new Object[parameters == null ? 0 : parameters.size()]; while(pItr.hasNext()) { Object row = pItr.next(); int i = 0; if ( parameters != null ) { for(PTFExpressionDef param : parameters) { argValues[i++] = param.getExprEvaluator().evaluate(row); } } wrappedEvaluator.aggregate(aggBuffer, argValues); } // The object [count LongWritable, sum ResultType] is reused during evaluating Object[] partial = (Object[])wrappedEvaluator.terminatePartial(aggBuffer); return new Object[] {((LongWritable)partial[0]).get(), ObjectInspectorUtils.copyToStandardObject(partial[1], outputOI)}; }
eval1.iterate(buffer1, new Object[]{150d, 210d}); eval1.iterate(buffer1, new Object[]{200d, 220d}); Object object1 = eval1.terminatePartial(buffer1); eval2.iterate(buffer2, new Object[]{300d, 250d}); eval2.iterate(buffer2, new Object[]{350d, 260d}); Object object2 = eval2.terminatePartial(buffer2);
final Object result; if (isPartial) { result = evaluator.terminatePartial(aggregationBuffer); } else { result = evaluator.terminate(aggregationBuffer);
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. */ public Object evaluate(AggregationBuffer agg) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { return terminatePartial(agg); } else { return terminate(agg); } }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. */ public Object evaluate(AggregationBuffer agg) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { return terminatePartial(agg); } else { return terminate(agg); } }
@Override public Tuple exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.PARTIAL1, constantsInfo); inited = true; } DataBag b = (DataBag)input.get(0); AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); for (Iterator<Tuple> it = b.iterator(); it.hasNext();) { Tuple t = it.next(); List inputs = schemaAndEvaluatorInfo.inputObjectInspector.getStructFieldsDataAsList(t); schemaAndEvaluatorInfo.evaluator.iterate(agg, inputs.toArray()); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminatePartial(agg); Tuple result = tf.newTuple(); result.append(HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.intermediateOutputObjectInspector, null)); return result; } catch (Exception e) { throw new IOException(e); } } }
@Override public Tuple exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.PARTIAL2, constantsInfo); inited = true; } DataBag b = (DataBag)input.get(0); AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); for (Iterator<Tuple> it = b.iterator(); it.hasNext();) { Tuple t = it.next(); schemaAndEvaluatorInfo.evaluator.merge(agg, t.get(0)); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminatePartial(agg); Tuple result = tf.newTuple(); result.append(HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.intermediateOutputObjectInspector, null)); return result; } catch (Exception e) { throw new IOException(e); } } }