@Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { return wrappedEval.init(m, parameters); }
ObjectInspector poi1 = eval1.init(GenericUDAFEvaluator.Mode.PARTIAL1, new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector}); ObjectInspector poi2 = eval2.init(GenericUDAFEvaluator.Mode.PARTIAL1, new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, PrimitiveObjectInspectorFactory.javaDoubleObjectInspector}); GenericUDAFEvaluator.AggregationBuffer buffer1 = eval1.getNewAggregationBuffer(); eval1.iterate(buffer1, new Object[]{100d, 200d}); eval1.iterate(buffer1, new Object[]{150d, 210d}); eval1.iterate(buffer1, new Object[]{200d, 220d}); Object object1 = eval1.terminatePartial(buffer1); GenericUDAFEvaluator.AggregationBuffer buffer2 = eval2.getNewAggregationBuffer(); eval2.iterate(buffer2, new Object[]{250d, 230d}); eval2.iterate(buffer2, new Object[]{250d, 240d}); eval2.iterate(buffer2, new Object[]{300d, 250d}); eval2.iterate(buffer2, new Object[]{350d, 260d}); Object object2 = eval2.terminatePartial(buffer2); ObjectInspector coi = eval2.init(GenericUDAFEvaluator.Mode.FINAL, new ObjectInspector[]{poi1}); GenericUDAFEvaluator.AggregationBuffer buffer3 = eval2.getNewAggregationBuffer(); eval2.merge(buffer3, object1); eval2.merge(buffer3, object2); Object result = eval2.terminate(buffer3); assertEquals("0.987829161147262", String.valueOf(result));
protected void resetAggregations(AggregationBuffer[] aggs) throws HiveException { for (int i = 0; i < aggs.length; i++) { aggregationEvaluators[i].reset(aggs[i]); } }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. */ public Object evaluate(AggregationBuffer agg) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { return terminatePartial(agg); } else { return terminate(agg); } }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. * @param parameters * The row, can be inspected by the OIs passed in init(). */ public void aggregate(AggregationBuffer agg, Object[] parameters) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { iterate(agg, parameters); } else { assert (parameters.length == 1); merge(agg, parameters[0]); } }
private Object runPartialFinal(List<Object[]> values) throws Exception { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.FINAL, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (Object partialResult : runPartial1(values)) { eval.merge(buf, partialResult); } return eval.terminate(buf); }
private Object runComplete(List<Object[]> values) throws SemanticException, HiveException { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.COMPLETE, info.getParameterObjectInspectors()); AggregationBuffer agg = eval.getNewAggregationBuffer(); for (Object[] parameters : values) { eval.iterate(agg, parameters); } return eval.terminate(agg); }
private List<Object> runPartial1(List<Object[]> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object[]> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL1, info.getParameterObjectInspectors()); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.iterate(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; }
private List<Object> runPartial2(List<Object> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL2, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.merge(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; } }
fn.init(Mode.COMPLETE, inputOIs); fn = fn.getWindowingEvaluator(wdwFrame(numPreceding, numFollowing)); AggregationBuffer agg = fn.getNewAggregationBuffer(); ISupportStreamingModeForWindowing oS = (ISupportStreamingModeForWindowing) fn; while (inVals.hasNext()) { typeHandler.set(inVals.next(), in[0]); fn.aggregate(agg, in); Object out = oS.getNextResult(agg); if (out != null) { fn.terminate(agg);
aggregationBuffer = evaluator.getNewAggregationBuffer(); aggregationBuffers[key] = aggregationBuffer; parameterArray[0] = row[1]; evaluator.aggregate(aggregationBuffer, parameterArray); final Object result; if (isPartial) { result = evaluator.terminatePartial(aggregationBuffer); } else { result = evaluator.terminate(aggregationBuffer);
protected AggregationBuffer[] newAggregations() throws HiveException { AggregationBuffer[] aggs = new AggregationBuffer[aggregationEvaluators.length]; for (int i = 0; i < aggregationEvaluators.length; i++) { aggs[i] = aggregationEvaluators[i].getNewAggregationBuffer(); // aggregationClasses[i].reset(aggs[i]); } return aggs; }
@Test public void testGetPartitionWindowingEvaluatorWithInitCall() throws HiveException { BasePartitionEvaluator partition1Evaluator1 = udafEvaluator.getPartitionWindowingEvaluator( winFrame, partition1, parameters, outputOI, false); udafEvaluator.init(GenericUDAFEvaluator.Mode.COMPLETE, null); BasePartitionEvaluator newPartitionEvaluator = udafEvaluator.getPartitionWindowingEvaluator( winFrame, partition1, parameters, outputOI, false); Assert.assertNotEquals(partition1Evaluator1, newPartitionEvaluator); }
/** * Given a partition iterator, calculate the function value * @param pItr the partition pointer * @return the function value * @throws HiveException */ protected Object calcFunctionValue(PTFPartitionIterator<Object> pItr, LeadLagInfo leadLagInfo) throws HiveException { // To handle the case like SUM(LAG(f)) over(), aggregation function includes // LAG/LEAD call PTFOperator.connectLeadLagFunctionsToPartition(leadLagInfo, pItr); AggregationBuffer aggBuffer = wrappedEvaluator.getNewAggregationBuffer(); Object[] argValues = new Object[parameters == null ? 0 : parameters.size()]; while(pItr.hasNext()) { Object row = pItr.next(); int i = 0; if ( parameters != null ) { for(PTFExpressionDef param : parameters) { argValues[i++] = param.getExprEvaluator().evaluate(row); } } wrappedEvaluator.aggregate(aggBuffer, argValues); } // The object is reused during evaluating, make a copy here return ObjectInspectorUtils.copyToStandardObject(wrappedEvaluator.evaluate(aggBuffer), outputOI); }
/** * Calculate the partial result sum + count giving a parition range * @return a 2-element Object array of [count long, sum ResultType] */ private Object[] calcPartialResult(PTFPartitionIterator<Object> pItr, LeadLagInfo leadLagInfo) throws HiveException { // To handle the case like SUM(LAG(f)) over(), aggregation function includes // LAG/LEAD call PTFOperator.connectLeadLagFunctionsToPartition(leadLagInfo, pItr); AggregationBuffer aggBuffer = wrappedEvaluator.getNewAggregationBuffer(); Object[] argValues = new Object[parameters == null ? 0 : parameters.size()]; while(pItr.hasNext()) { Object row = pItr.next(); int i = 0; if ( parameters != null ) { for(PTFExpressionDef param : parameters) { argValues[i++] = param.getExprEvaluator().evaluate(row); } } wrappedEvaluator.aggregate(aggBuffer, argValues); } // The object [count LongWritable, sum ResultType] is reused during evaluating Object[] partial = (Object[])wrappedEvaluator.terminatePartial(aggBuffer); return new Object[] {((LongWritable)partial[0]).get(), ObjectInspectorUtils.copyToStandardObject(partial[1], outputOI)}; }
@Override public Object exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.FINAL, constantsInfo); schemaAndEvaluatorInfo.evaluator.configure(instantiateMapredContext()); inited = true; } DataBag b = (DataBag)input.get(0); AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); for (Iterator<Tuple> it = b.iterator(); it.hasNext();) { Tuple t = it.next(); schemaAndEvaluatorInfo.evaluator.merge(agg, t.get(0)); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminate(agg); Object result = HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.outputObjectInspector, null); return result; } catch (Exception e) { throw new IOException(e); } } }
wrappedEval.iterate(fb, parameters);
@Override public Object exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.COMPLETE, constantsInfo); inited = true; } AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); DataBag bg = (DataBag) input.get(0); Tuple tp = null; for (Iterator<Tuple> it = bg.iterator(); it.hasNext();) { tp = it.next(); List inputs = schemaAndEvaluatorInfo.inputObjectInspector.getStructFieldsDataAsList(tp); schemaAndEvaluatorInfo.evaluator.iterate(agg, inputs.toArray()); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminate(agg); Object result = HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.outputObjectInspector, null); return result; } catch (Exception e) { throw new IOException(e); } }
@Override public Tuple exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.PARTIAL1, constantsInfo); inited = true; } DataBag b = (DataBag)input.get(0); AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); for (Iterator<Tuple> it = b.iterator(); it.hasNext();) { Tuple t = it.next(); List inputs = schemaAndEvaluatorInfo.inputObjectInspector.getStructFieldsDataAsList(t); schemaAndEvaluatorInfo.evaluator.iterate(agg, inputs.toArray()); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminatePartial(agg); Tuple result = tf.newTuple(); result.append(HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.intermediateOutputObjectInspector, null)); return result; } catch (Exception e) { throw new IOException(e); } } }
@Override public Object terminate(AggregationBuffer agg) throws HiveException { SumAvgStreamingState ss = (SumAvgStreamingState) agg; Object o = wrappedEval.terminate(ss.wrappedBuf); // After all the rows are processed, continue to generate results for the rows that results haven't generated. // For the case: X following and Y following, process first Y-X results and then insert X nulls. // For the case X preceding and Y following, process Y results. for (int i = Math.max(0, wFrameDef.getStart().getRelativeOffset()); i < wFrameDef.getEnd().getRelativeOffset(); i++) { if (ss.hasResultReady()) { ss.results.add(getNextResult(ss)); } ss.numRows++; } for (int i = 0; i < wFrameDef.getStart().getRelativeOffset(); i++) { if (ss.hasResultReady()) { ss.results.add(null); } ss.numRows++; } return o; }