private void setupRankingArgs(WindowTableFunctionDef wdwTFnDef, WindowFunctionDef wFnDef, WindowFunctionSpec wSpec) throws SemanticException { if (wSpec.getArgs().size() > 0) { throw new SemanticException("Ranking Functions can take no arguments"); } OrderDef oDef = wdwTFnDef.getOrder(); List<OrderExpressionDef> oExprs = oDef.getExpressions(); for (OrderExpressionDef oExpr : oExprs) { wFnDef.addArg(oExpr); } }
if (wFnDef.getArgs() != null) { for (PTFExpressionDef arg : wFnDef.getArgs()) { initialize(arg, inpShape); if (wFnDef.getWindowFrame() != null) { WindowFrameDef wFrmDef = wFnDef.getWindowFrame(); initialize(wFrmDef, inpShape); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { aliases.add(wFnDef.getAlias()); if (wFnDef.isPivotResult()) { fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector()); } else { fieldOIs.add(wFnDef.getOI());
static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException { List<PTFExpressionDef> args = def.getArgs(); List<ObjectInspector> argOIs = new ArrayList<ObjectInspector>(); ObjectInspector[] funcArgOIs = null; if (args != null) { for (PTFExpressionDef arg : args) { argOIs.add(arg.getOI()); } funcArgOIs = new ObjectInspector[args.size()]; funcArgOIs = argOIs.toArray(funcArgOIs); } GenericUDAFEvaluator wFnEval = FunctionRegistry.getGenericWindowingEvaluator(def.getName(), argOIs, def.isDistinct(), def.isStar()); ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs); def.setWFnEval(wFnEval); def.setOI(OI); }
private Object evaluateFunctionOnPartition(WindowFunctionDef wFn, PTFPartition partition) throws HiveException { BasePartitionEvaluator partitionEval = wFn.getWFnEval() .getPartitionWindowingEvaluator(wFn.getWindowFrame(), partition, wFn.getArgs(), wFn.getOI(), nullsLast); return partitionEval.getPartitionAgg(); }
static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException { List<PTFExpressionDef> args = def.getArgs(); List<ObjectInspector> argOIs = new ArrayList<ObjectInspector>(); ObjectInspector[] funcArgOIs = null; if (args != null) { for (PTFExpressionDef arg : args) { argOIs.add(arg.getOI()); } funcArgOIs = new ObjectInspector[args.size()]; funcArgOIs = argOIs.toArray(funcArgOIs); } GenericUDAFEvaluator wFnEval = def.getWFnEval(); ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs); def.setWFnEval(wFnEval); def.setOI(OI); }
throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(spec.getName())); WindowFunctionDef def = new WindowFunctionDef(); def.setName(spec.getName()); def.setAlias(spec.getAlias()); def.setDistinct(spec.isDistinct()); def.setExpressionTreeString(spec.getExpression().toStringTree()); def.setStar(spec.isStar()); def.setPivotResult(wFnInfo.isPivotResult()); ShapeDetails inpShape = wdwTFnDef.getRawInputShape(); throw new SemanticException(he); def.addArg(argDef); def.setWindowFrame(wdwFrame);
@Override public void initializeStreaming(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide) throws HiveException { int[] span = setCanAcceptInputAsStream(cfg); if (!canAcceptInputAsStream) { return; } WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i); WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); GenericUDAFEvaluator fnEval = wFnDef.getWFnEval(); GenericUDAFEvaluator streamingEval = fnEval .getWindowingEvaluator(wdwFrame); if (streamingEval != null) { wFnDef.setWFnEval(streamingEval); if (wFnDef.isPivotResult()) { ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI(); wFnDef.setOI(listOI.getListElementObjectInspector()); } } } if ( tabDef.getRankLimit() != -1 ) { rnkLimitDef = new RankLimit(tabDef.getRankLimit(), tabDef.getRankLimitFunction(), tabDef.getWindowFunctions()); } streamingState = new StreamingState(cfg, inputOI, isMapSide, tabDef, span[0], span[1]); }
ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression(); ObjectInspector wFnOI = null; if (wFnDef.isPivotResult()) { wFnOI = ((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector(); wFnOI = wFnDef.getOI(); ColumnInfo cInfo = new ColumnInfo(wFnDef.getAlias(), TypeInfoUtils.getTypeInfoFromObjectInspector(wFnOI), null, true, true);
for (int j : wFnsToProcess) { WindowFunctionDef wFn = wTFnDef.getWindowFunctions().get(j); if (wFn.getWFnEval() instanceof ISupportStreamingModeForWindowing) { Object iRow = iPart.getAt(currIdx); int a = 0; if (wFn.getArgs() != null) { for (PTFExpressionDef arg : wFn.getArgs()) { args[j][a++] = arg.getExprEvaluator().evaluate(iRow); wFn.getWFnEval().aggregate(aggBuffers[j], args[j]); Object out = ((ISupportStreamingModeForWindowing) wFn.getWFnEval()) .getNextResult(aggBuffers[j]); if (out != null) { out = null; } else { out = ObjectInspectorUtils.copyToStandardObject(out, wFn.getOI());
WindowingIterator(PTFPartition iPart, ArrayList<Object> output, List<?>[] outputFromPivotFunctions, int[] wFnsToProcess) { this.iPart = iPart; this.output = output; this.outputFromPivotFunctions = outputFromPivotFunctions; this.wFnsToProcess = wFnsToProcess; this.currIdx = 0; wTFnDef = (WindowTableFunctionDef) getTableDef(); ptfDesc = getQueryDef(); inputOI = iPart.getOutputOI(); aggBuffers = new AggregationBuffer[wTFnDef.getWindowFunctions().size()]; args = new Object[wTFnDef.getWindowFunctions().size()][]; try { for (int j : wFnsToProcess) { WindowFunctionDef wFn = wTFnDef.getWindowFunctions().get(j); aggBuffers[j] = wFn.getWFnEval().getNewAggregationBuffer(); args[j] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; } } catch (HiveException he) { throw new RuntimeException(he); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); GenericUDAFEvaluator fnEval = wFn.getWFnEval(); int numRowsRemaining = wFn.getWindowFrame().getEnd().getRelativeOffset(); if (fnEval != null && fnEval instanceof ISupportStreamingModeForWindowing) { fnEval.terminate(streamingState.aggBuffers[i]); WindowingFunctionInfoHelper wFnInfo = getWindowingFunctionInfoHelper(wFn.getName()); if (!wFnInfo.isSupportsWindow()) { numRowsRemaining = ((ISupportStreamingModeForWindowing) fnEval)
private boolean canPushLimitToReduceSink(WindowTableFunctionDef wTFn) { for(WindowFunctionDef wFnDef : wTFn.getWindowFunctions() ) { if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) || (wFnDef.getWFnEval() instanceof GenericUDAFLeadEvaluator ) ) { continue; } WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); BoundaryDef end = wdwFrame.getEnd(); if (wdwFrame.getWindowType() == WindowType.RANGE) { return false; } if ( end.getDirection() == Direction.FOLLOWING ) { return false; } } return true; }
GenericUDAFEvaluator fnEval = wFn.getWFnEval(); if (wFn.getArgs() != null) { for (PTFExpressionDef arg : wFn.getArgs()) { streamingState.funcArgs[i][a++] = arg.getExprEvaluator().evaluate(row); int rowToProcess = streamingState.rollingPart.rowToProcess(wFn.getWindowFrame()); if (rowToProcess >= 0) { Object out = evaluateWindowFunction(wFn, rowToProcess, streamingState.rollingPart);
private static void fillInPTFEvaluators( List<WindowFunctionDef> windowsFunctions, String[] evaluatorFunctionNames, WindowFrameDef[] evaluatorWindowFrameDefs, List<ExprNodeDesc>[] evaluatorInputExprNodeDescLists) throws HiveException { final int functionCount = windowsFunctions.size(); for (int i = 0; i < functionCount; i++) { WindowFunctionDef winFunc = windowsFunctions.get(i); evaluatorFunctionNames[i] = winFunc.getName(); evaluatorWindowFrameDefs[i] = winFunc.getWindowFrame(); List<PTFExpressionDef> args = winFunc.getArgs(); if (args != null) { List<ExprNodeDesc> exprNodeDescList = new ArrayList<ExprNodeDesc>(); for (PTFExpressionDef arg : args) { exprNodeDescList.add(arg.getExprNode()); } evaluatorInputExprNodeDescLists[i] = exprNodeDescList; } } }
private boolean streamingPossible(Configuration cfg, WindowFunctionDef wFnDef) throws HiveException { WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); WindowingFunctionInfoHelper wFnInfo = getWindowingFunctionInfoHelper(wFnDef.getName()); if (!wFnInfo.isSupportsWindow()) { return true;
boolean processWindow = processWindow(wFn.getWindowFrame()); pItr.reset(); if ( !processWindow ) { Object out = evaluateFunctionOnPartition(wFn, iPart); if ( !wFn.isPivotResult()) { out = new SameList(iPart.size(), out);
private List<FieldNode> getWindowFunctionColumns(WindowTableFunctionDef tDef) { List<FieldNode> columns = new ArrayList<>(); if (tDef.getWindowFunctions() != null) { for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { columns.add(new FieldNode(wDef.getAlias())); } } return columns; }
private List<Integer> rankingFunctions(WindowTableFunctionDef wTFn) { List<Integer> rFns = new ArrayList<Integer>(); for(int i=0; i < wTFn.getWindowFunctions().size(); i++ ) { WindowFunctionDef wFnDef = wTFn.getWindowFunctions().get(i); if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) ) { rFns.add(i); } } return rFns; }
private void initializeWindowingFunctionInfoHelpers() throws SemanticException { // getWindowFunctionInfo() cannot be called during map/reduce tasks. So cache necessary // values during query compilation, and rely on plan serialization to bring this info // to the object during the map/reduce tasks. if (windowingFunctionHelpers != null) { return; } windowingFunctionHelpers = new HashMap<String, WindowingFunctionInfoHelper>(); WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); GenericUDAFEvaluator fnEval = wFn.getWFnEval(); WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getName()); boolean supportsWindow = wFnInfo.isSupportsWindow(); windowingFunctionHelpers.put(wFn.getName(), new WindowingFunctionInfoHelper(supportsWindow)); } }
RankLimit(int rankLimit, int rankFnIdx, List<WindowFunctionDef> wdwFnDefs) { this.rankLimit = rankLimit; this.rankFnIdx = rankFnIdx; this.fnOutOI = (PrimitiveObjectInspector) wdwFnDefs.get(rankFnIdx).getOI(); this.currentRank = -1; }