ptfDesc.setCfg(hCfg); ptfDesc.setLlInfo(llInfo); WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef(); ptfDesc.setFuncDef(wdwTFnDef); wdwTFnDef.setInput(translate(inpSpec, 0)); ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape(); wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION); wdwTFnDef.setResolverClassName(tFn.getClass().getName()); wdwTFnDef.setAlias("ptf_" + 1); wdwTFnDef.setExpressionTreeString(null); wdwTFnDef.setTransformsRawInput(false); tFn.initialize(hCfg, ptfDesc, wdwTFnDef); TableFunctionEvaluator tEval = tFn.getEvaluator(); wdwTFnDef.setTFunction(tEval); wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames()); wdwTFnDef.setRawInputShape(inpShape); wdwTFnDef.setPartition(partDef); wdwTFnDef.setOrder(ordDef); wdwTFnDef.setWindowFunctions(windowFunctions); RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef); ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR); wdwTFnDef.setOutputShape(wdwOutShape);
if (pDef instanceof WindowTableFunctionDef) { WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; if (tDef.getWindowFunctions() != null) { for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { if (wDef.getArgs() == null) { continue; if (tDef.getPartition() != null) { for (PTFExpressionDef col : tDef.getPartition().getExpressions()) { ExprNodeDesc exprNode = col.getExprNode(); prunedCols = mergeFieldNodesWithDesc(prunedCols, exprNode); if (tDef.getOrder() != null) { for (PTFExpressionDef col : tDef.getOrder().getExpressions()) { ExprNodeDesc exprNode = col.getExprNode(); prunedCols = mergeFieldNodesWithDesc(prunedCols, exprNode);
public void initializeWindowing(WindowTableFunctionDef def) throws HiveException { ShapeDetails inpShape = def.getInput().getOutputShape(); TableFunctionEvaluator tEval = def.getTFunction(); WindowingTableFunctionResolver tResolver = (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName()); tResolver.initialize(hConf, ptfDesc, def, tEval); for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { if (wFnDef.getArgs() != null) { for (PTFExpressionDef arg : wFnDef.getArgs()) { for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { aliases.add(wFnDef.getAlias()); if (wFnDef.isPivotResult()) { aliases, fieldOIs); tResolver.setWdwProcessingOutputOI(wdwOutOI); initialize(def.getOutputShape(), wdwOutOI); tResolver.initializeOutputOI();
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
@Override public void initializeStreaming(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide) throws HiveException { int[] span = setCanAcceptInputAsStream(cfg); if (!canAcceptInputAsStream) { return; } WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i); WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); GenericUDAFEvaluator fnEval = wFnDef.getWFnEval(); GenericUDAFEvaluator streamingEval = fnEval .getWindowingEvaluator(wdwFrame); if (streamingEval != null) { wFnDef.setWFnEval(streamingEval); if (wFnDef.isPivotResult()) { ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI(); wFnDef.setOI(listOI.getListElementObjectInspector()); } } } if ( tabDef.getRankLimit() != -1 ) { rnkLimitDef = new RankLimit(tabDef.getRankLimit(), tabDef.getRankLimitFunction(), tabDef.getWindowFunctions()); } streamingState = new StreamingState(cfg, inputOI, isMapSide, tabDef, span[0], span[1]); }
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { SerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); order = tabDef.getOrder().getExpressions().get(0).getOrder(); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression(); ObjectInspector wFnOI = null; RowResolver inpRR = def.getRawInputShape().getRr();
WindowingIterator(PTFPartition iPart, ArrayList<Object> output, List<?>[] outputFromPivotFunctions, int[] wFnsToProcess) { this.iPart = iPart; this.output = output; this.outputFromPivotFunctions = outputFromPivotFunctions; this.wFnsToProcess = wFnsToProcess; this.currIdx = 0; wTFnDef = (WindowTableFunctionDef) getTableDef(); order = wTFnDef.getOrder().getExpressions().get(0).getOrder(); ptfDesc = getQueryDef(); inputOI = iPart.getOutputOI(); aggBuffers = new AggregationBuffer[wTFnDef.getWindowFunctions().size()]; args = new Object[wTFnDef.getWindowFunctions().size()][]; try { for (int j : wFnsToProcess) { WindowFunctionDef wFn = wTFnDef.getWindowFunctions().get(j); aggBuffers[j] = wFn.getWFnEval().getNewAggregationBuffer(); args[j] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; } } catch (HiveException he) { throw new RuntimeException(he); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
private void setupRankingArgs(WindowTableFunctionDef wdwTFnDef, WindowFunctionDef wFnDef, WindowFunctionSpec wSpec) throws SemanticException { if (wSpec.getArgs().size() > 0) { throw new SemanticException("Ranking Functions can take no arguments"); } OrderDef oDef = wdwTFnDef.getOrder(); List<OrderExpressionDef> oExprs = oDef.getExpressions(); for (OrderExpressionDef oExpr : oExprs) { wFnDef.addArg(oExpr); } }
def.setStar(spec.isStar()); def.setPivotResult(wFnInfo.isPivotResult()); ShapeDetails inpShape = wdwTFnDef.getRawInputShape();
@Override public void initializeStreaming(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide) throws HiveException { int[] span = setCanAcceptInputAsStream(cfg); if (!canAcceptInputAsStream) { return; } WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i); WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); GenericUDAFEvaluator fnEval = wFnDef.getWFnEval(); GenericUDAFEvaluator streamingEval = fnEval .getWindowingEvaluator(wdwFrame); if (streamingEval != null) { wFnDef.setWFnEval(streamingEval); if (wFnDef.isPivotResult()) { ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI(); wFnDef.setOI(listOI.getListElementObjectInspector()); } } } if ( tabDef.getRankLimit() != -1 ) { rnkLimitDef = new RankLimit(tabDef.getRankLimit(), tabDef.getRankLimitFunction(), tabDef.getWindowFunctions()); } streamingState = new StreamingState(cfg, inputOI, isMapSide, tabDef, span[0], span[1]); }
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression(); ObjectInspector wFnOI = null; RowResolver inpRR = def.getRawInputShape().getRr();
Order order = wTFnDef.getOrder().getExpressions().get(0).getOrder(); for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) { boolean processWindow = processWindow(wFn); pItr.reset();
private void setupRankingArgs(WindowTableFunctionDef wdwTFnDef, WindowFunctionDef wFnDef, WindowFunctionSpec wSpec) throws SemanticException { if (wSpec.getArgs().size() > 0) { throw new SemanticException("Ranking Functions can take no arguments"); } OrderDef oDef = wdwTFnDef.getOrder(); List<OrderExpressionDef> oExprs = oDef.getExpressions(); for (OrderExpressionDef oExpr : oExprs) { wFnDef.addArg(oExpr); } }
def.setStar(spec.isStar()); def.setPivotResult(wFnInfo.isPivotResult()); ShapeDetails inpShape = wdwTFnDef.getRawInputShape();
ptfDesc.setCfg(hCfg); ptfDesc.setLlInfo(llInfo); WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef(); ptfDesc.setFuncDef(wdwTFnDef); wdwTFnDef.setInput(translate(inpSpec, 0)); ShapeDetails inpShape = wdwTFnDef.getInput().getOutputShape(); wdwTFnDef.setName(FunctionRegistry.WINDOWING_TABLE_FUNCTION); wdwTFnDef.setResolverClassName(tFn.getClass().getName()); wdwTFnDef.setAlias("ptf_" + 1); wdwTFnDef.setExpressionTreeString(null); wdwTFnDef.setTransformsRawInput(false); tFn.initialize(hCfg, ptfDesc, wdwTFnDef); TableFunctionEvaluator tEval = tFn.getEvaluator(); wdwTFnDef.setTFunction(tEval); wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames()); wdwTFnDef.setRawInputShape(inpShape); wdwTFnDef.setPartition(partDef); wdwTFnDef.setOrder(ordDef); wdwTFnDef.setWindowFunctions(windowFunctions); RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef); ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR); wdwTFnDef.setOutputShape(wdwOutShape);
public void initializeWindowing(WindowTableFunctionDef def) throws HiveException { ShapeDetails inpShape = def.getInput().getOutputShape(); TableFunctionEvaluator tEval = def.getTFunction(); WindowingTableFunctionResolver tResolver = (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName()); tResolver.initialize(ptfDesc, def, tEval); for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { if (wFnDef.getArgs() != null) { for (PTFExpressionDef arg : wFnDef.getArgs()) { for (WindowFunctionDef wFnDef : def.getWindowFunctions()) { aliases.add(wFnDef.getAlias()); if (wFnDef.isPivotResult()) { aliases, fieldOIs); tResolver.setWdwProcessingOutputOI(wdwOutOI); initialize(def.getOutputShape(), wdwOutOI); tResolver.initializeOutputOI();
WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef(); ArrayList<Object> output = new ArrayList<Object>(); List<?>[] outputFromPivotFunctions = new List<?>[wTFnDef.getWindowFunctions().size()]; ArrayList<Integer> wFnsWithWindows = new ArrayList<Integer>(); PTFPartition iPart = pItr.getPartition(); for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) { boolean processWindow = processWindow(wFn.getWindowFrame()); pItr.reset(); if ( wTFnDef.getRankLimit() != -1 ) { rnkLimitDef = new RankLimit(wTFnDef.getRankLimit(), wTFnDef.getRankLimitFunction(), wTFnDef.getWindowFunctions());
if (pDef instanceof WindowTableFunctionDef) { WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; if (tDef.getWindowFunctions() != null) { for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { if (wDef.getArgs() == null) { continue; if (tDef.getPartition() != null) { for (PTFExpressionDef col : tDef.getPartition().getExpressions()) { ExprNodeDesc exprNode = col.getExprNode(); prunedCols = mergeFieldNodesWithDesc(prunedCols, exprNode); if (tDef.getOrder() != null) { for (PTFExpressionDef col : tDef.getOrder().getExpressions()) { ExprNodeDesc exprNode = col.getExprNode(); prunedCols = mergeFieldNodesWithDesc(prunedCols, exprNode);