private List<Integer> rankingFunctions(WindowTableFunctionDef wTFn) { List<Integer> rFns = new ArrayList<Integer>(); for(int i=0; i < wTFn.getWindowFunctions().size(); i++ ) { WindowFunctionDef wFnDef = wTFn.getWindowFunctions().get(i); if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) ) { rFns.add(i); } } return rFns; }
private List<FieldNode> getWindowFunctionColumns(WindowTableFunctionDef tDef) { List<FieldNode> columns = new ArrayList<>(); if (tDef.getWindowFunctions() != null) { for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { columns.add(new FieldNode(wDef.getAlias())); } } return columns; }
private List<Integer> rankingFunctions(WindowTableFunctionDef wTFn) { List<Integer> rFns = new ArrayList<Integer>(); for(int i=0; i < wTFn.getWindowFunctions().size(); i++ ) { WindowFunctionDef wFnDef = wTFn.getWindowFunctions().get(i); if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) ) { rFns.add(i); } } return rFns; }
private List<FieldNode> getWindowFunctionColumns(WindowTableFunctionDef tDef) { List<FieldNode> columns = new ArrayList<>(); if (tDef.getWindowFunctions() != null) { for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { columns.add(new FieldNode(wDef.getAlias())); } } return columns; }
void reset(WindowTableFunctionDef tabDef) throws HiveException { int numFns = tabDef.getWindowFunctions().size(); rollingPart.reset(); for (int i = 0; i < fnOutputs.length; i++) { fnOutputs[i].clear(); } for (int i = 0; i < numFns; i++) { WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( rnkLimit != null ) { rnkLimit.reset(); } }
void reset(WindowTableFunctionDef tabDef) throws HiveException { int numFns = tabDef.getWindowFunctions().size(); rollingPart.reset(); for (int i = 0; i < fnOutputs.length; i++) { fnOutputs[i].clear(); } for (int i = 0; i < numFns; i++) { WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( rnkLimit != null ) { rnkLimit.reset(); } }
private void initializeWindowingFunctionInfoHelpers() throws SemanticException { // getWindowFunctionInfo() cannot be called during map/reduce tasks. So cache necessary // values during query compilation, and rely on plan serialization to bring this info // to the object during the map/reduce tasks. if (windowingFunctionHelpers != null) { return; } windowingFunctionHelpers = new HashMap<String, WindowingFunctionInfoHelper>(); WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getName()); boolean supportsWindow = wFnInfo.isSupportsWindow(); windowingFunctionHelpers.put(wFn.getName(), new WindowingFunctionInfoHelper(supportsWindow)); } }
private void initializeWindowingFunctionInfoHelpers() throws SemanticException { // getWindowFunctionInfo() cannot be called during map/reduce tasks. So cache necessary // values during query compilation, and rely on plan serialization to bring this info // to the object during the map/reduce tasks. if (windowingFunctionHelpers != null) { return; } windowingFunctionHelpers = new HashMap<String, WindowingFunctionInfoHelper>(); WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getName()); boolean supportsWindow = wFnInfo.isSupportsWindow(); windowingFunctionHelpers.put(wFn.getName(), new WindowingFunctionInfoHelper(supportsWindow)); } }
WindowingIterator(PTFPartition iPart, ArrayList<Object> output, List<?>[] outputFromPivotFunctions, int[] wFnsToProcess) { this.iPart = iPart; this.output = output; this.outputFromPivotFunctions = outputFromPivotFunctions; this.wFnsToProcess = wFnsToProcess; this.currIdx = 0; wTFnDef = (WindowTableFunctionDef) getTableDef(); ptfDesc = getQueryDef(); inputOI = iPart.getOutputOI(); aggBuffers = new AggregationBuffer[wTFnDef.getWindowFunctions().size()]; args = new Object[wTFnDef.getWindowFunctions().size()][]; try { for (int j : wFnsToProcess) { WindowFunctionDef wFn = wTFnDef.getWindowFunctions().get(j); aggBuffers[j] = wFn.getWFnEval().getNewAggregationBuffer(); args[j] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; } } catch (HiveException he) { throw new RuntimeException(he); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
WindowingIterator(PTFPartition iPart, ArrayList<Object> output, List<?>[] outputFromPivotFunctions, int[] wFnsToProcess) { this.iPart = iPart; this.output = output; this.outputFromPivotFunctions = outputFromPivotFunctions; this.wFnsToProcess = wFnsToProcess; this.currIdx = 0; wTFnDef = (WindowTableFunctionDef) getTableDef(); ptfDesc = getQueryDef(); inputOI = iPart.getOutputOI(); aggBuffers = new AggregationBuffer[wTFnDef.getWindowFunctions().size()]; args = new Object[wTFnDef.getWindowFunctions().size()][]; try { for (int j : wFnsToProcess) { WindowFunctionDef wFn = wTFnDef.getWindowFunctions().get(j); aggBuffers[j] = wFn.getWFnEval().getNewAggregationBuffer(); args[j] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; } } catch (HiveException he) { throw new RuntimeException(he); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
String fAlias = wTFn.getWindowFunctions().get(i).getAlias(); if ( fAlias.equals(colName)) { return new int[] {limit,i};
@Override public void initializeStreaming(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide) throws HiveException { int[] span = setCanAcceptInputAsStream(cfg); if (!canAcceptInputAsStream) { return; } WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i); WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); GenericUDAFEvaluator fnEval = wFnDef.getWFnEval(); GenericUDAFEvaluator streamingEval = fnEval .getWindowingEvaluator(wdwFrame); if (streamingEval != null) { wFnDef.setWFnEval(streamingEval); if (wFnDef.isPivotResult()) { ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI(); wFnDef.setOI(listOI.getListElementObjectInspector()); } } } if ( tabDef.getRankLimit() != -1 ) { rnkLimitDef = new RankLimit(tabDef.getRankLimit(), tabDef.getRankLimitFunction(), tabDef.getWindowFunctions()); } streamingState = new StreamingState(cfg, inputOI, isMapSide, tabDef, span[0], span[1]); }
for(WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) { boolean processWindow = processWindow(wFn.getWindowFrame()); pItr.reset();
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
@Override public void initializeStreaming(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide) throws HiveException { int[] span = setCanAcceptInputAsStream(cfg); if (!canAcceptInputAsStream) { return; } WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef(); for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i); WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); GenericUDAFEvaluator fnEval = wFnDef.getWFnEval(); GenericUDAFEvaluator streamingEval = fnEval .getWindowingEvaluator(wdwFrame); if (streamingEval != null) { wFnDef.setWFnEval(streamingEval); if (wFnDef.isPivotResult()) { ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI(); wFnDef.setOI(listOI.getListElementObjectInspector()); } } } if ( tabDef.getRankLimit() != -1 ) { rnkLimitDef = new RankLimit(tabDef.getRankLimit(), tabDef.getRankLimitFunction(), tabDef.getWindowFunctions()); } streamingState = new StreamingState(cfg, inputOI, isMapSide, tabDef, span[0], span[1]); }
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
int endPos = Integer.MIN_VALUE; for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) { WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i); WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); GenericUDAFEvaluator fnEval = wFnDef.getWFnEval();
private boolean canPushLimitToReduceSink(WindowTableFunctionDef wTFn) { for(WindowFunctionDef wFnDef : wTFn.getWindowFunctions() ) { if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) || (wFnDef.getWFnEval() instanceof GenericUDAFLeadEvaluator ) ) { continue; } WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); BoundaryDef end = wdwFrame.getEnd(); if (wdwFrame.getWindowType() == WindowType.RANGE) { return false; } if ( end.getDirection() == Direction.FOLLOWING ) { return false; } } return true; }
private boolean canPushLimitToReduceSink(WindowTableFunctionDef wTFn) { for(WindowFunctionDef wFnDef : wTFn.getWindowFunctions() ) { if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) || (wFnDef.getWFnEval() instanceof GenericUDAFLeadEvaluator ) ) { continue; } WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); BoundaryDef end = wdwFrame.getEnd(); if (wdwFrame.getWindowType() == WindowType.RANGE) { return false; } if ( end.getDirection() == Direction.FOLLOWING ) { return false; } } return true; }
if (pDef instanceof WindowTableFunctionDef) { WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; if (tDef.getWindowFunctions() != null) { for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { if (wDef.getArgs() == null) { continue;