@Explain(displayName = "window frame") public String getWindowFrameExplain() { return windowFrame == null ? null : windowFrame.toString(); }
public static ValueBoundaryScanner getScanner(WindowFrameDef winFrameDef, boolean nullsLast) throws HiveException { OrderDef orderDef = winFrameDef.getOrderDef(); int numOrders = orderDef.getExpressions().size(); if (numOrders != 1) { return new MultiValueBoundaryScanner(winFrameDef.getStart(), winFrameDef.getEnd(), orderDef, nullsLast); } else { return SingleValueBoundaryScanner.getScanner(winFrameDef.getStart(), winFrameDef.getEnd(), orderDef, nullsLast); } } }
public State(AggregationBuffer buf) { super(buf); maxChain = new ArrayDeque<Object[]>(wFrameDef.isStartUnbounded() ? 1 : wFrameDef.getWindowSize()); }
protected void initialize(WindowFrameDef winFrame, ShapeDetails inpShape) throws HiveException { if (winFrame.getWindowType() == WindowType.RANGE) { for (OrderExpressionDef exprDef : winFrame.getOrderDef().getExpressions()) { initialize(exprDef, inpShape); } } }
@Override public Object terminate(AggregationBuffer agg) throws HiveException { State s = (State) agg; Object[] r = s.maxChain.isEmpty() ? null : s.maxChain.getFirst(); // After all the rows are processed, continue to generate results for the rows that results haven't generated. // For the case: X following and Y following, process first Y-X results and then insert X nulls. // For the case X preceding and Y following, process Y results. for (int i = Math.max(0, wFrameDef.getStart().getRelativeOffset()); i < wFrameDef.getEnd().getRelativeOffset(); i++) { if (s.hasResultReady()) { s.results.add(r == null ? null : r[0]); } s.numRows++; if (r != null) { int fIdx = (Integer) r[1]; if (!wFrameDef.isStartUnbounded() && s.numRows >= fIdx + wFrameDef.getWindowSize() && !s.maxChain.isEmpty()) { s.maxChain.removeFirst(); r = !s.maxChain.isEmpty() ? s.maxChain.getFirst() : null; } } } for (int i = 0; i < wFrameDef.getStart().getRelativeOffset(); i++) { if (s.hasResultReady()) { s.results.add(null); } s.numRows++; } return null; }
protected static Range getRange(WindowFrameDef winFrame, int currRow, PTFPartition p, boolean nullsLast) throws HiveException { BoundaryDef startB = winFrame.getStart(); BoundaryDef endB = winFrame.getEnd(); int start, end; if (winFrame.getWindowType() == WindowType.ROWS) { start = getRowBoundaryStart(startB, currRow); end = getRowBoundaryEnd(endB, currRow, p); } else { ValueBoundaryScanner vbs = ValueBoundaryScanner.getScanner(winFrame, nullsLast); start = vbs.computeStart(currRow, p); end = vbs.computeEnd(currRow, p); } start = start < 0 ? 0 : start; end = end > p.size() ? p.size() : end; return new Range(start, end, p); }
for (int i = wFrameDef.getEnd().getRelativeOffset(); i < 0; i++) { s.results.add(null); s.lastIdx = s.numRows; } else if (lb.skipNulls && s.lastIdx != -1) { if (!wFrameDef.isStartUnbounded() && s.numRows >= s.lastIdx + wFrameDef.getWindowSize()) { s.lastValue = null; s.lastIdx = -1; if (s.numRows >= wFrameDef.getEnd().getRelativeOffset()) { s.results.add(s.lastValue);
@Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { SumAvgStreamingState ss = (SumAvgStreamingState) agg; wrappedEval.iterate(ss.wrappedBuf, parameters); // We need to insert 'null' before processing first row for the case: X preceding and y preceding if (ss.numRows == 0) { for (int i = wFrameDef.getEnd().getRelativeOffset(); i < 0; i++) { ss.results.add(null); } } // Generate the result for the windowing ending at the current row if (ss.hasResultReady()) { ss.results.add(getNextResult(ss)); } if (!wFrameDef.isStartUnbounded() && ss.numRows + 1 >= wFrameDef.getStart().getRelativeOffset()) { ss.intermediateVals.add(getCurrentIntermediateResult(ss)); } ss.numRows++; }
private WindowFrameDef translate(ShapeDetails inpShape, WindowFrameSpec spec, List<OrderExpression> orderExpressions) throws SemanticException { if (spec == null) { return null; } BoundarySpec s = spec.getStart(); BoundarySpec e = spec.getEnd(); int cmp = s.compareTo(e); if (cmp > 0) { throw new SemanticException(String.format( "Window range invalid, start boundary is greater than end boundary: %s", spec)); } WindowFrameDef winFrame = new WindowFrameDef( spec.getWindowType(), new BoundaryDef(s.direction, s.getAmt()), new BoundaryDef(e.direction, e.getAmt())); if (winFrame.getWindowType() == WindowType.RANGE) { winFrame.setOrderDef(buildOrderExpressions(inpShape, orderExpressions)); } return winFrame; }
if (!windowFrameDef.isStartUnbounded()) { setOperatorIssue(functionName + " only UNBOUNDED start frame is supported"); return false; ((PrimitiveTypeInfo) singleTypeInfo).getPrimitiveCategory() : null); switch (windowFrameDef.getWindowType()) { case RANGE: if (!windowFrameDef.getEnd().isCurrentRow()) { setOperatorIssue(functionName + " only CURRENT ROW end frame is supported for RANGE"); return false; (windowFrameDef.getEnd().isCurrentRow() && (supportedFunctionType == SupportedFunctionType.AVG || supportedFunctionType == SupportedFunctionType.MAX || isSingleParameter && singlePrimitiveCategory != null); if (!isRowEndCurrent && !windowFrameDef.isEndUnbounded()) { setOperatorIssue( functionName + " UNBOUNDED end frame is required for ROWS window type"); throw new RuntimeException("Unexpected window type " + windowFrameDef.getWindowType());
private boolean canPushLimitToReduceSink(WindowTableFunctionDef wTFn) { for(WindowFunctionDef wFnDef : wTFn.getWindowFunctions() ) { if ( (wFnDef.getWFnEval() instanceof GenericUDAFRankEvaluator) || (wFnDef.getWFnEval() instanceof GenericUDAFDenseRankEvaluator ) || (wFnDef.getWFnEval() instanceof GenericUDAFLeadEvaluator ) ) { continue; } WindowFrameDef wdwFrame = wFnDef.getWindowFrame(); BoundaryDef end = wdwFrame.getEnd(); if (wdwFrame.getWindowType() == WindowType.RANGE) { return false; } if ( end.getDirection() == Direction.FOLLOWING ) { return false; } } return true; }
/** * Gets the next row index that the data within the window are available and can be processed * @param wFrameDef * @return */ public int rowToProcess(WindowFrameDef wFrameDef) { int rowToProcess = numRowsReceived - 1 - Math.max(0, wFrameDef.getEnd().getRelativeOffset()); return rowToProcess >= 0 ? rowToProcess : -1; }
private WindowFrameDef translate(ShapeDetails inpShape, WindowFrameSpec spec) throws SemanticException { if (spec == null) { return null; } BoundarySpec s = spec.getStart(); BoundarySpec e = spec.getEnd(); int cmp = s.compareTo(e); if (cmp > 0) { throw new SemanticException(String.format( "Window range invalid, start boundary is greater than end boundary: %s", spec)); } WindowFrameDef wfDef = new WindowFrameDef(); wfDef.setStart(translate(inpShape, s)); wfDef.setEnd(translate(inpShape, e)); return wfDef; }
public static WindowFrameDef wdwFrame(int p, int f) { BoundaryDef start, end; if (p == 0) { start = new BoundaryDef(Direction.CURRENT, 0); } else { start = new BoundaryDef(Direction.PRECEDING, p); } if (f == 0) { end = new BoundaryDef(Direction.CURRENT, 0); } else { end = new BoundaryDef(Direction.FOLLOWING, f); } return new WindowFrameDef(WindowType.ROWS, start, end); }
@Override public Object terminate(AggregationBuffer agg) throws HiveException { State s = (State) agg; LastValueBuffer lb = (LastValueBuffer) s.wrappedBuf; if (lb.skipNulls && s.lastIdx != -1) { if (!wFrameDef.isStartUnbounded() && s.numRows >= s.lastIdx + wFrameDef.getWindowSize()) { s.lastValue = null; s.lastIdx = -1; } } // After all the rows are processed, continue to generate results for the rows that results haven't generated. // For the case: X following and Y following, process first Y-X results and then insert X nulls. // For the case X preceding and Y following, process Y results. for (int i = Math.max(0, wFrameDef.getStart().getRelativeOffset()); i < wFrameDef.getEnd().getRelativeOffset(); i++) { if (s.hasResultReady()) { s.results.add(s.lastValue); } s.numRows++; } for (int i = 0; i < wFrameDef.getStart().getRelativeOffset(); i++) { if (s.hasResultReady()) { s.results.add(null); } s.numRows++; } return null; }
protected static Range getRange(WindowFrameDef winFrame, int currRow, PTFPartition p) throws HiveException { BoundaryDef startB = winFrame.getStart(); BoundaryDef endB = winFrame.getEnd(); int start, end; if (winFrame.getWindowType() == WindowType.ROWS) { start = getRowBoundaryStart(startB, currRow); end = getRowBoundaryEnd(endB, currRow, p); } else { ValueBoundaryScanner vbs = ValueBoundaryScanner.getScanner(winFrame); start = vbs.computeStart(currRow, p); end = vbs.computeEnd(currRow, p); } start = start < 0 ? 0 : start; end = end > p.size() ? p.size() : end; return new Range(start, end, p); }
for (int i = wFrameDef.getEnd().getRelativeOffset(); i < 0; i++) { s.results.add(null); if (!wFrameDef.isStartUnbounded() || s.maxChain.isEmpty()) { o = o == null ? null : ObjectInspectorUtils.copyToStandardObject(o, inputOI(), ObjectInspectorCopyOption.JAVA); if (!wFrameDef.isStartUnbounded() && s.numRows >= fIdx + wFrameDef.getWindowSize()) { s.maxChain.removeFirst();
public State(AggregationBuffer buf) { super(buf); valueChain = new ArrayDeque<ValIndexPair>(wFrameDef.isStartUnbounded() ? 1 : wFrameDef.getWindowSize()); }
@Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { SumAvgStreamingState ss = (SumAvgStreamingState) agg; wrappedEval.iterate(ss.wrappedBuf, parameters); // We need to insert 'null' before processing first row for the case: X preceding and y preceding if (ss.numRows == 0) { for (int i = wFrameDef.getEnd().getRelativeOffset(); i < 0; i++) { ss.results.add(null); } } // Generate the result for the windowing ending at the current row if (ss.hasResultReady()) { ss.results.add(getNextResult(ss)); } if (!wFrameDef.isStartUnbounded() && ss.numRows + 1 >= wFrameDef.getStart().getRelativeOffset()) { ss.intermediateVals.add(getCurrentIntermediateResult(ss)); } ss.numRows++; }