private PTFInvocation setupChain() { Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>(); PTFInputDef iDef = conf.getFuncDef(); while (iDef instanceof PartitionedTableFunctionDef) { fnDefs.push((PartitionedTableFunctionDef) iDef); iDef = ((PartitionedTableFunctionDef) iDef).getInput(); } PTFInvocation curr = null, first = null; while(!fnDefs.isEmpty()) { PartitionedTableFunctionDef currFn = fnDefs.pop(); curr = new PTFInvocation(curr, currFn.getTFunction()); if ( first == null ) { first = curr; } } return first; }
private PTFInvocation setupChain() { Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>(); PTFInputDef iDef = conf.getFuncDef(); while (iDef instanceof PartitionedTableFunctionDef) { fnDefs.push((PartitionedTableFunctionDef) iDef); iDef = ((PartitionedTableFunctionDef) iDef).getInput(); } PTFInvocation curr = null, first = null; while(!fnDefs.isEmpty()) { PartitionedTableFunctionDef currFn = fnDefs.pop(); curr = new PTFInvocation(curr, currFn.getTFunction()); if ( first == null ) { first = curr; } } return first; }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Override public void setupRawInputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setRawInputOI(OI); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Override public void setupRawInputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setRawInputOI(OI); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
tDef.getInput()); ResultExprInfo resultExprInfo = evaluator.resultExprInfo; ArrayList<ObjectInspector> selectListExprOIs = new ArrayList<ObjectInspector>();
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }
@Override public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException { while (pItr.hasNext()) { Object iRow = pItr.next(); SymbolFunctionResult syFnRes = SymbolFunction.match(syFn, iRow, pItr); if (syFnRes.matches ) { int sz = syFnRes.nextRow - (pItr.getIndex() - 1); Object selectListInput = MatchPath.getSelectListInput(iRow, tableDef.getInput().getOutputShape().getOI(), pItr, sz); ArrayList<Object> oRow = new ArrayList<Object>(); for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals) { oRow.add(resExprEval.evaluate(selectListInput)); } outP.append(oRow); } } }
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }
@Override public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException { while (pItr.hasNext()) { Object iRow = pItr.next(); SymbolFunctionResult syFnRes = SymbolFunction.match(syFn, iRow, pItr); if (syFnRes.matches ) { int sz = syFnRes.nextRow - (pItr.getIndex() - 1); Object selectListInput = MatchPath.getSelectListInput(iRow, tableDef.getInput().getOutputShape().getOI(), pItr, sz); ArrayList<Object> oRow = new ArrayList<Object>(); for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals) { oRow.add(resExprEval.evaluate(selectListInput)); } outP.append(oRow); } } }
PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); AbstractSerDe serde = conf.isMapSide() ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = conf.isMapSide() ? tabDef.getInput().getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); inputPart = PTFPartition.create(conf.getCfg(),
PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); AbstractSerDe serde = conf.isMapSide() ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = conf.isMapSide() ? tabDef.getInput().getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); inputPart = PTFPartition.create(conf.getCfg(),
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
protected void initialize(PartitionedTableFunctionDef def) throws HiveException { ShapeDetails inpShape = def.getInput().getOutputShape();
protected void initialize(PartitionedTableFunctionDef def) throws HiveException { ShapeDetails inpShape = def.getInput().getOutputShape();
PTFInputDef input = pDef.getInput(); if (input instanceof PartitionedTableFunctionDef) { return prunedColumnsList(prunedCols, oldRS, (PartitionedTableFunctionDef)input);