PartitionedTableFunctionDef def = new PartitionedTableFunctionDef(); def.setInput(inpDef); def.setName(spec.getName()); def.setResolverClassName(tFn.getClass().getName()); def.setAlias(spec.getAlias() == null ? "ptf_" + inpNum : spec.getAlias()); def.setExpressionTreeString(spec.getAstNode().toStringTree()); def.setTransformsRawInput(tFn.transformsRawInput()); def.addArg(argDef); def.setTFunction(tEval); def.setCarryForwardNames(tFn.carryForwardNames()); tFn.setupRawInputOI(); StructObjectInspector rawInOutOI = tEval.getRawInputOI(); List<String> rawInOutColNames = tFn.getRawInputColumnNames(); RowResolver rawInRR = buildRowResolverForPTF(def.getName(), spec.getAlias(), rawInOutOI, rawInOutColNames, inpDef.getOutputShape().getRr()); ShapeDetails rawInpShape = setupTableFnShape(def.getName(), inpDef.getOutputShape(), rawInOutOI, rawInOutColNames, rawInRR); def.setRawInputShape(rawInpShape); def.setRawInputShape(inpDef.getOutputShape());
protected void initialize(PartitionedTableFunctionDef def) throws HiveException { ShapeDetails inpShape = def.getInput().getOutputShape(); if (def.getArgs() != null) { for (PTFExpressionDef arg : def.getArgs()) { initialize(arg, inpShape); TableFunctionEvaluator tEval = def.getTFunction(); TableFunctionResolver tResolver = constructResolver(def.getResolverClassName()); tResolver.initialize(hConf, ptfDesc, def, tEval); initialize(def.getRawInputShape(), tEval.getRawInputOI()); } else { def.setRawInputShape(inpShape); inpShape = def.getRawInputShape(); initialize(def.getOutputShape(), tEval.getOutputOI());
StringBuilder nullOrderString) throws SemanticException { List<PTFExpressionDef> partColList = tabDef.getPartition().getExpressions(); List<OrderExpressionDef> orderColList = tabDef.getOrder().getExpressions(); for (int i = 0; i < orderColList.size(); i++) { OrderExpressionDef colDef = orderColList.get(i);
private PTFInvocation setupChain() { Stack<PartitionedTableFunctionDef> fnDefs = new Stack<PartitionedTableFunctionDef>(); PTFInputDef iDef = conf.getFuncDef(); while (iDef instanceof PartitionedTableFunctionDef) { fnDefs.push((PartitionedTableFunctionDef) iDef); iDef = ((PartitionedTableFunctionDef) iDef).getInput(); } PTFInvocation curr = null, first = null; while(!fnDefs.isEmpty()) { PartitionedTableFunctionDef currFn = fnDefs.pop(); curr = new PTFInvocation(curr, currFn.getTFunction()); if ( first == null ) { first = curr; } } return first; }
private void translatePartitioning(PartitionedTableFunctionDef def, PartitionedTableFunctionSpec spec) throws SemanticException { applyConstantPartition(spec); if (spec.getPartition() == null) { return; } PartitionDef partDef = translate(def.getRawInputShape(), spec.getPartition()); OrderDef orderDef = translate(def.getRawInputShape(), spec.getOrder(), partDef); def.setPartition(partDef); def.setOrder(orderDef); }
public static void alterOutputOIForStreaming(PTFDesc ptfDesc) { PartitionedTableFunctionDef tDef = ptfDesc.getFuncDef(); TableFunctionEvaluator tEval = tDef.getTFunction(); if ( tEval.canIterateOutput() ) { tDef.getOutputShape().setOI(tEval.getOutputOI()); } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
if (tabDef.isTransformsRawInput() ) RowResolver ptfMapRR = tabDef.getRawInputShape().getRr(); RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr(); input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()),
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Override protected void initializeOp(Configuration jobConf) throws HiveException { super.initializeOp(jobConf); hiveConf = jobConf; isMapOperator = conf.isMapSide(); currentKeys = null; reconstructQueryDef(hiveConf); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); outputObjInspector = tDef.getRawInputShape().getOI(); } else { outputObjInspector = conf.getFuncDef().getOutputShape().getOI(); } setupKeysWrapper(inputObjInspectors[0]); ptfInvocation = setupChain(); ptfInvocation.initializeStreaming(jobConf, isMapOperator); firstMapRow = true; }
PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); AbstractSerDe serde = conf.isMapSide() ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = conf.isMapSide() ? tabDef.getInput().getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); inputPart = PTFPartition.create(conf.getCfg(), serde,
private List<ColumnInfo> prunedColumnsList(List<FieldNode> prunedCols, RowSchema oldRS, PartitionedTableFunctionDef pDef) throws SemanticException { pDef.getOutputShape().setRr(null); pDef.getOutputShape().setColumnNames(null); if (pDef instanceof WindowTableFunctionDef) { WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRS)); PTFInputDef input = pDef.getInput(); if (input instanceof PartitionedTableFunctionDef) { return prunedColumnsList(prunedCols, oldRS, (PartitionedTableFunctionDef)input);
public PTFPartition execute(PTFPartition iPart) throws HiveException { if ( ptfDesc.isMapSide() ) { return transformRawInput(iPart); } PTFPartitionIterator<Object> pItr = iPart.iterator(); PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc.getLlInfo(), pItr); if ( outputPartition == null ) { outputPartition = PTFPartition.create(ptfDesc.getCfg(), tableDef.getOutputShape().getSerde(), OI, tableDef.getOutputShape().getOI()); } else { outputPartition.reset(); } execute(pItr, outputPartition); return outputPartition; }
List<String> partitionKeys = new ArrayList<>(); PartitionDef partition = ptfOp.getConf().getFuncDef().getPartition(); if (partition != null && partition.getExpressions() != null) {
protected PTFPartition transformRawInput(PTFPartition iPart) throws HiveException { if (!isTransformsRawInput()) { throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tableDef.getName())); } return _transformRawInput(iPart); }
private void translatePartitioning(PartitionedTableFunctionDef def, PartitionedTableFunctionSpec spec) throws SemanticException { applyConstantPartition(spec); if (spec.getPartition() == null) { return; } PartitionDef partDef = translate(def.getRawInputShape(), spec.getPartition()); OrderDef orderDef = translate(def.getRawInputShape(), spec.getOrder(), partDef); def.setPartition(partDef); def.setOrder(orderDef); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
if (tabDef.isTransformsRawInput() ) RowResolver ptfMapRR = tabDef.getRawInputShape().getRr(); RowResolver ptfOpRR = ptfDesc.getFuncDef().getOutputShape().getRr(); input = putOpInsertMap(OperatorFactory.getAndMakeChild(ptfDesc, new RowSchema(ptfOpRR.getColumnInfos()),