private ShapeDetails copyShape(ShapeDetails src) { ShapeDetails dest = new ShapeDetails(); dest.setSerdeClassName(src.getSerdeClassName()); dest.setSerdeProps(src.getSerdeProps()); dest.setColumnNames(src.getColumnNames()); dest.setOI(src.getOI()); dest.setSerde(src.getSerde()); dest.setRr(src.getRr()); dest.setTypeCheckCtx(src.getTypeCheckCtx()); return dest; }
private ShapeDetails setupShapeForNoop(ShapeDetails inpShape, StructObjectInspector OI, List<String> columnNames, RowResolver rr) throws SemanticException { ShapeDetails shp = new ShapeDetails(); shp.setRr(rr); shp.setOI(inpShape.getOI()); shp.setSerde(inpShape.getSerde()); shp.setSerdeClassName(inpShape.getSerde().getClass().getName()); shp.setSerdeProps(inpShape.getSerdeProps()); shp.setColumnNames(columnNames); TypeCheckCtx tCtx = new TypeCheckCtx(rr); tCtx.setUnparseTranslator(unparseT); shp.setTypeCheckCtx(tCtx); return shp; }
private ShapeDetails setupShape(StructObjectInspector OI, List<String> columnNames, RowResolver rr) throws SemanticException { Map<String, String> serdePropsMap = new LinkedHashMap<String, String>(); AbstractSerDe serde = null; ShapeDetails shp = new ShapeDetails(); try { serde = PTFTranslator.createLazyBinarySerDe(hCfg, OI, serdePropsMap); StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serde, OI); shp.setOI(outOI); } catch (SerDeException se) { throw new SemanticException(se); } shp.setRr(rr); shp.setSerde(serde); shp.setSerdeClassName(serde.getClass().getName()); shp.setSerdeProps(serdePropsMap); shp.setColumnNames(columnNames); TypeCheckCtx tCtx = new TypeCheckCtx(rr); tCtx.setUnparseTranslator(unparseT); shp.setTypeCheckCtx(tCtx); return shp; }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
protected void initialize(ShapeDetails shp, StructObjectInspector OI) throws HiveException { String serdeClassName = shp.getSerdeClassName(); Properties serDeProps = new Properties(); Map<String, String> serdePropsMap = new LinkedHashMap<String, String>(); addOIPropertiestoSerDePropsMap(OI, serdePropsMap); for (String serdeName : serdePropsMap.keySet()) { serDeProps.setProperty(serdeName, serdePropsMap.get(serdeName)); } try { AbstractSerDe serDe = ReflectionUtils.newInstance(hConf.getClassByName(serdeClassName). asSubclass(AbstractSerDe.class), hConf); SerDeUtils.initializeSerDe(serDe, hConf, serDeProps, null); shp.setSerde(serDe); StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serDe, OI); shp.setOI(outOI); } catch (Exception se) { throw new HiveException(se); } }
@Explain(displayName = "output shape") public String getOutputShapeExplain() { RowSchema schema = outputShape.getRr().getRowSchema(); return StringUtils.join(schema.getSignature(), ", "); }
public PTFExpressionDef buildExpressionDef(ShapeDetails inpShape, ASTNode arg) throws HiveException { PTFExpressionDef argDef = new PTFExpressionDef(); ExprNodeDesc exprNode = semAly.genExprNodeDesc(arg, inpShape.getRr(), inpShape.getTypeCheckCtx()); ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(llInfo, exprNode); ObjectInspector oi = initExprNodeEvaluator(exprEval, exprNode, inpShape); argDef.setExpressionTreeString(arg.toStringTree()); argDef.setExprNode(exprNode); argDef.setExprEvaluator(exprEval); argDef.setOI(oi); return argDef; }
private List<ColumnInfo> prunedColumnsList(List<FieldNode> prunedCols, RowSchema oldRS, PartitionedTableFunctionDef pDef) throws SemanticException { pDef.getOutputShape().setRr(null); pDef.getOutputShape().setColumnNames(null); if (pDef instanceof WindowTableFunctionDef) { WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRS)); input.getOutputShape().setRr(buildPrunedRR(inputColumns, oldRS)); input.getOutputShape().setColumnNames(toColumnNames(inputColumns));
public static void alterOutputOIForStreaming(PTFDesc ptfDesc) { PartitionedTableFunctionDef tDef = ptfDesc.getFuncDef(); TableFunctionEvaluator tEval = tDef.getTFunction(); if ( tEval.canIterateOutput() ) { tDef.getOutputShape().setOI(tEval.getOutputOI()); } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx; PTFOperator op = (PTFOperator) nd; PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. PartitionedTableFunctionDef funcDef = conf.getFuncDef(); List<String> referencedColumns = funcDef.getReferencedColumns(); if (!conf.forWindowing() && !conf.forNoop() && referencedColumns == null) { return super.process(nd, stack, cppCtx, nodeOutputs); } List<FieldNode> prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); if (conf.forWindowing()) { WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = mergeFieldNodes(prunedCols, getWindowFunctionColumns(def)); } else if (conf.forNoop()) { prunedCols = new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); } else { prunedCols = fromColumnNames(referencedColumns); } List<ColumnInfo> newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); op.getSchema().setSignature(new ArrayList<ColumnInfo>(newRS)); ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); cppCtx.getPrunedColLists().put(op, fromColumnNames(outputShape.getColumnNames())); return null; }
private ShapeDetails setupShapeForNoop(ShapeDetails inpShape, StructObjectInspector OI, List<String> columnNames, RowResolver rr) throws SemanticException { ShapeDetails shp = new ShapeDetails(); shp.setRr(rr); shp.setOI(inpShape.getOI()); shp.setSerde(inpShape.getSerde()); shp.setSerdeClassName(inpShape.getSerde().getClass().getName()); shp.setSerdeProps(inpShape.getSerdeProps()); shp.setColumnNames(columnNames); TypeCheckCtx tCtx = new TypeCheckCtx(rr); tCtx.setUnparseTranslator(unparseT); shp.setTypeCheckCtx(tCtx); return shp; }
private ShapeDetails setupShape(StructObjectInspector OI, List<String> columnNames, RowResolver rr) throws SemanticException { Map<String, String> serdePropsMap = new LinkedHashMap<String, String>(); AbstractSerDe serde = null; ShapeDetails shp = new ShapeDetails(); try { serde = PTFTranslator.createLazyBinarySerDe(hCfg, OI, serdePropsMap); StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serde, OI); shp.setOI(outOI); } catch (SerDeException se) { throw new SemanticException(se); } shp.setRr(rr); shp.setSerde(serde); shp.setSerdeClassName(serde.getClass().getName()); shp.setSerdeProps(serdePropsMap); shp.setColumnNames(columnNames); TypeCheckCtx tCtx = new TypeCheckCtx(rr); tCtx.setUnparseTranslator(unparseT); shp.setTypeCheckCtx(tCtx); return shp; }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
protected void initialize(ShapeDetails shp, StructObjectInspector OI) throws HiveException { String serdeClassName = shp.getSerdeClassName(); Properties serDeProps = new Properties(); Map<String, String> serdePropsMap = new LinkedHashMap<String, String>(); addOIPropertiestoSerDePropsMap(OI, serdePropsMap); for (String serdeName : serdePropsMap.keySet()) { serDeProps.setProperty(serdeName, serdePropsMap.get(serdeName)); } try { AbstractSerDe serDe = ReflectionUtils.newInstance(hConf.getClassByName(serdeClassName). asSubclass(AbstractSerDe.class), hConf); SerDeUtils.initializeSerDe(serDe, hConf, serDeProps, null); shp.setSerde(serDe); StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serDe, OI); shp.setOI(outOI); } catch (Exception se) { throw new HiveException(se); } }
@Explain(displayName = "output shape") public String getOutputShapeExplain() { RowSchema schema = outputShape.getRr().getRowSchema(); return StringUtils.join(schema.getSignature(), ", "); }
public PTFExpressionDef buildExpressionDef(ShapeDetails inpShape, ASTNode arg) throws HiveException { PTFExpressionDef argDef = new PTFExpressionDef(); ExprNodeDesc exprNode = semAly.genExprNodeDesc(arg, inpShape.getRr(), inpShape.getTypeCheckCtx()); ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(llInfo, exprNode); ObjectInspector oi = initExprNodeEvaluator(exprEval, exprNode, inpShape); argDef.setExpressionTreeString(arg.toStringTree()); argDef.setExprNode(exprNode); argDef.setExprEvaluator(exprEval); argDef.setOI(oi); return argDef; }
private List<ColumnInfo> prunedColumnsList(List<FieldNode> prunedCols, RowSchema oldRS, PartitionedTableFunctionDef pDef) throws SemanticException { pDef.getOutputShape().setRr(null); pDef.getOutputShape().setColumnNames(null); if (pDef instanceof WindowTableFunctionDef) { WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRS)); input.getOutputShape().setRr(buildPrunedRR(inputColumns, oldRS)); input.getOutputShape().setColumnNames(toColumnNames(inputColumns));
public static void alterOutputOIForStreaming(PTFDesc ptfDesc) { PartitionedTableFunctionDef tDef = ptfDesc.getFuncDef(); TableFunctionEvaluator tEval = tDef.getTFunction(); if ( tEval.canIterateOutput() ) { tDef.getOutputShape().setOI(tEval.getOutputOI()); } }