@Explain(displayName = "Function definitions", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public List<PTFInputDef> getFuncDefExplain() { if (funcDef == null) { return null; } List<PTFInputDef> inputs = new ArrayList<PTFInputDef>(); for (PTFInputDef current = funcDef; current != null; current = current.getInput()) { inputs.add(current); } Collections.reverse(inputs); return inputs; }
private ArrayList<FieldNode> prunedInputList(List<FieldNode> prunedCols, PTFInputDef tDef) { ArrayList<FieldNode> prunedInputCols = new ArrayList<>(); StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); FieldNode fn = lookupColumn(prunedCols, fName); if (fn != null) { prunedInputCols.add(fn); } } return prunedInputCols; } }
protected static StructObjectInspector createSelectListOI(MatchPath evaluator, PTFInputDef inpDef) { StructObjectInspector inOI = inpDef.getOutputShape().getOI(); ArrayList<String> inputColumnNames = new ArrayList<String>(); ArrayList<String> selectListNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for(StructField f : inOI.getAllStructFieldRefs()) { String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName()); if ( inputColName != null ) { inputColumnNames.add(inputColName); selectListNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } } StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs)); ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>(); selectFieldOIs.addAll(fieldOIs); selectFieldOIs.add(pathAttrOI); selectListNames.add(MatchPath.PATHATTR_NAME); return ObjectInspectorFactory.getStandardStructObjectInspector( selectListNames, selectFieldOIs); }
@Explain(displayName = "Function definitions", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public List<PTFInputDef> getFuncDefExplain() { if (funcDef == null) { return null; } List<PTFInputDef> inputs = new ArrayList<PTFInputDef>(); for (PTFInputDef current = funcDef; current != null; current = current.getInput()) { inputs.add(current); } Collections.reverse(inputs); return inputs; }
private ArrayList<FieldNode> prunedInputList(List<FieldNode> prunedCols, PTFInputDef tDef) { ArrayList<FieldNode> prunedInputCols = new ArrayList<>(); StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); FieldNode fn = lookupColumn(prunedCols, fName); if (fn != null) { prunedInputCols.add(fn); } } return prunedInputCols; } }
public void initializePTFChain(PartitionedTableFunctionDef tblFnDef) throws HiveException { Deque<PTFInputDef> ptfChain = new ArrayDeque<PTFInputDef>(); PTFInputDef currentDef = tblFnDef; while (currentDef != null) { ptfChain.push(currentDef); currentDef = currentDef.getInput(); } while (!ptfChain.isEmpty()) { currentDef = ptfChain.pop(); if (currentDef instanceof PTFQueryInputDef) { initialize((PTFQueryInputDef) currentDef, inputOI); } else if (currentDef instanceof WindowTableFunctionDef) { initializeWindowing((WindowTableFunctionDef) currentDef); } else { initialize((PartitionedTableFunctionDef) currentDef); } } PTFDeserializer.alterOutputOIForStreaming(ptfDesc); }
protected static StructObjectInspector createSelectListOI(MatchPath evaluator, PTFInputDef inpDef) { StructObjectInspector inOI = inpDef.getOutputShape().getOI(); ArrayList<String> inputColumnNames = new ArrayList<String>(); ArrayList<String> selectListNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for(StructField f : inOI.getAllStructFieldRefs()) { String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName()); if ( inputColName != null ) { inputColumnNames.add(inputColName); selectListNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } } StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs)); ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>(); selectFieldOIs.addAll(fieldOIs); selectFieldOIs.add(pathAttrOI); selectListNames.add(MatchPath.PATHATTR_NAME); return ObjectInspectorFactory.getStandardStructObjectInspector( selectListNames, selectFieldOIs); }
public void initializePTFChain(PartitionedTableFunctionDef tblFnDef) throws HiveException { Deque<PTFInputDef> ptfChain = new ArrayDeque<PTFInputDef>(); PTFInputDef currentDef = tblFnDef; while (currentDef != null) { ptfChain.push(currentDef); currentDef = currentDef.getInput(); } while (!ptfChain.isEmpty()) { currentDef = ptfChain.pop(); if (currentDef instanceof PTFQueryInputDef) { initialize((PTFQueryInputDef) currentDef, inputOI); } else if (currentDef instanceof WindowTableFunctionDef) { initializeWindowing((WindowTableFunctionDef) currentDef); } else { initialize((PartitionedTableFunctionDef) currentDef); } } PTFDeserializer.alterOutputOIForStreaming(ptfDesc); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Explain(displayName = "Function definitions", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public List<PTFInputDef> getFuncDefExplain() { if (funcDef == null) { return null; } List<PTFInputDef> inputs = new ArrayList<PTFInputDef>(); for (PTFInputDef current = funcDef; current != null; current = current.getInput()) { inputs.add(current); } Collections.reverse(inputs); return inputs; }
@Override public void setupRawInputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setRawInputOI(OI); }
public void initializePTFChain(PartitionedTableFunctionDef tblFnDef) throws HiveException { Deque<PTFInputDef> ptfChain = new ArrayDeque<PTFInputDef>(); PTFInputDef currentDef = tblFnDef; while (currentDef != null) { ptfChain.push(currentDef); currentDef = currentDef.getInput(); } while (!ptfChain.isEmpty()) { currentDef = ptfChain.pop(); if (currentDef instanceof PTFQueryInputDef) { initialize((PTFQueryInputDef) currentDef, inputOI); } else if (currentDef instanceof WindowTableFunctionDef) { initializeWindowing((WindowTableFunctionDef) currentDef); } else { initialize((PartitionedTableFunctionDef) currentDef); } } PTFDeserializer.alterOutputOIForStreaming(ptfDesc); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Override public void setupRawInputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setRawInputOI(OI); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@Override public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); }
@SuppressWarnings("unchecked") StreamingState(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide, WindowTableFunctionDef tabDef, int precedingSpan, int followingSpan) throws HiveException { AbstractSerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : tabDef.getRawInputShape().getSerde(); StructObjectInspector outputOI = isMapSide ? tabDef.getInput() .getOutputShape().getOI() : tabDef.getRawInputShape().getOI(); rollingPart = PTFPartition.createRolling(cfg, serde, inputOI, outputOI, precedingSpan, followingSpan); int numFns = tabDef.getWindowFunctions().size(); fnOutputs = new ArrayList[numFns]; aggBuffers = new AggregationBuffer[numFns]; funcArgs = new Object[numFns][]; for (int i = 0; i < numFns; i++) { fnOutputs[i] = new ArrayList<Object>(); WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i); funcArgs[i] = new Object[wFn.getArgs() == null ? 0 : wFn.getArgs().size()]; aggBuffers[i] = wFn.getWFnEval().getNewAggregationBuffer(); } if ( WindowingTableFunction.this.rnkLimitDef != null ) { rnkLimit = new RankLimit(WindowingTableFunction.this.rnkLimitDef); } }
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }
@Override public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException { while (pItr.hasNext()) { Object iRow = pItr.next(); SymbolFunctionResult syFnRes = SymbolFunction.match(syFn, iRow, pItr); if (syFnRes.matches ) { int sz = syFnRes.nextRow - (pItr.getIndex() - 1); Object selectListInput = MatchPath.getSelectListInput(iRow, tableDef.getInput().getOutputShape().getOI(), pItr, sz); ArrayList<Object> oRow = new ArrayList<Object>(); for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals) { oRow.add(resExprEval.evaluate(selectListInput)); } outP.append(oRow); } } }
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }