void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }
protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException { PartitionDef pDef = conf.getStartOfChain().getPartition(); List<PTFExpressionDef> exprs = pDef.getExpressions(); int numExprs = exprs.size(); ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs]; ObjectInspector[] keyOIs = new ObjectInspector[numExprs]; ObjectInspector[] currentKeyOIs = new ObjectInspector[numExprs]; for(int i=0; i<numExprs; i++) { PTFExpressionDef exprDef = exprs.get(i); /* * Why cannot we just use the ExprNodeEvaluator on the column? * - because on the reduce-side it is initialized based on the rowOI of the HiveTable * and not the OI of the parent of this Operator on the reduce-side */ keyFields[i] = ExprNodeEvaluatorFactory.get(exprDef.getExprNode()); keyOIs[i] = keyFields[i].initialize(inputOI); currentKeyOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(keyOIs[i], ObjectInspectorCopyOption.WRITABLE); } keyWrapperFactory = new KeyWrapperFactory(keyFields, keyOIs, currentKeyOIs); newKeys = keyWrapperFactory.getKeyWrapper(); }
protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException { PartitionDef pDef = conf.getStartOfChain().getPartition(); List<PTFExpressionDef> exprs = pDef.getExpressions(); int numExprs = exprs.size(); ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs]; ObjectInspector[] keyOIs = new ObjectInspector[numExprs]; ObjectInspector[] currentKeyOIs = new ObjectInspector[numExprs]; for(int i=0; i<numExprs; i++) { PTFExpressionDef exprDef = exprs.get(i); /* * Why cannot we just use the ExprNodeEvaluator on the column? * - because on the reduce-side it is initialized based on the rowOI of the HiveTable * and not the OI of the parent of this Operator on the reduce-side */ keyFields[i] = ExprNodeEvaluatorFactory.get(exprDef.getExprNode()); keyOIs[i] = keyFields[i].initialize(inputOI); currentKeyOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(keyOIs[i], ObjectInspectorCopyOption.WRITABLE); } keyWrapperFactory = new KeyWrapperFactory(keyFields, keyOIs, currentKeyOIs); newKeys = keyWrapperFactory.getKeyWrapper(); }
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }
PartitionedTableFunctionDef tabDef = ptfDesc.getStartOfChain();
PartitionedTableFunctionDef tabDef = ptfDesc.getStartOfChain();
@Override protected void initializeOp(Configuration jobConf) throws HiveException { super.initializeOp(jobConf); hiveConf = jobConf; isMapOperator = conf.isMapSide(); currentKeys = null; reconstructQueryDef(hiveConf); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); outputObjInspector = tDef.getRawInputShape().getOI(); } else { outputObjInspector = conf.getFuncDef().getOutputShape().getOI(); } setupKeysWrapper(inputObjInspectors[0]); ptfInvocation = setupChain(); ptfInvocation.initializeStreaming(jobConf, isMapOperator); firstMapRow = true; }
@Override protected void initializeOp(Configuration jobConf) throws HiveException { super.initializeOp(jobConf); hiveConf = jobConf; isMapOperator = conf.isMapSide(); currentKeys = null; reconstructQueryDef(hiveConf); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); outputObjInspector = tDef.getRawInputShape().getOI(); } else { outputObjInspector = conf.getFuncDef().getOutputShape().getOI(); } setupKeysWrapper(inputObjInspectors[0]); ptfInvocation = setupChain(); ptfInvocation.initializeStreaming(jobConf, isMapOperator); firstMapRow = true; }
PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI();
PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI();
void initializeStreaming(Configuration cfg, boolean isMapSide) throws HiveException { PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI(); tabFn.initializeStreaming(cfg, (StructObjectInspector) inputOI, isMapSide); if ( next != null ) { next.initializeStreaming(cfg, isMapSide); } }
protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException { PartitionDef pDef = conf.getStartOfChain().getPartition(); List<PTFExpressionDef> exprs = pDef.getExpressions(); int numExprs = exprs.size(); ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs]; ObjectInspector[] keyOIs = new ObjectInspector[numExprs]; ObjectInspector[] currentKeyOIs = new ObjectInspector[numExprs]; for(int i=0; i<numExprs; i++) { PTFExpressionDef exprDef = exprs.get(i); /* * Why cannot we just use the ExprNodeEvaluator on the column? * - because on the reduce-side it is initialized based on the rowOI of the HiveTable * and not the OI of the parent of this Operator on the reduce-side */ keyFields[i] = ExprNodeEvaluatorFactory.get(exprDef.getExprNode()); keyOIs[i] = keyFields[i].initialize(inputOI); currentKeyOIs[i] = ObjectInspectorUtils.getStandardObjectInspector(keyOIs[i], ObjectInspectorCopyOption.WRITABLE); } keyWrapperFactory = new KeyWrapperFactory(keyFields, keyOIs, currentKeyOIs); newKeys = keyWrapperFactory.getKeyWrapper(); }
PartitionedTableFunctionDef tabDef = ptfDesc.getStartOfChain();
@Override protected Collection<Future<?>> initializeOp(Configuration jobConf) throws HiveException { Collection<Future<?>> result = super.initializeOp(jobConf); hiveConf = jobConf; isMapOperator = conf.isMapSide(); reconstructQueryDef(hiveConf); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); outputObjInspector = tDef.getRawInputShape().getOI(); } else { outputObjInspector = conf.getFuncDef().getOutputShape().getOI(); } setupKeysWrapper(inputObjInspectors[0]); ptfInvocation = setupChain(); ptfInvocation.initializeStreaming(jobConf, isMapOperator); firstMapRow = true; return result; }
PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); PTFInputDef inputDef = tabDef.getInput(); ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? inputObjInspectors[0] : inputDef.getOutputShape().getOI();