@Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { @SuppressWarnings({ "unchecked" }) List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList(); Boolean retval = new Boolean(checkExpressions(exprs)); if (!retval) { LOG.info("Cannot run select operator [" + n + "] in llap mode"); } return new Boolean(retval); } });
@Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, Object... os) { @SuppressWarnings({ "unchecked" }) List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList(); Boolean retval = new Boolean(checkExpressions(exprs)); if (!retval) { LOG.info("Cannot run select operator [" + n + "] in llap mode"); } return new Boolean(retval); } });
private boolean mergeCardinalityViolationBranch(final Operator input) { if(input instanceof SelectOperator) { SelectOperator selectOp = (SelectOperator)input; if(selectOp.getConf().getColList().size() == 1) { ExprNodeDesc colExpr = selectOp.getConf().getColList().get(0); if(colExpr instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc)colExpr ; if(func.getGenericUDF() instanceof GenericUDFCardinalityViolation){ return true; } } } } return false; }
private boolean validateSelectOperator(SelectOperator op) { List<ExprNodeDesc> descList = op.getConf().getColList(); for (ExprNodeDesc desc : descList) { boolean ret = validateExprNodeDesc( desc, "Select", VectorExpressionDescriptor.Mode.PROJECTION, /* allowComplex */ true, /* allowVoidProjection */ true); if (!ret) { return false; } } return true; }
private boolean validateSelectOperator(SelectOperator op) { List<ExprNodeDesc> descList = op.getConf().getColList(); for (ExprNodeDesc desc : descList) { boolean ret = validateExprNodeDesc(desc, "Select"); if (!ret) { return false; } } return true; }
@Override public Object clone() { SelectDesc ret = new SelectDesc(); ret.setColList(getColList() == null ? null : new ArrayList<ExprNodeDesc>(getColList())); ret.setOutputColumnNames(getOutputColumnNames() == null ? null : new ArrayList<String>(getOutputColumnNames())); ret.setSelectStar(selectStar); ret.setSelStarNoCompute(selStarNoCompute); return ret; }
@Override public Object clone() { SelectDesc ret = new SelectDesc(); ret.setColList(getColList() == null ? null : new ArrayList<ExprNodeDesc>(getColList())); ret.setOutputColumnNames(getOutputColumnNames() == null ? null : new ArrayList<String>(getOutputColumnNames())); ret.setSelectStar(selectStar); ret.setSelStarNoCompute(selStarNoCompute); return ret; }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); // Just forward the row as is if (conf.isSelStarNoCompute()) { isSelectStarNoCompute = true; return; } List<ExprNodeDesc> colList = conf.getColList(); eval = new ExprNodeEvaluator[colList.size()]; for (int i = 0; i < colList.size(); i++) { assert (colList.get(i) != null); eval[i] = ExprNodeEvaluatorFactory.get(colList.get(i), hconf); } if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) { eval = ExprNodeEvaluatorFactory.toCachedEvals(eval); } output = new Object[eval.length]; if (LOG.isInfoEnabled()) { LOG.info("SELECT " + inputObjInspectors[0].getTypeName()); } outputObjInspector = initEvaluatorsAndReturnStruct(eval, conf.getOutputColumnNames(), inputObjInspectors[0]); }
private boolean checkExpressions(SelectOperator op) { SelectDesc desc = op.getConf(); if (desc.isSelectStar() || desc.isSelStarNoCompute()) { return true; } for (ExprNodeDesc expr : desc.getColList()) { if (!checkExpression(expr)) { return false; } } return true; }
ExprNodeDescUtils.backtrack(pSel.getConf().getColList(), pSel, parent); ReduceSinkOperator pRS = (ReduceSinkOperator) parent; for (int i = 0; i < pRS.getConf().getKeyCols().size(); i++) {
private boolean checkExpressions(SelectOperator op) { SelectDesc desc = op.getConf(); if (desc.isSelectStar() || desc.isSelStarNoCompute()) { return true; } for (ExprNodeDesc expr : desc.getColList()) { if (!checkExpression(expr)) { return false; } } return true; }
@Override public void process(Object row, int tag) throws HiveException { if (isSelectStarNoCompute) { forward(row, inputObjInspectors[tag]); return; } int i = 0; try { for (; i < eval.length; ++i) { output[i] = eval[i].evaluate(row); } } catch (HiveException e) { throw e; } catch (RuntimeException e) { throw new HiveException("Error evaluating " + conf.getColList().get(i).getExprString(), e); } forward(output, outputObjInspector); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); // Just forward the row as is if (conf.isSelStarNoCompute()) { return; } List<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>(); List<ExprNodeDesc> colList = conf.getColList(); valueWriters = VectorExpressionWriterFactory.getExpressionWriters(colList); for (VectorExpressionWriter vew : valueWriters) { objectInspectors.add(vew.getObjectInspector()); } List<String> outputFieldNames = conf.getOutputColumnNames(); outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector( outputFieldNames, objectInspectors); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); // Just forward the row as is if (conf.isSelStarNoCompute()) { isSelectStarNoCompute = true; return; } List<ExprNodeDesc> colList = conf.getColList(); eval = new ExprNodeEvaluator[colList.size()]; for (int i = 0; i < colList.size(); i++) { assert (colList.get(i) != null); eval[i] = ExprNodeEvaluatorFactory.get(colList.get(i), hconf); } if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) { eval = ExprNodeEvaluatorFactory.toCachedEvals(eval); } output = new Object[eval.length]; if (isLogInfoEnabled) { LOG.info("SELECT " + inputObjInspectors[0].getTypeName()); } outputObjInspector = initEvaluatorsAndReturnStruct(eval, conf.getOutputColumnNames(), inputObjInspectors[0]); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); // Just forward the row as is if (conf.isSelStarNoCompute()) { return; } VectorExpression.doTransientInit(vExpressions); List<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>(); List<ExprNodeDesc> colList = conf.getColList(); valueWriters = VectorExpressionWriterFactory.getExpressionWriters(colList); for (VectorExpressionWriter vew : valueWriters) { objectInspectors.add(vew.getObjectInspector()); } List<String> outputFieldNames = conf.getOutputColumnNames(); outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector( outputFieldNames, objectInspectors); }
@Override public void process(Object row, int tag) throws HiveException { if (isSelectStarNoCompute) { forward(row, inputObjInspectors[tag]); return; } int i = 0; try { for (; i < eval.length; ++i) { output[i] = eval[i].evaluate(row); } } catch (HiveException e) { throw e; } catch (RuntimeException e) { throw new HiveException("Error evaluating " + conf.getColList().get(i).getExprString(), e); } forward(output, outputObjInspector); }
/** * Creates the list of internal column names (represented by field nodes) * from select expressions in a select operator. This function is used for the * select operator instead of the genColLists function (which is used by * the rest of the operators). * * @param op The select operator. * @return a list of field nodes representing the internal column names. */ public List<FieldNode> getColsFromSelectExpr(SelectOperator op) { List<FieldNode> cols = new ArrayList<>(); SelectDesc conf = op.getConf(); if(conf.isSelStarNoCompute()) { for (ColumnInfo colInfo : op.getSchema().getSignature()) { cols.add(new FieldNode(colInfo.getInternalName())); } } else { List<ExprNodeDesc> exprList = conf.getColList(); for (ExprNodeDesc expr : exprList) { cols = mergeFieldNodesWithDesc(cols, expr); } } return cols; }
static void extractTraits(BucketingSortingCtx bctx, ReduceSinkOperator rop, Operator<?> childop) throws SemanticException { List<ExprNodeDesc> outputValues = Collections.emptyList(); if (childop instanceof SelectOperator) { SelectDesc select = ((SelectOperator)childop).getConf(); outputValues = ExprNodeDescUtils.backtrack(select.getColList(), childop, rop); } if (outputValues.isEmpty()) { return; } // Go through the set of partition columns, and find their representatives in the values // These represent the bucketed columns List<BucketCol> bucketCols = extractBucketCols(rop, outputValues); // Go through the set of key columns, and find their representatives in the values // These represent the sorted columns List<SortCol> sortCols = extractSortCols(rop, outputValues); List<ColumnInfo> colInfos = childop.getSchema().getSignature(); if (!bucketCols.isEmpty()) { List<BucketCol> newBucketCols = getNewBucketCols(bucketCols, colInfos); bctx.setBucketedCols(childop, newBucketCols); } if (!sortCols.isEmpty()) { List<SortCol> newSortCols = getNewSortCols(sortCols, colInfos); bctx.setSortedCols(childop, newSortCols); } }
@Override public void process(Object row, int tag) throws HiveException { // Just forward the row as is if (conf.isSelStarNoCompute()) { vectorForward((VectorizedRowBatch) row); return; } VectorizedRowBatch vrg = (VectorizedRowBatch) row; for (int i = 0; i < vExpressions.length; i++) { try { vExpressions[i].evaluate(vrg); } catch (RuntimeException e) { throw new HiveException("Error evaluating " + conf.getColList().get(i).getExprString(), e); } } // Prepare output, set the projections int[] originalProjections = vrg.projectedColumns; int originalProjectionSize = vrg.projectionSize; vrg.projectionSize = projectedOutputColumns.length; vrg.projectedColumns = this.projectedOutputColumns; vectorForward((VectorizedRowBatch) row); // Revert the projected columns back, because vrg will be re-used. vrg.projectionSize = originalProjectionSize; vrg.projectedColumns = originalProjections; }
/** * Creates the list of internal column names (represented by field nodes) * from select expressions in a select operator. This function is used for the * select operator instead of the genColLists function (which is used by * the rest of the operators). * * @param op The select operator. * @return a list of field nodes representing the internal column names. */ public List<FieldNode> getColsFromSelectExpr(SelectOperator op) { List<FieldNode> cols = new ArrayList<>(); SelectDesc conf = op.getConf(); if(conf.isSelStarNoCompute()) { for (ColumnInfo colInfo : op.getSchema().getSignature()) { cols.add(new FieldNode(colInfo.getInternalName())); } } else { List<ExprNodeDesc> exprList = conf.getColList(); for (ExprNodeDesc expr : exprList) { cols = mergeFieldNodesWithDesc(cols, expr); } } return cols; }