private ArrayList<ExprNodeDesc> getPositionsToExprNodes(List<Integer> pos, List<ColumnInfo> colInfos) { ArrayList<ExprNodeDesc> cols = Lists.newArrayList(); for (Integer idx : pos) { ColumnInfo ci = colInfos.get(idx); ExprNodeColumnDesc encd = new ExprNodeColumnDesc(ci); cols.add(encd); } return cols; }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSink(List<String> keepColNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); int pos = 0; for (int i = 0; i < index.length; i++) { ColumnInfo info = inputOp.getSchema().getSignature().get(i); if (pos < keepColNames.size() && info.getInternalName().equals(keepColNames.get(pos))) { String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, info.getTabAlias(), info.getIsVirtualCol()); columnDescriptors.put(keepColNames.get(pos), desc); pos++; } } return columnDescriptors; }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSinkForJoin(int initialPos, List<String> outputColumnNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp, String tabAlias) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); for (int i = 0; i < index.length; i++) { ColumnInfo info = new ColumnInfo(inputOp.getSchema().getSignature().get(i)); String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, tabAlias, info.getIsVirtualCol()); columnDescriptors.put(outputColumnNames.get(initialPos + i), desc); } return columnDescriptors; }
private void createTestSarg( ObjectInspector inspector, GenericUDF udf, List<ExprNodeDesc> childExpr) { childExpr.add(new ExprNodeColumnDesc(Long.class, "userid", "T", false)); childExpr.add(new ExprNodeConstantDesc(100)); conf.set("hive.io.filter.expr.serialized", SerializationUtilities.serializeExpression( new ExprNodeGenericFuncDesc(inspector, udf, childExpr))); }
private static void addGrpSetCol(boolean createConstantExpr, String grpSetIDExprName, boolean addReducePrefixToColInfoName, List<ExprNodeDesc> exprLst, List<String> outputColumnNames, List<ColumnInfo> colInfoLst, Map<String, ExprNodeDesc> colExprMap) throws SemanticException { String outputColName = null; ExprNodeDesc grpSetColExpr = null; if (createConstantExpr) { grpSetColExpr = new ExprNodeConstantDesc("0L"); } else { grpSetColExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, grpSetIDExprName, null, false); } exprLst.add(grpSetColExpr); outputColName = SemanticAnalyzer.getColumnInternalName(exprLst.size() - 1); outputColumnNames.add(outputColName); String internalColName = outputColName; if (addReducePrefixToColInfoName) { internalColName = Utilities.ReduceField.KEY.toString() + "." + outputColName; } colInfoLst.add(new ColumnInfo(internalColName, grpSetColExpr.getTypeInfo(), null, true)); colExprMap.put(internalColName, grpSetColExpr); }
columns.add("a"); columns.add("b"); columns.add("c"); VectorizationContext vc = new VectorizationContext("name", columns); ExprNodeColumnDesc colDesc1 = new ExprNodeColumnDesc(Long.class, "a", "table", false); ExprNodeColumnDesc colDesc2 = new ExprNodeColumnDesc(Long.class, "b", "table", false); ExprNodeColumnDesc colDesc3 = new ExprNodeColumnDesc(Long.class, "c", "table", false); ExprNodeGenericFuncDesc plusDesc = new ExprNodeGenericFuncDesc(); GenericUDF gudf = new GenericUDFOPPlus(); VectorExpression[] vectorSelectExprs = new VectorExpression[selectColList.size()]; for (int i = 0; i < selectColList.size(); i++) { ExprNodeDesc expr = selectColList.get(i); VectorExpression ve = vc.getVectorExpression(expr); vectorSelectExprs[i] = ve;
Operator<?> handleInsertStatement(String dest, Operator<?> input, RowResolver inputRR, QB qb) throws SemanticException { ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col)); } ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, qb, selExprList); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < colList.size(); i++) { String outputCol = getColumnInternalName(i); colExprMap.put(outputCol, colList.get(i)); columnNames.add(outputCol); } Operator<?> output = putOpInsertMap(OperatorFactory.getAndMakeChild(new SelectDesc(colList, columnNames), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); output.setColumnExprMap(colExprMap); return output; }
@Test public void testAggregateOnUDF() throws HiveException, VectorizerCannotVectorizeException { ExprNodeColumnDesc colExprA = new ExprNodeColumnDesc(Integer.class, "col1", "T", false); ExprNodeColumnDesc colExprB = new ExprNodeColumnDesc(Integer.class, "col2", "T", false); children.add(colExprA); ExprNodeGenericFuncDesc exprNodeDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo, new GenericUDFAbs(), children); params.add(exprNodeDesc); paramOIs.add(exprNodeDesc.getWritableObjectInspector()); outputColumnNames.add("_col0"); aggDescList.add(aggDesc);
@Override public ExprNodeDesc visitInputRef(RexInputRef inputRef) { RelDataTypeField f = inputRowType.getFieldList().get(inputRef.getIndex()); return new ExprNodeColumnDesc(TypeConverter.convert(f.getType()), f.getName(), tabAlias, inputVCols.contains(inputRef.getIndex())); }
args.add(planExpr); } else if (useBucketCols) { for (String col : bucketCols) { ColumnInfo ci = rwsch.get(alias, col); args.add(new ExprNodeColumnDesc(ci)); args.add(genExprNodeDesc(expr, rwsch));
private ArrayList<ExprNodeDesc> getPartitionColsFromBucketColsForUpdateDelete( Operator input, boolean convert) throws SemanticException { //return genConvertCol(dest, qb, tab, table_desc, input, Arrays.asList(0), convert); // In the case of update and delete the bucketing column is always the first column, // and it isn't in the table info. So rather than asking the table for it, // we'll construct it ourself and send it back. This is based on the work done in // genConvertCol below. ColumnInfo rowField = opParseCtx.get(input).getRowResolver().getColumnInfos().get(0); TypeInfo rowFieldTypeInfo = rowField.getType(); ExprNodeDesc column = new ExprNodeColumnDesc(rowFieldTypeInfo, rowField.getInternalName(), rowField.getTabAlias(), true); if (convert) { column = ParseUtils.createConversionCast(column, TypeInfoFactory.intTypeInfo); } ArrayList<ExprNodeDesc> rlist = new ArrayList<ExprNodeDesc>(1); rlist.add(column); return rlist; }
/** * Find ExprNodeDesc for the expression cached in the RowResolver. Returns null if not exists. */ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input) throws SemanticException { ColumnInfo colInfo = input.getExpression(expr); if (colInfo != null) { ASTNode source = input.getExpressionSource(expr); if (source != null) { unparseTranslator.addCopyTranslation(expr, source); } return new ExprNodeColumnDesc(colInfo.getType(), colInfo .getInternalName(), colInfo.getTabAlias(), colInfo .getIsVirtualCol(), colInfo.isSkewedCol()); } return null; }
private Operator genSelectAllDesc(Operator input) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col, true)); columnNames.add(col.getInternalName()); columnExprMap.put(col.getInternalName(), new ExprNodeColumnDesc(col, true)); } RowResolver outputRR = inputRR.duplicate(); Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(colList, columnNames, true), outputRR.getRowSchema(), input), outputRR); output.setColumnExprMap(columnExprMap); return output; }
@Test public void testUnaryMinusColumnLong() throws HiveException { ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false); GenericUDF gudf = new GenericUDFOPNegative(); List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1); children.add(col1Expr); ExprNodeGenericFuncDesc negExprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.longTypeInfo, gudf, children); List<String> columns = new ArrayList<String>(); columns.add("col0"); columns.add("col1"); VectorizationContext vc = new VectorizationContext("name", columns); VectorExpression ve = vc.getVectorExpression(negExprDesc, VectorExpressionDescriptor.Mode.PROJECTION); assertTrue( ve instanceof LongColUnaryMinus); }
private static void addGrpSetCol(boolean createConstantExpr, String grpSetIDExprName, boolean addReducePrefixToColInfoName, List<ExprNodeDesc> exprLst, List<String> outputColumnNames, List<ColumnInfo> colInfoLst, Map<String, ExprNodeDesc> colExprMap) throws SemanticException { String outputColName = null; ExprNodeDesc grpSetColExpr = null; if (createConstantExpr) { grpSetColExpr = new ExprNodeConstantDesc("0"); } else { grpSetColExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, grpSetIDExprName, null, false); } exprLst.add(grpSetColExpr); outputColName = SemanticAnalyzer.getColumnInternalName(exprLst.size() - 1); outputColumnNames.add(outputColName); String internalColName = outputColName; if (addReducePrefixToColInfoName) { internalColName = Utilities.ReduceField.KEY.toString() + "." + outputColName; } colInfoLst.add(new ColumnInfo(internalColName, grpSetColExpr.getTypeInfo(), null, true)); colExprMap.put(internalColName, grpSetColExpr); }
children.add(new ExprNodeColumnDesc(typeInfoStr, "col0", "tablename", false)); children.add(new ExprNodeConstantDesc(typeInfoStr, "UNKNOWN")); argDescs[1].setConstant((ExprNodeConstantDesc) children.get(1)); funcDesc = new ExprNodeGenericFuncDesc(typeInfoStr, genericUDF, "myisnull", children);
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSinkForJoin(int initialPos, List<String> outputColumnNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp, String tabAlias) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); for (int i = 0; i < index.length; i++) { ColumnInfo info = new ColumnInfo(inputOp.getSchema().getSignature().get(i)); String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, tabAlias, info.getIsVirtualCol()); columnDescriptors.put(outputColumnNames.get(initialPos + i), desc); } return columnDescriptors; }
Operator<?> handleInsertStatement(String dest, Operator<?> input, RowResolver inputRR, QB qb) throws SemanticException { ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col)); } ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, inputRR, qb, selExprList); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < colList.size(); i++) { String outputCol = getColumnInternalName(i); colExprMap.put(outputCol, colList.get(i)); columnNames.add(outputCol); } Operator<?> output = putOpInsertMap(OperatorFactory.getAndMakeChild(new SelectDesc(colList, columnNames), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); output.setColumnExprMap(colExprMap); return output; }
private ArrayList<ExprNodeDesc> getPositionsToExprNodes(List<Integer> pos, List<ColumnInfo> colInfos) { ArrayList<ExprNodeDesc> cols = Lists.newArrayList(); for (Integer idx : pos) { ColumnInfo ci = colInfos.get(idx); ExprNodeColumnDesc encd = new ExprNodeColumnDesc(ci); cols.add(encd); } return cols; }
@Override public ExprNodeDesc visitInputRef(RexInputRef inputRef) { RelDataTypeField f = inputRowType.getFieldList().get(inputRef.getIndex()); return new ExprNodeColumnDesc(TypeConverter.convert(f.getType()), f.getName(), tabAlias, inputVCols.contains(inputRef.getIndex())); }