private Operator genSelectAllDesc(Operator input) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col, true)); columnNames.add(col.getInternalName()); columnExprMap.put(col.getInternalName(), new ExprNodeColumnDesc(col, true)); } RowResolver outputRR = inputRR.duplicate(); Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(colList, columnNames, true), outputRR.getRowSchema(), input), outputRR); output.setColumnExprMap(columnExprMap); return output; }
Operator<?> handleInsertStatement(String dest, Operator<?> input, RowResolver inputRR, QB qb) throws SemanticException { ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col)); } ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, qb, selExprList); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < colList.size(); i++) { String outputCol = getColumnInternalName(i); colExprMap.put(outputCol, colList.get(i)); columnNames.add(outputCol); } Operator<?> output = putOpInsertMap(OperatorFactory.getAndMakeChild(new SelectDesc(colList, columnNames), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); output.setColumnExprMap(colExprMap); return output; }
private Operator genSelectAllDesc(Operator input) throws SemanticException { OpParseContext inputCtx = opParseCtx.get(input); RowResolver inputRR = inputCtx.getRowResolver(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col, true)); columnNames.add(col.getInternalName()); columnExprMap.put(col.getInternalName(), new ExprNodeColumnDesc(col, true)); } RowResolver outputRR = inputRR.duplicate(); Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new SelectDesc(colList, columnNames, true), outputRR.getRowSchema(), input), outputRR); output.setColumnExprMap(columnExprMap); return output; }
Operator<?> handleInsertStatement(String dest, Operator<?> input, RowResolver inputRR, QB qb) throws SemanticException { ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>(); ArrayList<ColumnInfo> columns = inputRR.getColumnInfos(); for (int i = 0; i < columns.size(); i++) { ColumnInfo col = columns.get(i); colList.add(new ExprNodeColumnDesc(col)); } ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, inputRR, qb, selExprList); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); for (int i = 0; i < colList.size(); i++) { String outputCol = getColumnInternalName(i); colExprMap.put(outputCol, colList.get(i)); columnNames.add(outputCol); } Operator<?> output = putOpInsertMap(OperatorFactory.getAndMakeChild(new SelectDesc(colList, columnNames), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch); output.setColumnExprMap(colExprMap); return output; }
private SelectOperator buildSelectOperator( List<ExprNodeDesc> colList, List<FieldNode> outputCols) { SelectOperator selectOperator = mock(SelectOperator.class); SelectDesc selectDesc = new SelectDesc(colList, ColumnPrunerProcCtx.toColumnNames(outputCols)); selectDesc.setSelStarNoCompute(false); when(selectOperator.getConf()).thenReturn(selectDesc); return selectOperator; } }
@Override public Object clone() { SelectDesc ret = new SelectDesc(); ret.setColList(getColList() == null ? null : new ArrayList<ExprNodeDesc>(getColList())); ret.setOutputColumnNames(getOutputColumnNames() == null ? null : new ArrayList<String>(getOutputColumnNames())); ret.setSelectStar(selectStar); ret.setSelStarNoCompute(selStarNoCompute); return ret; }
private Operator<? extends OperatorDesc> genOutputSelectForGroupBy( Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) { assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size()); Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator(); Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator(); List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>(); List<String> colName = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); while (pIter.hasNext()) { ColumnInfo pInfo = pIter.next(); ColumnInfo cInfo = cIter.next(); ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(), pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol()); columns.add(column); colName.add(cInfo.getInternalName()); columnExprMap.put(cInfo.getInternalName(), column); } return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp.getSchema().getSignature()), columnExprMap, parentOp); } }
@Override public Object clone() { SelectDesc ret = new SelectDesc(); ret.setColList(getColList() == null ? null : new ArrayList<ExprNodeDesc>(getColList())); ret.setOutputColumnNames(getOutputColumnNames() == null ? null : new ArrayList<String>(getOutputColumnNames())); ret.setSelectStar(selectStar); ret.setSelStarNoCompute(selStarNoCompute); return ret; }
private Operator<? extends OperatorDesc> genOutputSelectForGroupBy( Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) { assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size()); Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator(); Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator(); List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>(); List<String> colName = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); while (pIter.hasNext()) { ColumnInfo pInfo = pIter.next(); ColumnInfo cInfo = cIter.next(); ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(), pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol()); columns.add(column); colName.add(cInfo.getInternalName()); columnExprMap.put(cInfo.getInternalName(), column); } return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp.getSchema().getSignature()), columnExprMap, parentOp); } }
return OperatorFactory.getAndMakeChild(new SelectDesc( columns, colName), new RowSchema(uColumnInfo), columnExprMap, origInputOp); } else {
SelectDesc select = new SelectDesc(exprs, outputs, false);
SelectDesc selectDesc = new SelectDesc(selectExprList, selectOutputColumnNameList); Operator<SelectDesc> selectOperator = OperatorFactory.get(new CompilationOpContext(), selectDesc);
@SuppressWarnings("unchecked") private void populateMapRedPlan1(Table src) throws SemanticException { ArrayList<String> outputColumns = new ArrayList<String>(); for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); } // map-side work Operator<ReduceSinkDesc> op1 = OperatorFactory.get(ctx, PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), Utilities.makeList(getStringColumn("value")), outputColumns, true, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); addMapWork(mr, src, "a", op1); ReduceWork rWork = new ReduceWork(); rWork.setNumReduceTasks(Integer.valueOf(1)); rWork.setKeyDesc(op1.getConf().getKeySerializeInfo()); rWork.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); mr.setReduceWork(rWork); // reduce side work Operator<FileSinkDesc> op3 = OperatorFactory.get(ctx, new FileSinkDesc(new Path(tmpdir + File.separator + "mapredplan1.out"), Utilities.defaultTd, false)); List<ExprNodeDesc> cols = new ArrayList<ExprNodeDesc>(); cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); List<String> colNames = new ArrayList<String>(); colNames.add(HiveConf.getColumnInternalName(2)); Operator<SelectDesc> op2 = OperatorFactory.get(new SelectDesc(cols, colNames), op3); rWork.setReducer(op2); }
@SuppressWarnings("unchecked") private void populateMapRedPlan5(Table src) throws SemanticException { // map-side work ArrayList<String> outputColumns = new ArrayList<String>(); for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); } Operator<ReduceSinkDesc> op0 = OperatorFactory.get(ctx, PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("0")), Utilities .makeList(getStringColumn("0"), getStringColumn("1")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); Operator<SelectDesc> op4 = OperatorFactory.get(new SelectDesc(Utilities .makeList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0); addMapWork(mr, src, "a", op4); ReduceWork rWork = new ReduceWork(); mr.setReduceWork(rWork); rWork.setNumReduceTasks(Integer.valueOf(1)); rWork.setKeyDesc(op0.getConf().getKeySerializeInfo()); rWork.getTagToValueDesc().add(op0.getConf().getValueSerializeInfo()); // reduce side work Operator<FileSinkDesc> op3 = OperatorFactory.get(ctx, new FileSinkDesc(new Path(tmpdir + File.separator + "mapredplan5.out"), Utilities.defaultTd, false)); List<ExprNodeDesc> cols = new ArrayList<ExprNodeDesc>(); cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); Operator<SelectDesc> op2 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op3); rWork.setReducer(op2); }
/** * This method drives the test. It takes the data from getBaseTable() and * feeds it through a SELECT operator with a COLLECT operator after. Each * row that is produced by the collect operator is compared to getExpectedResult() * and if every row is the expected result the method completes without asserting. * @throws HiveException */ public final void testUdf() throws HiveException { InspectableObject [] data = getBaseTable(); List<ExprNodeDesc> expressionList = getExpressionList(); SelectDesc selectCtx = new SelectDesc(expressionList, OperatorTestUtils.createOutputColumnNames(expressionList)); Operator<SelectDesc> op = OperatorFactory.get(new CompilationOpContext(), SelectDesc.class); op.setConf(selectCtx); CollectDesc cd = new CollectDesc(Integer.valueOf(10)); CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild(cd, op); op.initialize(new JobConf(OperatorTestUtils.class), new ObjectInspector[] {data[0].oi}); OperatorTestUtils.assertResults(op, cdop, data, getExpectedResult()); }
@SuppressWarnings("unchecked") private void populateMapRedPlan2(Table src) throws Exception { ArrayList<String> outputColumns = new ArrayList<String>(); for (int i = 0; i < 2; i++) { outputColumns.add("_col" + i); } // map-side work Operator<ReduceSinkDesc> op1 = OperatorFactory.get(ctx, PlanUtils .getReduceSinkDesc(Utilities.makeList(getStringColumn("key")), Utilities .makeList(getStringColumn("key"), getStringColumn("value")), outputColumns, false, -1, 1, -1, AcidUtils.Operation.NOT_ACID)); addMapWork(mr, src, "a", op1); ReduceWork rWork = new ReduceWork(); rWork.setNumReduceTasks(Integer.valueOf(1)); rWork.setKeyDesc(op1.getConf().getKeySerializeInfo()); rWork.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); mr.setReduceWork(rWork); // reduce side work Operator<FileSinkDesc> op4 = OperatorFactory.get(ctx, new FileSinkDesc(new Path(tmpdir + File.separator + "mapredplan2.out"), Utilities.defaultTd, false)); Operator<FilterDesc> op3 = OperatorFactory.get(getTestFilterDesc("0"), op4); List<ExprNodeDesc> cols = new ArrayList<ExprNodeDesc>(); cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); Operator<SelectDesc> op2 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op3); rWork.setReducer(op2); }
protected static SelectOperator replaceReduceSinkWithSelectOperator(ReduceSinkOperator childRS, ParseContext context, AbstractCorrelationProcCtx procCtx) throws SemanticException { RowSchema inputRS = childRS.getSchema(); SelectDesc select = new SelectDesc(childRS.getConf().getValueCols(), childRS.getConf().getOutputValueColumnNames()); Operator<?> parent = getSingleParent(childRS); parent.getChildOperators().clear(); SelectOperator sel = (SelectOperator) OperatorFactory.getAndMakeChild( select, new RowSchema(inputRS.getSignature()), parent); sel.setColumnExprMap(childRS.getColumnExprMap()); sel.setChildOperators(childRS.getChildOperators()); for (Operator<? extends Serializable> ch : childRS.getChildOperators()) { ch.replaceParent(childRS, sel); } childRS.setChildOperators(null); childRS.setParentOperators(null); procCtx.addRemovedOperator(childRS); return sel; }
TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key")), op1); Operator<SelectDesc> op4 = OperatorFactory.get(new SelectDesc(Utilities .makeList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0); cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); Operator<SelectDesc> op2 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op3); rWork.setReducer(op2);
TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key")), op1); Operator<SelectDesc> op4 = OperatorFactory.get(new SelectDesc(Utilities .makeList(getStringColumn("key"), getStringColumn("value")), outputColumns), op0); cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); Operator<SelectDesc> op5 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op2);
protected static SelectOperator replaceReduceSinkWithSelectOperator(ReduceSinkOperator childRS, ParseContext context, AbstractCorrelationProcCtx procCtx) throws SemanticException { RowSchema inputRS = childRS.getSchema(); SelectDesc select = new SelectDesc(childRS.getConf().getValueCols(), childRS.getConf().getOutputValueColumnNames()); Operator<?> parent = getSingleParent(childRS); parent.removeChild(childRS); SelectOperator sel = (SelectOperator) OperatorFactory.getAndMakeChild( select, new RowSchema(inputRS.getSignature()), parent); sel.setColumnExprMap(childRS.getColumnExprMap()); sel.setChildOperators(childRS.getChildOperators()); for (Operator<? extends Serializable> ch : childRS.getChildOperators()) { ch.replaceParent(childRS, sel); } childRS.setChildOperators(null); childRS.setParentOperators(null); procCtx.addRemovedOperator(childRS); return sel; }