private Operator<? extends OperatorDesc> genOutputSelectForGroupBy( Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) { assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size()); Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator(); Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator(); List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>(); List<String> colName = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); while (pIter.hasNext()) { ColumnInfo pInfo = pIter.next(); ColumnInfo cInfo = cIter.next(); ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(), pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol()); columns.add(column); colName.add(cInfo.getInternalName()); columnExprMap.put(cInfo.getInternalName(), column); } return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp.getSchema().getSignature()), columnExprMap, parentOp); } }
private static void pruneOperator(NodeProcessorCtx ctx, Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { // the pruning needs to preserve the order of columns in the input schema RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>(); RowSchema oldRS = op.getSchema(); for(ColumnInfo i : oldRS.getSignature()) { if (lookupColumn(cols, i.getInternalName()) != null) { rs.add(i); } } op.getSchema().setSignature(rs); } }
/** * Create the list of internal columns for select tag of LV */ public List<FieldNode> getSelectColsFromLVJoin(RowSchema rs, List<FieldNode> colList) throws SemanticException { List<FieldNode> columns = new ArrayList<>(); for (FieldNode col : colList) { if (rs.getColumnInfo(col.getFieldName()) != null) { columns.add(col); } } return columns; }
private GroupByOperator genMapGroupby1(Operator<? extends OperatorDesc> mGby, int indexOfDist) throws CloneNotSupportedException { GroupByOperator mGby1 = (GroupByOperator) mGby.clone(); // distinct is at lost position. String fieldString = mGby1.getConf().getOutputColumnNames().get(indexOfDist + 1); mGby1.getColumnExprMap().remove(fieldString); mGby1.getConf().getOutputColumnNames().remove(indexOfDist + 1); mGby1.getConf().getAggregators().remove(indexOfDist); mGby1.getConf().setDistinct(false); mGby1.getSchema().getColumnNames().remove(indexOfDist + 1); mGby1.getSchema().getSignature().remove(indexOfDist + 1); return mGby1; }
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); List<ColumnInfo> inputColumns = input.getSchema().getSignature(); ArrayList<ColumnInfo> outputColumns = new ArrayList<ColumnInfo>(); List<String> outputColumnNames = new ArrayList<String>(); rsDesc, new RowSchema(outputColumns), input); rsOp.setInputAliases(input.getSchema().getTableNames() .toArray(new String[input.getSchema().getTableNames().size()]));
public ArrayList<ColumnInfo> getColumnInfos() { return rowSchema.getSignature(); }
LOG.info("RS " + reduce.getIdentifier() + " oldColExprMap: " + oldMap); RowSchema oldRS = reduce.getSchema(); ArrayList<ColumnInfo> old_signature = oldRS.getSignature(); ArrayList<ColumnInfo> signature = new ArrayList<ColumnInfo>(old_signature); ExprNodeDesc outputColExpr = valueExprs.get(i); if (!retainFlags[i]) { ColumnInfo colInfo = oldRS.getColumnInfo(outputCol); if (colInfo == null) { outputCol = Utilities.ReduceField.VALUE.toString() + "." + outputCol; colInfo = oldRS.getColumnInfo(outputCol); oldRS.setSignature(signature); reduce.getSchema().setSignature(signature); reduceConf.setOutputValueColumnNames(newValueColNames); reduceConf.setValueCols(newValueExprs);
ExprNodeDesc exprNodeDesc = new ExprNodeColumnDesc(col); colList.add(exprNodeDesc); String internalName = selRS.getColumnNames().get(i); columnNames.add(internalName); columnExprMap.put(internalName, exprNodeDesc); signature.add(selRS.getSignature().get(i)); TypeInfo destType = selRS.getSignature().get(this.columns.size() + i).getType(); if (!srcType.equals(destType)) { ColumnInfo col = columns.get(this.columns.size() + dynamicPartBegin); TypeInfo srcType = col.getType(); TypeInfo destType = selRS.getSignature().get(this.columns.size() + i).getType(); exprNodeDesc = new ExprNodeColumnDesc(col); if (!srcType.equals(destType)) { String internalName = selRS.getColumnNames().get(this.columns.size() + i); columnNames.add(internalName); columnExprMap.put(internalName, exprNodeDesc); signature.add(selRS.getSignature().get(this.columns.size() + i)); selRS.setSignature(signature); operator.setSchema(selRS);
Operator<? extends OperatorDesc> parent = op.getParentOperators().get(0); RowSchema parentRS = parent.getSchema(); List<ColumnInfo> sig = parentRS.getSignature(); List<FieldNode> colList = new ArrayList<>(); for (ColumnInfo cI : sig) { if (prunedCols.size() != inputRS.getSignature().size() && !(op.getChildOperators().get(0) instanceof SelectOperator)) { ArrayList<ExprNodeDesc> exprs = new ArrayList<ExprNodeDesc>(); for (FieldNode internalCol: prunedCols) { String internalName = internalCol.getFieldName(); ColumnInfo valueInfo = inputRS.getColumnInfo(internalName); ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName(), valueInfo.getTabAlias(), valueInfo.getIsVirtualCol()); op.removeChild(child); SelectOperator sel = (SelectOperator) OperatorFactory.getAndMakeChild( select, new RowSchema(outputRS), op); OperatorFactory.makeChild(sel, child);
Set<String> tableNames = input.getSchema().getTableNames(); for (String tableName : tableNames) { if (tableName != null) { ArrayList<ColumnInfo> cinfoLst = createColInfosSubset(input, keepColNames); SelectOperator selectOp = (SelectOperator) OperatorFactory.getAndMakeChild( selectDesc, new RowSchema(cinfoLst), rsOp); selectOp.setColumnExprMap(descriptors);
for (ColumnInfo ci : rowSchema.getSignature()) { String outColName = ci.getInternalName(); ExprNodeDesc end = colExprMap.get(outColName); if (rowSchema.getColumnInfo(pair.getKey()) == null) { ColStatistics colStat = getColStatisticsFromExpression(conf, parentStats, pair.getValue()); if (colStat != null) {
ColumnInfo valueInfo = inputRS.getColumnInfo(internalName); ExprNodeDesc colDesc = new ExprNodeColumnDesc(valueInfo.getType(), valueInfo .getInternalName(), valueInfo.getTabAlias(), valueInfo.getIsVirtualCol()); select, new RowSchema(outputRS), input);
public RowResolver() { rowSchema = new RowSchema(); rslvMap = new LinkedHashMap<String, LinkedHashMap<String, ColumnInfo>>(); invRslvMap = new HashMap<String, String[]>(); altInvRslvMap = new HashMap<String, String[]>(); expressionMap = new HashMap<String, ASTNode>(); isExprResolver = false; ambiguousColumns = new LinkedHashMap<String, LinkedHashMap<String, String>>(); checkForAmbiguity = false; }
Map<Integer, ExprNodeDesc> map = new HashMap<>(); for (Entry<ColumnInfo, ExprNodeDesc> entry : constMap.entrySet()) { map.put(parent.getSchema().getPosition(entry.getKey().getInternalName()), entry.getValue()); ArrayList<ColumnInfo> signature = op.getSchema().getSignature(); if (op instanceof LimitOperator || op instanceof FilterOperator) { String parentColName = ((ExprNodeColumnDesc) expr).getColumn(); int parentPos = parent.getSchema().getPosition(parentColName); if (parentsToConstant.get(tag).containsKey(parentPos)) { if (entry.getValue().isSame(expr)) { constants.put(signature.get(op.getSchema().getPosition(entry.getKey())), parentsToConstant.get(tag).get(parentPos)); if (op.getColumnExprMap() != null && op.getColumnExprMap().entrySet() != null) { for (Entry<String, ExprNodeDesc> entry : op.getColumnExprMap().entrySet()) { if (op.getSchema().getPosition(entry.getKey()) == -1) { int parentPos = parent.getSchema().getPosition(parentColName); if (parentsToConstant.get(0).containsKey(parentPos)) { constants.put(signature.get(op.getSchema().getPosition(entry.getKey())), parentsToConstant.get(0).get(parentPos));
ColumnInfo col = joinRS.getColumnInfo(internalName); rs.add(col); op.getSchema().setSignature(rs); cppCtx.getJoinPrunedColLists().put(op, prunedColLists);
protected Set<String> getAliases(Node nd) throws SemanticException { return ((Operator)nd).getSchema().getTableNames(); }
ColumnInfo colInfo = inputRS.getColumnInfo(column); if (colInfo == null) { continue; int position = inputRS.getPosition(column); if (position >= 0) {
RowSchema toRS = ctx.getOp().getSchema(); Operator<? extends OperatorDesc> op = ctx.getOp(); ColumnInfo ci = toRS.getColumnInfo(colref.getColumn()); String tabAlias = null; if (ci != null) { if (groupBy.getConf().isGroupingSetsPresent()) { int groupingSetPlaceholderPos = groupBy.getConf().getKeys().size() - 1; if (colref.getColumn().equals(groupBy.getSchema().getColumnNames().get(groupingSetPlaceholderPos))) { exp = null;
ColumnInfo columnInfo = rowSchema.getColumnInfo(colName); if (columnInfo != null) { tableNeedToCheck.add(columnInfo.getTabAlias()); LinkedHashSet<ReduceSinkOperator> correlatedRsops = null; for (Operator<? extends OperatorDesc> parent : current.getParentOperators()) { Set<String> tableNames = parent.getSchema().getTableNames(); for (String tbl : tableNames) { if (tableNeedToCheck.contains(tbl)) {
private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { GroupByDesc gbyDesc = gbyOp.getConf(); int numCols = gbyDesc.getOutputColumnNames().size(); int aggCols = gbyDesc.getAggregators().size(); // If the Group by operator has null key if (numCols == aggCols) { return GbyKeyType.NULL; } // If the Gby key is a constant List<String> dpCols = gbyOp.getSchema().getColumnNames().subList(0, numCols - aggCols); for(String dpCol : dpCols) { ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, gbyOp); if (!(end instanceof ExprNodeConstantDesc)) { return GbyKeyType.OTHER; } } return GbyKeyType.CONSTANT; }