private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { GroupByDesc gbyDesc = gbyOp.getConf(); int numCols = gbyDesc.getOutputColumnNames().size(); int aggCols = gbyDesc.getAggregators().size(); // If the Group by operator has null key if (numCols == aggCols) { return GbyKeyType.NULL; } // If the Gby key is a constant List<String> dpCols = gbyOp.getSchema().getColumnNames().subList(0, numCols - aggCols); for(String dpCol : dpCols) { ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, gbyOp); if (!(end instanceof ExprNodeConstantDesc)) { return GbyKeyType.OTHER; } } return GbyKeyType.CONSTANT; }
private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { GroupByDesc gbyDesc = gbyOp.getConf(); int numCols = gbyDesc.getOutputColumnNames().size(); int aggCols = gbyDesc.getAggregators().size(); // If the Group by operator has null key if (numCols == aggCols) { return GbyKeyType.NULL; } // If the Gby key is a constant List<String> dpCols = gbyOp.getSchema().getColumnNames().subList(0, numCols - aggCols); for(String dpCol : dpCols) { ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, gbyOp); if (!(end instanceof ExprNodeConstantDesc)) { return GbyKeyType.OTHER; } } return GbyKeyType.CONSTANT; }
private GroupByOperator genMapGroupby1(Operator<? extends OperatorDesc> mGby, int indexOfDist) throws CloneNotSupportedException { GroupByOperator mGby1 = (GroupByOperator) mGby.clone(); // distinct is at lost position. String fieldString = mGby1.getConf().getOutputColumnNames().get(indexOfDist + 1); mGby1.getColumnExprMap().remove(fieldString); mGby1.getConf().getOutputColumnNames().remove(indexOfDist + 1); mGby1.getConf().getAggregators().remove(indexOfDist); mGby1.getConf().setDistinct(false); mGby1.getSchema().getColumnNames().remove(indexOfDist + 1); mGby1.getSchema().getSignature().remove(indexOfDist + 1); return mGby1; }
private boolean allStaticPartitions(Operator<? extends OperatorDesc> op, final DynamicPartitionCtx dynPartCtx) { int numDpCols = dynPartCtx.getNumDPCols(); int numCols = op.getSchema().getColumnNames().size(); List<String> dpCols = op.getSchema().getColumnNames().subList(numCols - numDpCols, numCols); if (op.getColumnExprMap() == null) { // find first operator upstream with valid (non-null) column expression map for(Operator<? extends OperatorDesc> parent : op.getParentOperators()) { if (parent.getColumnExprMap() != null) { op = parent; break; } } } if (op.getColumnExprMap() != null) { for(String dpCol : dpCols) { ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, op); if (!(end instanceof ExprNodeConstantDesc)) { return false; } } } else { return false; } return true; }
private boolean allStaticPartitions(Operator<? extends OperatorDesc> op, final DynamicPartitionCtx dynPartCtx) { int numDpCols = dynPartCtx.getNumDPCols(); int numCols = op.getSchema().getColumnNames().size(); List<String> dpCols = op.getSchema().getColumnNames().subList(numCols - numDpCols, numCols); if (op.getColumnExprMap() == null) { // find first operator upstream with valid (non-null) column expression map for(Operator<? extends OperatorDesc> parent : op.getParentOperators()) { if (parent.getColumnExprMap() != null) { op = parent; break; } } } if (op.getColumnExprMap() != null) { for(String dpCol : dpCols) { ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCol, op); if (!(end instanceof ExprNodeConstantDesc)) { return false; } } } else { return false; } return true; }
ExprNodeDesc exprNodeDesc = new ExprNodeColumnDesc(col); colList.add(exprNodeDesc); String internalName = selRS.getColumnNames().get(i); columnNames.add(internalName); columnExprMap.put(internalName, exprNodeDesc); String internalName = selRS.getColumnNames().get(this.columns.size() + i); columnNames.add(internalName); columnExprMap.put(internalName, exprNodeDesc);
ExprNodeDesc exprNodeDesc = new ExprNodeColumnDesc(col); colList.add(exprNodeDesc); String internalName = selRS.getColumnNames().get(i); columnNames.add(internalName); columnExprMap.put(internalName, exprNodeDesc); String internalName = selRS.getColumnNames().get(this.columns.size() + i); columnNames.add(internalName); columnExprMap.put(internalName, exprNodeDesc);
private static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input, ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order, String nullOrder, int numReducers, Operation acidOperation, HiveConf hiveConf, List<String> keepColNames) throws SemanticException { // 1. Generate RS operator // 1.1 Prune the tableNames, only count the tableNames that are not empty strings // as empty string in table aliases is only allowed for virtual columns. String tableAlias = null; Set<String> tableNames = input.getSchema().getTableNames(); for (String tableName : tableNames) { if (tableName != null) { if (tableName.length() == 0) { if (tableAlias == null) { tableAlias = tableName; } } else { if (tableAlias == null || tableAlias.length() == 0) { tableAlias = tableName; } else { if (!tableName.equals(tableAlias)) { throw new SemanticException( "In CBO return path, genReduceSinkAndBacktrackSelect is expecting only one tableAlias but there is more than one"); } } } } } if (tableAlias == null) { throw new SemanticException( "In CBO return path, genReduceSinkAndBacktrackSelect is expecting only one tableAlias but there is none");
private static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input, ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order, String nullOrder, int numReducers, Operation acidOperation, HiveConf hiveConf, List<String> keepColNames) throws SemanticException { // 1. Generate RS operator // 1.1 Prune the tableNames, only count the tableNames that are not empty strings // as empty string in table aliases is only allowed for virtual columns. String tableAlias = null; Set<String> tableNames = input.getSchema().getTableNames(); for (String tableName : tableNames) { if (tableName != null) { if (tableName.length() == 0) { if (tableAlias == null) { tableAlias = tableName; } } else { if (tableAlias == null || tableAlias.length() == 0) { tableAlias = tableName; } else { if (!tableName.equals(tableAlias)) { throw new SemanticException( "In CBO return path, genReduceSinkAndBacktrackSelect is expecting only one tableAlias but there is more than one"); } } } } } if (tableAlias == null) { throw new SemanticException( "In CBO return path, genReduceSinkAndBacktrackSelect is expecting only one tableAlias but there is none");
for (String colName : jop.getSchema().getColumnNames()) { if (!colNameStatsAvailable.contains(colName)) { neededColumns.add(colName);
for (String colName : jop.getSchema().getColumnNames()) { if (!colNameStatsAvailable.contains(colName)) { neededColumns.add(colName);
aspCtx, jop.getSchema().getColumnNames(), jop, stats.getNumRows()); aspCtx, jop.getSchema().getColumnNames(), jop, wcStats.getNumRows());
int numCols = gbyDesc.getOutputColumnNames().size(); int aggCols = gbyDesc.getAggregators().size(); List<String> dpCols = cgbyOp.getSchema().getColumnNames().subList(0, numCols - aggCols); for(int i = 0; i < dpCols.size(); i++) { ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCols.get(i), cgbyOp);
if (groupBy.getConf().isGroupingSetsPresent()) { int groupingSetPlaceholderPos = groupBy.getConf().getKeys().size() - 1; if (colref.getColumn().equals(groupBy.getSchema().getColumnNames().get(groupingSetPlaceholderPos))) { exp = null;
if (groupBy.getConf().isGroupingSetsPresent()) { int groupingSetPlaceholderPos = groupBy.getConf().getKeys().size() - 1; if (colref.getColumn().equals(groupBy.getSchema().getColumnNames().get(groupingSetPlaceholderPos))) { exp = null;
rsOp.setInputAliases(input.getSchema().getColumnNames() .toArray(new String[input.getSchema().getColumnNames().size()]));
private static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input, ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order, int numReducers, Operation acidOperation, boolean strictMode, List<String> keepColNames) throws SemanticException { // 1. Generate RS operator ReduceSinkOperator rsOp = genReduceSink(input, keys, tag, partitionCols, order, numReducers, acidOperation, strictMode); // 2. Generate backtrack Select operator Map<String, ExprNodeDesc> descriptors = buildBacktrackFromReduceSink(keepColNames, rsOp.getConf().getOutputKeyColumnNames(), rsOp.getConf().getOutputValueColumnNames(), rsOp.getValueIndex(), input); SelectDesc selectDesc = new SelectDesc(new ArrayList<ExprNodeDesc>(descriptors.values()), new ArrayList<String>(descriptors.keySet())); ArrayList<ColumnInfo> cinfoLst = createColInfosSubset(input, keepColNames); SelectOperator selectOp = (SelectOperator) OperatorFactory.getAndMakeChild(selectDesc, new RowSchema(cinfoLst), rsOp); selectOp.setColumnExprMap(descriptors); if (LOG.isDebugEnabled()) { LOG.debug("Generated " + selectOp + " with row schema: [" + selectOp.getSchema() + "]"); } return selectOp; }