Refine search
private static ArrayList<ColumnInfo> createColInfos(Operator<?> input) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); for (ColumnInfo ci : input.getSchema().getSignature()) { cInfoLst.add(new ColumnInfo(ci)); } return cInfoLst; }
private static ArrayList<ColumnInfo> createColInfos(Operator<?> input) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); for (ColumnInfo ci : input.getSchema().getSignature()) { cInfoLst.add(new ColumnInfo(ci)); } return cInfoLst; }
private static ArrayList<ColumnInfo> createColInfosSubset(Operator<?> input, List<String> keepColNames) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); int pos = 0; for (ColumnInfo ci : input.getSchema().getSignature()) { if (pos < keepColNames.size() && ci.getInternalName().equals(keepColNames.get(pos))) { cInfoLst.add(new ColumnInfo(ci)); pos++; } } return cInfoLst; }
private static ArrayList<ColumnInfo> createColInfos(Operator<?> input, String tableAlias) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); for (ColumnInfo ci : input.getSchema().getSignature()) { ColumnInfo copyOfColumnInfo = new ColumnInfo(ci); copyOfColumnInfo.setTabAlias(tableAlias); cInfoLst.add(copyOfColumnInfo); } return cInfoLst; }
private static void pruneOperator(NodeProcessorCtx ctx, Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { // the pruning needs to preserve the order of columns in the input schema RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>(); RowSchema oldRS = op.getSchema(); for(ColumnInfo i : oldRS.getSignature()) { if (lookupColumn(cols, i.getInternalName()) != null) { rs.add(i); } } op.getSchema().setSignature(rs); } }
private static void pruneOperator(NodeProcessorCtx ctx, Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { // the pruning needs to preserve the order of columns in the input schema RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<ColumnInfo> rs = new ArrayList<ColumnInfo>(); RowSchema oldRS = op.getSchema(); for(ColumnInfo i : oldRS.getSignature()) { if (lookupColumn(cols, i.getInternalName()) != null) { rs.add(i); } } op.getSchema().setSignature(rs); } }
private static ArrayList<ColumnInfo> createColInfosSubset(Operator<?> input, List<String> keepColNames) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); int pos = 0; for (ColumnInfo ci : input.getSchema().getSignature()) { if (pos < keepColNames.size() && ci.getInternalName().equals(keepColNames.get(pos))) { cInfoLst.add(new ColumnInfo(ci)); pos++; } } return cInfoLst; }
private static ArrayList<ColumnInfo> createColInfos(Operator<?> input, String tableAlias) { ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>(); for (ColumnInfo ci : input.getSchema().getSignature()) { ColumnInfo copyOfColumnInfo = new ColumnInfo(ci); copyOfColumnInfo.setTabAlias(tableAlias); cInfoLst.add(copyOfColumnInfo); } return cInfoLst; }
private Operator<? extends OperatorDesc> genOutputSelectForGroupBy( Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) { assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size()); Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator(); Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator(); List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>(); List<String> colName = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); while (pIter.hasNext()) { ColumnInfo pInfo = pIter.next(); ColumnInfo cInfo = cIter.next(); ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(), pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol()); columns.add(column); colName.add(cInfo.getInternalName()); columnExprMap.put(cInfo.getInternalName(), column); } return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp.getSchema().getSignature()), columnExprMap, parentOp); } }
/** * Set the lineage information for the associated directory. * * @param dir The directory containing the query results. * @param dc The associated data container. * @param cols The list of columns. */ public synchronized void setLineage(Path dir, DataContainer dc, List<FieldSchema> cols) { // First lookup the file sink operator from the load work. Operator<?> op = dirToFop.get(dir.toUri().toString()); // Go over the associated fields and look up the dependencies // by position in the row schema of the filesink operator. if (op == null) { return; } List<ColumnInfo> signature = op.getSchema().getSignature(); int i = 0; for (FieldSchema fs : cols) { linfo.putDependency(dc, fs, index.getDependency(op, signature.get(i++))); } }
/** * The pruning needs to preserve the order of columns in the input schema * @param op * @param cols * @return * @throws SemanticException */ private static List<FieldNode> preserveColumnOrder(Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<FieldNode> rs = new ArrayList<>(); ArrayList<ColumnInfo> inputCols = inputSchema.getSignature(); for (ColumnInfo i: inputCols) { FieldNode fn = lookupColumn(cols, i.getInternalName()); if (fn != null) { rs.add(fn); } } return rs; } else { return cols; } }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSinkForJoin(int initialPos, List<String> outputColumnNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp, String tabAlias) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); for (int i = 0; i < index.length; i++) { ColumnInfo info = new ColumnInfo(inputOp.getSchema().getSignature().get(i)); String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, tabAlias, info.getIsVirtualCol()); columnDescriptors.put(outputColumnNames.get(initialPos + i), desc); } return columnDescriptors; }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSink(List<String> keepColNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); int pos = 0; for (int i = 0; i < index.length; i++) { ColumnInfo info = inputOp.getSchema().getSignature().get(i); if (pos < keepColNames.size() && info.getInternalName().equals(keepColNames.get(pos))) { String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, info.getTabAlias(), info.getIsVirtualCol()); columnDescriptors.put(keepColNames.get(pos), desc); pos++; } } return columnDescriptors; }
/** * Set the lineage information for the associated directory. * * @param dir The directory containing the query results. * @param dc The associated data container. * @param cols The list of columns. */ public void setLineage(Path dir, DataContainer dc, List<FieldSchema> cols) { // First lookup the file sink operator from the load work. Operator<?> op = dirToFop.get(dir); // Go over the associated fields and look up the dependencies // by position in the row schema of the filesink operator. if (op == null) { return; } List<ColumnInfo> signature = op.getSchema().getSignature(); int i = 0; for (FieldSchema fs : cols) { linfo.putDependency(dc, fs, index.getDependency(op, signature.get(i++))); } }
private Operator<? extends OperatorDesc> genOutputSelectForGroupBy( Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) { assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size()); Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator(); Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator(); List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>(); List<String> colName = new ArrayList<String>(); Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>(); while (pIter.hasNext()) { ColumnInfo pInfo = pIter.next(); ColumnInfo cInfo = cIter.next(); ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(), pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol()); columns.add(column); colName.add(cInfo.getInternalName()); columnExprMap.put(cInfo.getInternalName(), column); } return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp.getSchema().getSignature()), columnExprMap, parentOp); } }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSinkForJoin(int initialPos, List<String> outputColumnNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp, String tabAlias) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); for (int i = 0; i < index.length; i++) { ColumnInfo info = new ColumnInfo(inputOp.getSchema().getSignature().get(i)); String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, tabAlias, info.getIsVirtualCol()); columnDescriptors.put(outputColumnNames.get(initialPos + i), desc); } return columnDescriptors; }
/** * The pruning needs to preserve the order of columns in the input schema * @param op * @param cols * @return * @throws SemanticException */ private static List<FieldNode> preserveColumnOrder(Operator<? extends OperatorDesc> op, List<FieldNode> cols) throws SemanticException { RowSchema inputSchema = op.getSchema(); if (inputSchema != null) { ArrayList<FieldNode> rs = new ArrayList<>(); ArrayList<ColumnInfo> inputCols = inputSchema.getSignature(); for (ColumnInfo i: inputCols) { FieldNode fn = lookupColumn(cols, i.getInternalName()); if (fn != null) { rs.add(fn); } } return rs; } else { return cols; } }
private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSink(List<String> keepColNames, List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp) { Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>(); int pos = 0; for (int i = 0; i < index.length; i++) { ColumnInfo info = inputOp.getSchema().getSignature().get(i); if (pos < keepColNames.size() && info.getInternalName().equals(keepColNames.get(pos))) { String field; if (index[i] >= 0) { field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]); } else { field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1); } ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, info.getTabAlias(), info.getIsVirtualCol()); columnDescriptors.put(keepColNames.get(pos), desc); pos++; } } return columnDescriptors; }
@SuppressWarnings("unchecked") @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { // Assert that there is at least one item in the stack. This should never // be called for leafs. assert(!stack.isEmpty()); // LineageCtx LineageCtx lCtx = (LineageCtx) procCtx; Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>)nd; // Get the row schema of the input operator. // The row schema of the parent operator Operator<? extends OperatorDesc> inpOp = getParent(stack); lCtx.getIndex().copyPredicates(inpOp, op); RowSchema rs = op.getSchema(); ArrayList<ColumnInfo> inp_cols = inpOp.getSchema().getSignature(); int cnt = 0; for(ColumnInfo ci : rs.getSignature()) { lCtx.getIndex().putDependency(op, ci, lCtx.getIndex().getDependency(inpOp, inp_cols.get(cnt++))); } return null; } }
Map<String, ExprNodeDesc> colExprMap, boolean addEmptyTabAlias, boolean setColToNonVirtual) throws SemanticException { List<ColumnInfo> mapGBColInfoLst = inOp.getSchema().getSignature(); ArrayList<ExprNodeDesc> valueKeys = null; if (aggStartPos >= mapGBColInfoLst.size()) {