Dependency dep = new Dependency(); DependencyType new_type = LineageInfo.DependencyType.EXPRESSION; StringBuilder sb = new StringBuilder(); if (expr_dep != null && !expr_dep.getBaseCols().isEmpty()) { new_type = LineageCtx.getNewDependencyType(expr_dep.getType(), new_type); bci_set.addAll(expr_dep.getBaseCols()); if (expr_dep.getType() == LineageInfo.DependencyType.SIMPLE) { BaseColumnInfo col = expr_dep.getBaseCols().iterator().next(); Table t = col.getTabAlias().getTable(); if (t != null) { if (expr_dep == null || expr_dep.getBaseCols().isEmpty() || expr_dep.getType() != LineageInfo.DependencyType.SIMPLE) { sb.append(expr_dep != null && expr_dep.getExpr() != null ? expr_dep.getExpr() : ExprProcFactory.getExprString(rs, expr, lctx, inpOp, null)); expr = sb.toString(); dep.setExpr(expr); for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); dep.setBaseCols(bci_set); dep.setType(new_type); lctx.getIndex().putDependency(gop, col_infos.get(cnt++), dep);
columnLineageProcess.setAttribute(ATTRIBUTE_OUTPUTS, Collections.singletonList(getObjectId(outputColumn))); columnLineageProcess.setAttribute(ATTRIBUTE_QUERY, getObjectId(hiveProcess)); columnLineageProcess.setAttribute(ATTRIBUTE_DEPENDENCY_TYPE, entry.getValue().getType()); columnLineageProcess.setAttribute(ATTRIBUTE_EXPRESSION, entry.getValue().getExpr());
Dependency dep = new Dependency(); DependencyType new_type = LineageInfo.DependencyType.EXPRESSION; dep.setExpr(null); LinkedHashSet<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(); for(ExprNodeDesc expr : agg.getParameters()) { Dependency expr_dep = ExprProcFactory.getExprDependency(lctx, inpOp, expr); if (expr_dep != null) { new_type = LineageCtx.getNewDependencyType(expr_dep.getType(), new_type); bci_set.addAll(expr_dep.getBaseCols()); for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set)); dep.setType(new_type); lctx.getIndex().putDependency(gop, col_infos.get(cnt++), dep);
dep.getType() + " "); for(BaseColumnInfo col: dep.getBaseCols()) { sb.append("("+col.getTabAlias().getTable().getTableName() + ")" + col.getTabAlias().getAlias() + "."
Dependency dep = new Dependency(); DependencyType new_type = LineageInfo.DependencyType.EXPRESSION; dep.setExpr(null); LinkedHashSet<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(); for(ExprNodeDesc expr : agg.getParameters()) { Dependency expr_dep = ExprProcFactory.getExprDependency(lctx, inpOp, expr); if (expr_dep != null) { new_type = LineageCtx.getNewDependencyType(expr_dep.getType(), new_type); bci_set.addAll(expr_dep.getBaseCols()); for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set)); dep.setType(new_type); lctx.getIndex().putDependency(gop, col_infos.get(cnt++), dep);
for(ColumnInfo ci : rs.getSignature()) { Dependency dep = new Dependency(); BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); dep.setType(LineageInfo.DependencyType.SIMPLE); dep.setExpr(null); dep.setBaseCols(new ArrayList<BaseColumnInfo>()); dep.getBaseCols().add(bci);
for(ColumnInfo ci : rs.getSignature()) { Dependency dep = new Dependency(); BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); dep.setType(LineageInfo.DependencyType.SIMPLE); dep.setExpr(null); dep.setBaseCols(new ArrayList<BaseColumnInfo>()); dep.getBaseCols().add(bci);
Dependency dep = new Dependency(); DependencyType new_type = LineageInfo.DependencyType.SCRIPT; dep.setType(LineageInfo.DependencyType.SCRIPT); dep.setExpr(null); Dependency d = lCtx.getIndex().getDependency(inpOp, ci); if (d != null) { new_type = LineageCtx.getNewDependencyType(d.getType(), new_type); col_set.addAll(d.getBaseCols()); dep.setType(new_type); dep.setBaseCols(new ArrayList<BaseColumnInfo>(col_set));
Dependency dep = new Dependency(); DependencyType new_type = LineageInfo.DependencyType.SCRIPT; dep.setType(LineageInfo.DependencyType.SCRIPT); dep.setExpr(null); Dependency d = lCtx.getIndex().getDependency(inpOp, ci); if (d != null) { new_type = LineageCtx.getNewDependencyType(d.getType(), new_type); col_set.addAll(d.getBaseCols()); dep.setType(new_type); dep.setBaseCols(new ArrayList<BaseColumnInfo>(col_set));
/** * Merges the new dependencies in dep to the existing dependencies * of (op, ci). * * @param op The operator of the column whose dependency is being modified. * @param ci The column info of the associated column. * @param dep The new dependency. */ public void mergeDependency(Operator<? extends OperatorDesc> op, ColumnInfo ci, Dependency dep) { Dependency old_dep = getDependency(op, ci); if (old_dep == null) { putDependency(op, ci, dep); } else { LineageInfo.DependencyType new_type = LineageCtx.getNewDependencyType(old_dep.getType(), LineageInfo.DependencyType.EXPRESSION); old_dep.setType(new_type); Set<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(old_dep.getBaseCols()); bci_set.addAll(dep.getBaseCols()); old_dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set)); // TODO: Fix the expressions later. old_dep.setExpr(null); } }
/** * Merges the new dependencies in dep to the existing dependencies * of (op, ci). * * @param op The operator of the column whose dependency is being modified. * @param ci The column info of the associated column. * @param dependency The new dependency. */ public void mergeDependency(Operator<? extends Serializable> op, ColumnInfo ci, Dependency dep) { Dependency old_dep = getDependency(op, ci); if (old_dep == null) { putDependency(op, ci, dep); } else { LineageInfo.DependencyType new_type = LineageCtx.getNewDependencyType(old_dep.getType(), LineageInfo.DependencyType.EXPRESSION); old_dep.setType(new_type); Set<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(old_dep.getBaseCols()); bci_set.addAll(dep.getBaseCols()); old_dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set)); // TODO: Fix the expressions later. old_dep.setExpr(null); } } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { assert (nd instanceof ExprNodeGenericFuncDesc || nd instanceof ExprNodeFieldDesc); // Concatenate the dependencies of all the children to compute the new // dependency. Dependency dep = new Dependency(); LinkedHashSet<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(); LineageInfo.DependencyType new_type = LineageInfo.DependencyType.EXPRESSION; for (Object child : nodeOutputs) { if (child == null) { continue; } Dependency child_dep = (Dependency) child; new_type = LineageCtx.getNewDependencyType(child_dep.getType(), new_type); bci_set.addAll(child_dep.getBaseCols()); } dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set)); dep.setType(new_type); return dep; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { assert (nd instanceof ExprNodeGenericFuncDesc || nd instanceof ExprNodeFieldDesc); // Concatenate the dependencies of all the children to compute the new // dependency. Dependency dep = new Dependency(); LinkedHashSet<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(); LineageInfo.DependencyType new_type = LineageInfo.DependencyType.EXPRESSION; for (Object child : nodeOutputs) { if (child == null) { continue; } Dependency child_dep = (Dependency) child; new_type = LineageCtx.getNewDependencyType(child_dep.getType(), new_type); bci_set.addAll(child_dep.getBaseCols()); } dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set)); dep.setType(new_type); return dep; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { assert (nd instanceof ExprNodeConstantDesc || nd instanceof ExprNodeNullDesc); // Create a dependency that has no basecols Dependency dep = new Dependency(); dep.setType(LineageInfo.DependencyType.SIMPLE); dep.setBaseCols(new ArrayList<BaseColumnInfo>()); return dep; } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { assert (nd instanceof ExprNodeConstantDesc); // Create a dependency that has no basecols Dependency dep = new Dependency(); dep.setType(LineageInfo.DependencyType.SIMPLE); dep.setBaseCols(new ArrayList<BaseColumnInfo>()); return dep; } }
targets.add(target); Dependency dep = dependencies.get(i); addEdge(vertexCache, edges, dep.getBaseCols(), target, dep.getExpr(), Edge.Type.PROJECTION);
dep.getType() + " "); for(BaseColumnInfo col: dep.getBaseCols()) { sb.append("("+col.getTabAlias().getTable().getTableName() + ")" + col.getTabAlias().getAlias() + "."
&& (dep != null && dep.getType() == DependencyType.SIMPLE)) { Set<BaseColumnInfo> baseCols = dep.getBaseCols(); if (baseCols != null && !baseCols.isEmpty()) { BaseColumnInfo baseCol = baseCols.iterator().next(); && !tabAlias.startsWith("_") && !tabAlias.startsWith("$")) { if (cond != null && !findSourceColumn(lctx, cond, tabAlias, alias) && dep != null) { cond.getBaseCols().addAll(dep.getBaseCols()); cond.getBaseCols().addAll(dep.getBaseCols()); if (dep.getExpr() != null) { return dep.getExpr();
targets.add(target); Dependency dep = dependencies.get(i); addEdge(vertexCache, edges, dep.getBaseCols(), target, dep.getExpr(), Edge.Type.PROJECTION);