if (expr_dep.getType() == LineageInfo.DependencyType.SIMPLE) { BaseColumnInfo col = expr_dep.getBaseCols().iterator().next(); Table t = col.getTabAlias().getTable(); if (t != null) { sb.append(t.getDbName()).append(".").append(t.getTableName()).append("."); sb.append(col.getColumn().getName()); for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(null); bci_set.add(bci);
for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(null); bci_set.add(bci);
for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(null); bci_set.add(bci);
BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(fieldSchemaMap.get(ci.getInternalName()));
BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(fieldSchemaMap.get(ci.getInternalName()));
sb.append("("+col.getTabAlias().getTable().getTableName() + ")" + col.getTabAlias().getAlias() + "." + col.getColumn() + ", ");
public static Map<String, List<HiveColumnLineageInfo>> buildLineageMap(LineageInfo lInfo) { Map<String, List<HiveColumnLineageInfo>> m = new HashMap<>(); for (Map.Entry<LineageInfo.DependencyKey, LineageInfo.Dependency> e : lInfo.entrySet()) { List<HiveColumnLineageInfo> l = new ArrayList<>(); String k = getQualifiedName(e.getKey()); if (LOG.isDebugEnabled()) { LOG.debug("buildLineageMap(): key={}; value={}", e.getKey(), e.getValue()); } Collection<LineageInfo.BaseColumnInfo> baseCols = getBaseCols(e.getValue()); if (baseCols != null) { for (LineageInfo.BaseColumnInfo iCol : baseCols) { String db = iCol.getTabAlias().getTable().getDbName(); String table = iCol.getTabAlias().getTable().getTableName(); String colQualifiedName = iCol.getColumn() == null ? db + "." + table : db + "." + table + "." + iCol.getColumn().getName(); l.add(new HiveColumnLineageInfo(e.getValue(), colQualifiedName)); } if (LOG.isDebugEnabled()) { LOG.debug("Setting lineage --> Input: {} ==> Output : {}", l, k); } m.put(k, l); } } return m; }
protected String getQualifiedName(BaseColumnInfo column) { String dbName = column.getTabAlias().getTable().getDbName(); String tableName = column.getTabAlias().getTable().getTableName(); String colName = column.getColumn() != null ? column.getColumn().getName() : null; if (colName == null) { return (dbName + QNAME_SEP_ENTITY_NAME + tableName + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); } else { return (dbName + QNAME_SEP_ENTITY_NAME + tableName + QNAME_SEP_ENTITY_NAME + colName + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName(); } }
sb.append("("+col.getTabAlias().getTable().getTableName() + ")" + col.getTabAlias().getAlias() + "." + col.getColumn() + ", ");
if (expr_dep.getType() == LineageInfo.DependencyType.SIMPLE) { BaseColumnInfo col = expr_dep.getBaseCols().iterator().next(); Table t = col.getTabAlias().getTable(); if (t != null) { sb.append(Warehouse.getQualifiedName(t)).append("."); sb.append(col.getColumn().getName()); for(BaseColumnInfo bci : inp_dep.getBaseCols()) { new_type = LineageCtx.getNewDependencyType(inp_dep.getType(), new_type); tai_set.add(bci.getTabAlias()); BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(null); bci_set.add(bci);
if (baseCols != null && !baseCols.isEmpty()) { BaseColumnInfo baseCol = baseCols.iterator().next(); tabAlias = baseCol.getTabAlias().getAlias(); alias = baseCol.getColumn().getName();
if (baseCols != null && !baseCols.isEmpty()) { BaseColumnInfo baseCol = baseCols.iterator().next(); tabAlias = baseCol.getTabAlias().getAlias(); alias = baseCol.getColumn().getName();
sb.append("("+col.getTabAlias().getTable().getTableName() + ")" + col.getTabAlias().getAlias() + "." + col.getColumn() + ", ");
BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(fieldSchemaMap.get(ci.getInternalName()));
BaseColumnInfo bci = new BaseColumnInfo(); bci.setTabAlias(tai); bci.setColumn(fieldSchemaMap.get(ci.getInternalName()));
/** * Convert a list of columns to a set of vertices. * Use cached vertices if possible. */ private Set<Vertex> createSourceVertices( Map<String, Vertex> vertexCache, Collection<BaseColumnInfo> baseCols) { Set<Vertex> sources = new LinkedHashSet<Vertex>(); if (baseCols != null && !baseCols.isEmpty()) { for(BaseColumnInfo col: baseCols) { Table table = col.getTabAlias().getTable(); if (table.isTemporary()) { // Ignore temporary tables continue; } Vertex.Type type = Vertex.Type.TABLE; String tableName = table.getDbName() + "." + table.getTableName(); FieldSchema fieldSchema = col.getColumn(); String label = tableName; if (fieldSchema != null) { type = Vertex.Type.COLUMN; label = tableName + "." + fieldSchema.getName(); } sources.add(getOrCreateVertex(vertexCache, label, type)); } } return sources; }
private static boolean findSourceColumn( LineageCtx lctx, Predicate cond, String tabAlias, String alias) { for (Map.Entry<String, TableScanOperator> topOpMap: lctx.getParseCtx().getTopOps().entrySet()) { TableScanOperator tableScanOp = topOpMap.getValue(); Table tbl = tableScanOp.getConf().getTableMetadata(); if (tbl.getTableName().equals(tabAlias) || tabAlias.equals(tableScanOp.getConf().getAlias())) { for (FieldSchema column: tbl.getCols()) { if (column.getName().equals(alias)) { TableAliasInfo table = new TableAliasInfo(); table.setTable(tbl.getTTable()); table.setAlias(tabAlias); BaseColumnInfo colInfo = new BaseColumnInfo(); colInfo.setColumn(column); colInfo.setTabAlias(table); cond.getBaseCols().add(colInfo); return true; } } } } return false; }
/** * Convert a list of columns to a set of vertices. * Use cached vertices if possible. */ private static Set<Vertex> createSourceVertices( Map<String, Vertex> vertexCache, Collection<BaseColumnInfo> baseCols) { Set<Vertex> sources = new LinkedHashSet<Vertex>(); if (baseCols != null && !baseCols.isEmpty()) { for(BaseColumnInfo col: baseCols) { Table table = col.getTabAlias().getTable(); if (table.isTemporary()) { // Ignore temporary tables continue; } Vertex.Type type = Vertex.Type.TABLE; String tableName = Warehouse.getQualifiedName(table); FieldSchema fieldSchema = col.getColumn(); String label = tableName; if (fieldSchema != null) { type = Vertex.Type.COLUMN; label = tableName + "." + fieldSchema.getName(); } sources.add(getOrCreateVertex(vertexCache, label, type)); } } return sources; }
private static boolean findSourceColumn( LineageCtx lctx, Predicate cond, String tabAlias, String alias) { for (Map.Entry<String, TableScanOperator> topOpMap: lctx.getParseCtx().getTopOps().entrySet()) { TableScanOperator tableScanOp = topOpMap.getValue(); Table tbl = tableScanOp.getConf().getTableMetadata(); if (tbl.getTableName().equals(tabAlias) || tabAlias.equals(tableScanOp.getConf().getAlias())) { for (FieldSchema column: tbl.getCols()) { if (column.getName().equals(alias)) { TableAliasInfo table = new TableAliasInfo(); table.setTable(tbl.getTTable()); table.setAlias(tabAlias); BaseColumnInfo colInfo = new BaseColumnInfo(); colInfo.setColumn(column); colInfo.setTabAlias(table); cond.getBaseCols().add(colInfo); return true; } } } } return false; }