/** * Recursively traverse the graph to calculate the incoming operation. * * @param currentOperation the current operation from which the graph needs to explored * @param visitedOperations all the operations visited so far */ private void getIncomingOperationsForFieldHelper(Operation currentOperation, Set<Operation> visitedOperations) { if (!visitedOperations.add(currentOperation)) { return; } // reached the end of backward traversal if (currentOperation.getType() == OperationType.READ) { return; } // for transform we traverse backward in graph further through the inputs of the transform if (currentOperation.getType() == OperationType.TRANSFORM) { TransformOperation transform = (TransformOperation) currentOperation; for (InputField field : transform.getInputs()) { getIncomingOperationsForFieldHelper(operationsMap.get(field.getOrigin()), visitedOperations); } } }
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPointField, Set<EndPointField>> summary = new HashMap<>(); for (WriteOperation write : writeOperations) { List<InputField> inputs = write.getInputs(); for (InputField input : inputs) { computeIncomingSummaryHelper(new EndPointField(write.getDestination(), input.getName()), operationsMap.get(input.getOrigin()), write, summary); } } return summary; }
/** * Recursively traverse the graph to calculate the incoming operation. * * @param currentOperation the current operation from which the graph needs to explored * @param visitedOperations all the operations visited so far */ private void getIncomingOperationsForFieldHelper(Operation currentOperation, Set<Operation> visitedOperations) { if (!visitedOperations.add(currentOperation)) { return; } // reached the end of backward traversal if (currentOperation.getType() == OperationType.READ) { return; } // for transform we traverse backward in graph further through the inputs of the transform if (currentOperation.getType() == OperationType.TRANSFORM) { TransformOperation transform = (TransformOperation) currentOperation; for (InputField field : transform.getInputs()) { getIncomingOperationsForFieldHelper(operationsMap.get(field.getOrigin()), visitedOperations); } } }
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPointField, Set<EndPointField>> summary = new HashMap<>(); for (WriteOperation write : writeOperations) { List<InputField> inputs = write.getInputs(); for (InputField input : inputs) { computeIncomingSummaryHelper(new EndPointField(write.getDestination(), input.getName()), operationsMap.get(input.getOrigin()), write, summary); } } return summary; }
getIncomingOperationsForFieldHelper(operationsMap.get(input.getOrigin()), visitedOperations);
getIncomingOperationsForFieldHelper(operationsMap.get(input.getOrigin()), visitedOperations);
EndPoint source = read.getSource(); for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) { sourceEndPointFields.add(new EndPointField(source, inputField.getName())); TransformOperation transform = (TransformOperation) currentOperation; for (InputField inputField : transform.getInputs()) { computeIncomingSummaryHelper(field, operationsMap.get(inputField.getOrigin()), currentOperation, summary);
EndPoint source = read.getSource(); for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) { sourceEndPointFields.add(new EndPointField(source, inputField.getName())); TransformOperation transform = (TransformOperation) currentOperation; for (InputField inputField : transform.getInputs()) { computeIncomingSummaryHelper(field, operationsMap.get(inputField.getOrigin()), currentOperation, summary);