private void populateSourcesAndDestinations() { sources = new HashSet<>(); destinations = new HashSet<>(); for (Operation operation : operations) { if (OperationType.READ == operation.getType()) { ReadOperation read = (ReadOperation) operation; sources.add(read.getSource()); } else if (OperationType.WRITE == operation.getType()) { WriteOperation write = (WriteOperation) operation; destinations.add(write.getDestination()); } } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!super.equals(o)) { return false; } TransformOperation that = (TransformOperation) o; return Objects.equals(inputs, that.inputs) && Objects.equals(outputs, that.outputs); }
@Override public int hashCode() { return Objects.hash(super.hashCode(), inputs, destination); } }
if (operationsMap.containsKey(operation.getName())) { throw new IllegalArgumentException(String.format("All operations provided for creating field " + "level lineage info must have unique names. " + "Operation name '%s' is repeated.", operation.getName())); operationsMap.put(operation.getName(), operation); switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation;
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
if (operationsMap.containsKey(operation.getName())) { throw new IllegalArgumentException(String.format("All operations provided for creating field " + "level lineage info must have unique names. " + "Operation name '%s' is repeated.", operation.getName())); operationsMap.put(operation.getName(), operation); switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation;
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
Operation previousOperation, Map<EndPointField, Set<EndPointField>> summary) { if (currentOperation.getType() == OperationType.READ) { if (OperationType.WRITE == previousOperation.getType()) { WriteOperation previousWrite = (WriteOperation) previousOperation; inputFields = new HashSet<>(previousWrite.getInputs()); } else if (OperationType.TRANSFORM == previousOperation.getType()) { TransformOperation previousTransform = (TransformOperation) previousOperation; inputFields = new HashSet<>(previousTransform.getInputs()); EndPoint source = read.getSource(); for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) { sourceEndPointFields.add(new EndPointField(source, inputField.getName())); if (currentOperation.getType() == OperationType.TRANSFORM) { TransformOperation transform = (TransformOperation) currentOperation; for (InputField inputField : transform.getInputs()) {
private void populateSourcesAndDestinations() { sources = new HashSet<>(); destinations = new HashSet<>(); for (Operation operation : operations) { if (OperationType.READ == operation.getType()) { ReadOperation read = (ReadOperation) operation; sources.add(read.getSource()); } else if (OperationType.WRITE == operation.getType()) { WriteOperation write = (WriteOperation) operation; destinations.add(write.getDestination()); } } }
@Override public int hashCode() { return Objects.hash(super.hashCode(), source, outputs); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!super.equals(o)) { return false; } WriteOperation that = (WriteOperation) o; return Objects.equals(inputs, that.inputs) && Objects.equals(destination, that.destination); }
operationMap.put(operation.getName(), operation); if (OperationType.READ == operation.getType()) { readOperations.add(operation.getName()); switch (operation.getType()) { case READ: incomingOperations.put(operation.getName(), new HashSet<>()); break; case TRANSFORM: inputFields.addAll(write.getInputs()); outgoingOperations.put(operation.getName(), new HashSet<>()); break; outgoings.add(operation.getName()); Set<String> incomings = incomingOperations.computeIfAbsent(operation.getName(), k -> new HashSet<>()); incomings.add(inputField.getOrigin());
/** * Checks whether the given field is used in the next operations or not * * @param nextOperation the next operation which should either be a {@link TransformOperation} or {@link * WriteOperation} * @param inputField the field whose usage needs to be checked * @return true if the field is used in the nextOperation */ private boolean containsInputField(Operation nextOperation, InputField inputField) { Set<InputField> inputFields = new HashSet<>(); if (OperationType.WRITE == nextOperation.getType()) { WriteOperation nextWrite = (WriteOperation) nextOperation; inputFields = new HashSet<>(nextWrite.getInputs()); } else if (OperationType.TRANSFORM == nextOperation.getType()) { TransformOperation nextTransform = (TransformOperation) nextOperation; inputFields = new HashSet<>(nextTransform.getInputs()); } // if the next operation inputFields does contains the given fieldName return true return inputFields.contains(inputField); }
@Override public int hashCode() { return Objects.hash(super.hashCode(), inputs, outputs); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!super.equals(o)) { return false; } ReadOperation that = (ReadOperation) o; return Objects.equals(source, that.source) && Objects.equals(outputs, that.outputs); }
operationMap.put(operation.getName(), operation); if (OperationType.READ == operation.getType()) { readOperations.add(operation.getName()); switch (operation.getType()) { case READ: incomingOperations.put(operation.getName(), new HashSet<>()); break; case TRANSFORM: inputFields.addAll(write.getInputs()); outgoingOperations.put(operation.getName(), new HashSet<>()); break; outgoings.add(operation.getName()); Set<String> incomings = incomingOperations.computeIfAbsent(operation.getName(), k -> new HashSet<>()); incomings.add(inputField.getOrigin());
/** * Checks whether the given field is used in the next operations or not * * @param nextOperation the next operation which should either be a {@link TransformOperation} or {@link * WriteOperation} * @param inputField the field whose usage needs to be checked * @return true if the field is used in the nextOperation */ private boolean containsInputField(Operation nextOperation, InputField inputField) { Set<InputField> inputFields = new HashSet<>(); if (OperationType.WRITE == nextOperation.getType()) { WriteOperation nextWrite = (WriteOperation) nextOperation; inputFields = new HashSet<>(nextWrite.getInputs()); } else if (OperationType.TRANSFORM == nextOperation.getType()) { TransformOperation nextTransform = (TransformOperation) nextOperation; inputFields = new HashSet<>(nextTransform.getInputs()); } // if the next operation inputFields does contains the given fieldName return true return inputFields.contains(inputField); }
Operation previousOperation, Map<EndPointField, Set<EndPointField>> summary) { if (currentOperation.getType() == OperationType.READ) { if (OperationType.WRITE == previousOperation.getType()) { WriteOperation previousWrite = (WriteOperation) previousOperation; inputFields = new HashSet<>(previousWrite.getInputs()); } else if (OperationType.TRANSFORM == previousOperation.getType()) { TransformOperation previousTransform = (TransformOperation) previousOperation; inputFields = new HashSet<>(previousTransform.getInputs()); EndPoint source = read.getSource(); for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) { sourceEndPointFields.add(new EndPointField(source, inputField.getName())); if (currentOperation.getType() == OperationType.TRANSFORM) { TransformOperation transform = (TransformOperation) currentOperation; for (InputField inputField : transform.getInputs()) {
/** * Helper method to compute the outgoing connections * @param currentOperation current operation which needs to evaluated * @param visitedOperations a {@link Set} containing all the operations which has been processed so * far. */ private void computeOutgoing(Operation currentOperation, Set<Operation> visitedOperations) { // mark this operation if not already done if (!visitedOperations.add(currentOperation)) { return; } // base condition: if the current operation is write we have reached the end if (currentOperation.getType() == OperationType.WRITE) { return; } // if this is a transform operation then traverse down to all the outgoing operation from this operation // expanding further the traversal and exploring the operations if (currentOperation.getType() == OperationType.TRANSFORM) { TransformOperation transform = (TransformOperation) currentOperation; Set<Operation> operations = operationOutgoingConnections.get(transform.getName()); for (Operation operation : operations) { computeOutgoing(operation, visitedOperations); } } }