private Map<EndPoint, Set<String>> computeDestinationFields() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPoint, Set<String>> destinationFields = new HashMap<>(); for (WriteOperation write : this.writeOperations) { Set<String> endPointFields = destinationFields.computeIfAbsent(write.getDestination(), k -> new HashSet<>()); for (InputField field : write.getInputs()) { endPointFields.add(field.getName()); } } return destinationFields; }
private Map<EndPoint, Set<String>> computeDestinationFields() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPoint, Set<String>> destinationFields = new HashMap<>(); for (WriteOperation write : this.writeOperations) { Set<String> endPointFields = destinationFields.computeIfAbsent(write.getDestination(), k -> new HashSet<>()); for (InputField field : write.getInputs()) { endPointFields.add(field.getName()); } } return destinationFields; }
/** * Checks whether the given field is used in the next operations or not * * @param nextOperation the next operation which should either be a {@link TransformOperation} or {@link * WriteOperation} * @param inputField the field whose usage needs to be checked * @return true if the field is used in the nextOperation */ private boolean containsInputField(Operation nextOperation, InputField inputField) { Set<InputField> inputFields = new HashSet<>(); if (OperationType.WRITE == nextOperation.getType()) { WriteOperation nextWrite = (WriteOperation) nextOperation; inputFields = new HashSet<>(nextWrite.getInputs()); } else if (OperationType.TRANSFORM == nextOperation.getType()) { TransformOperation nextTransform = (TransformOperation) nextOperation; inputFields = new HashSet<>(nextTransform.getInputs()); } // if the next operation inputFields does contains the given fieldName return true return inputFields.contains(inputField); }
/** * Checks whether the given field is used in the next operations or not * * @param nextOperation the next operation which should either be a {@link TransformOperation} or {@link * WriteOperation} * @param inputField the field whose usage needs to be checked * @return true if the field is used in the nextOperation */ private boolean containsInputField(Operation nextOperation, InputField inputField) { Set<InputField> inputFields = new HashSet<>(); if (OperationType.WRITE == nextOperation.getType()) { WriteOperation nextWrite = (WriteOperation) nextOperation; inputFields = new HashSet<>(nextWrite.getInputs()); } else if (OperationType.TRANSFORM == nextOperation.getType()) { TransformOperation nextTransform = (TransformOperation) nextOperation; inputFields = new HashSet<>(nextTransform.getInputs()); } // if the next operation inputFields does contains the given fieldName return true return inputFields.contains(inputField); }
origins = write.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
origins = write.getInputs().stream().map(InputField::getOrigin).collect(Collectors.toSet());
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPointField, Set<EndPointField>> summary = new HashMap<>(); for (WriteOperation write : writeOperations) { List<InputField> inputs = write.getInputs(); for (InputField input : inputs) { computeIncomingSummaryHelper(new EndPointField(write.getDestination(), input.getName()), operationsMap.get(input.getOrigin()), write, summary); } } return summary; }
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPointField, Set<EndPointField>> summary = new HashMap<>(); for (WriteOperation write : writeOperations) { List<InputField> inputs = write.getInputs(); for (InputField input : inputs) { computeIncomingSummaryHelper(new EndPointField(write.getDestination(), input.getName()), operationsMap.get(input.getOrigin()), write, summary); } } return summary; }
case WRITE: WriteOperation write = (WriteOperation) operation; inputFields.addAll(write.getInputs());
case WRITE: WriteOperation write = (WriteOperation) operation; inputFields.addAll(write.getInputs());
write.getInputs().stream().filter(input -> input.getName().equals(destinationField.getField())) .collect(Collectors.toSet());
write.getInputs().stream().filter(input -> input.getName().equals(destinationField.getField())) .collect(Collectors.toSet());
if (OperationType.WRITE == previousOperation.getType()) { WriteOperation previousWrite = (WriteOperation) previousOperation; inputFields = new HashSet<>(previousWrite.getInputs()); } else if (OperationType.TRANSFORM == previousOperation.getType()) { TransformOperation previousTransform = (TransformOperation) previousOperation;
if (OperationType.WRITE == previousOperation.getType()) { WriteOperation previousWrite = (WriteOperation) previousOperation; inputFields = new HashSet<>(previousWrite.getInputs()); } else if (OperationType.TRANSFORM == previousOperation.getType()) { TransformOperation previousTransform = (TransformOperation) previousOperation;
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }