/** * Return the EndPoint as defined by the provided namespace and name. * * @param namespace the name of the namespace * @param name the name of the EndPoint * @return the EndPoint */ public static EndPoint of(String namespace, String name) { return new EndPoint(namespace, name); }
/** * Creates an instance of an input field. * * @param origin the name of the operation which created this input field * @param name the associated with the input field * @return the {@link InputField} */ public static InputField of(String origin, String name) { return new InputField(origin, name); }
private void populateSourcesAndDestinations() { sources = new HashSet<>(); destinations = new HashSet<>(); for (Operation operation : operations) { if (OperationType.READ == operation.getType()) { ReadOperation read = (ReadOperation) operation; sources.add(read.getSource()); } else if (OperationType.WRITE == operation.getType()) { WriteOperation write = (WriteOperation) operation; destinations.add(write.getDestination()); } } }
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
@Test public void testDisjointBranches() { // read1 -----> write1 // read2 -----> write2 ReadOperation read1 = new ReadOperation("read1", "read descr", EndPoint.of("ns", "input1"), "offset", "body"); WriteOperation write1 = new WriteOperation("write1", "write descr", EndPoint.of("ns", "output"), InputField.of("read1", "offset")); ReadOperation read2 = new ReadOperation("read2", "read descr", EndPoint.of("ns", "input2"), "offset", "body"); WriteOperation write2 = new WriteOperation("write2", "write descr", EndPoint.of("ns", "output"), InputField.of("read2", "offset")); Set<Operation> operations = new LinkedHashSet<>(); operations.add(write1); operations.add(write2); operations.add(read2); operations.add(read1); List<Operation> topologicallySortedOperations = FieldLineageInfo.getTopologicallySortedOperations(operations); assertBefore(topologicallySortedOperations, read1, write1); assertBefore(topologicallySortedOperations, read2, write2); }
/** * Checks whether the given field is used in the next operations or not * * @param nextOperation the next operation which should either be a {@link TransformOperation} or {@link * WriteOperation} * @param inputField the field whose usage needs to be checked * @return true if the field is used in the nextOperation */ private boolean containsInputField(Operation nextOperation, InputField inputField) { Set<InputField> inputFields = new HashSet<>(); if (OperationType.WRITE == nextOperation.getType()) { WriteOperation nextWrite = (WriteOperation) nextOperation; inputFields = new HashSet<>(nextWrite.getInputs()); } else if (OperationType.TRANSFORM == nextOperation.getType()) { TransformOperation nextTransform = (TransformOperation) nextOperation; inputFields = new HashSet<>(nextTransform.getInputs()); } // if the next operation inputFields does contains the given fieldName return true return inputFields.contains(inputField); }
@Test(expected = IllegalArgumentException.class) public void testSelfReferentialOperations() { TransformOperation parse = new TransformOperation("parse", "parse", Arrays.asList(InputField.of("read", "body"), InputField.of("parse", "name")), "name", "address"); FieldLineageInfo.getTopologicallySortedOperations(Collections.singleton(parse)); }
/** * Recursively traverse the graph to calculate the incoming operation. * * @param currentOperation the current operation from which the graph needs to explored * @param visitedOperations all the operations visited so far */ private void getIncomingOperationsForFieldHelper(Operation currentOperation, Set<Operation> visitedOperations) { if (!visitedOperations.add(currentOperation)) { return; } // reached the end of backward traversal if (currentOperation.getType() == OperationType.READ) { return; } // for transform we traverse backward in graph further through the inputs of the transform if (currentOperation.getType() == OperationType.TRANSFORM) { TransformOperation transform = (TransformOperation) currentOperation; for (InputField field : transform.getInputs()) { getIncomingOperationsForFieldHelper(operationsMap.get(field.getOrigin()), visitedOperations); } } }
private void addEndPoint(MDSKey.Builder keyBuilder, EndPoint endPoint) { keyBuilder.add(endPoint.getNamespace()) .add(endPoint.getName()); }
@Override public int hashCode() { return Objects.hash(super.hashCode(), inputs, destination); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!super.equals(o)) { return false; } ReadOperation that = (ReadOperation) o; return Objects.equals(source, that.source) && Objects.equals(outputs, that.outputs); }
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
private void populateSourcesAndDestinations() { sources = new HashSet<>(); destinations = new HashSet<>(); for (Operation operation : operations) { if (OperationType.READ == operation.getType()) { ReadOperation read = (ReadOperation) operation; sources.add(read.getSource()); } else if (OperationType.WRITE == operation.getType()) { WriteOperation write = (WriteOperation) operation; destinations.add(write.getDestination()); } } }
/** * Checks whether the given field is used in the next operations or not * * @param nextOperation the next operation which should either be a {@link TransformOperation} or {@link * WriteOperation} * @param inputField the field whose usage needs to be checked * @return true if the field is used in the nextOperation */ private boolean containsInputField(Operation nextOperation, InputField inputField) { Set<InputField> inputFields = new HashSet<>(); if (OperationType.WRITE == nextOperation.getType()) { WriteOperation nextWrite = (WriteOperation) nextOperation; inputFields = new HashSet<>(nextWrite.getInputs()); } else if (OperationType.TRANSFORM == nextOperation.getType()) { TransformOperation nextTransform = (TransformOperation) nextOperation; inputFields = new HashSet<>(nextTransform.getInputs()); } // if the next operation inputFields does contains the given fieldName return true return inputFields.contains(inputField); }
private void addEndPoint(MDSKey.Builder keyBuilder, EndPoint endPoint) { keyBuilder.add(endPoint.getNamespace()) .add(endPoint.getName()); }
@Override public int hashCode() { return Objects.hash(super.hashCode(), inputs, outputs); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!super.equals(o)) { return false; } TransformOperation that = (TransformOperation) o; return Objects.equals(inputs, that.inputs) && Objects.equals(outputs, that.outputs); }
/** * Return the EndPoint as defined by the provided name. * * @param name the name of the EndPoint * @return the EndPoint */ public static EndPoint of(String name) { return new EndPoint(name); }
@Override public int hashCode() { return Objects.hash(super.hashCode(), source, outputs); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!super.equals(o)) { return false; } WriteOperation that = (WriteOperation) o; return Objects.equals(inputs, that.inputs) && Objects.equals(destination, that.destination); }