if (!(readOperation.getSource().equals(sourceField.getEndPoint()) && readOperation.getOutputs().contains(sourceField.getField()))) { continue; for (Operation outgoingOperation : operationOutgoingConnections.get(readOperation.getName())) { InputField inputField = InputField.of(readOperation.getName(), sourceField.getField()); if (containsInputField(outgoingOperation, inputField)) { computeOutgoing(outgoingOperation, visitedOperations);
case READ: FieldReadOperation read = (FieldReadOperation) fieldOperation; newOperation = new ReadOperation(newOperationName, read.getDescription(), read.getSource(), read.getOutputFields()); currentOperationOutputs.addAll(read.getOutputFields());
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
private void populateSourcesAndDestinations() { sources = new HashSet<>(); destinations = new HashSet<>(); for (Operation operation : operations) { if (OperationType.READ == operation.getType()) { ReadOperation read = (ReadOperation) operation; sources.add(read.getSource()); } else if (OperationType.WRITE == operation.getType()) { WriteOperation write = (WriteOperation) operation; destinations.add(write.getDestination()); } } }
readOutput.add("file_name"); ReadOperation read = new ReadOperation("read", "some read", EndPoint.of("ns1", "endpoint1"), readOutput);
private FieldOperationInfo convertToFieldOperationInfo(Operation operation) { FieldOperationInput inputs = null; FieldOperationOutput outputs = null; switch (operation.getType()) { case READ: ReadOperation read = (ReadOperation) operation; inputs = FieldOperationInput.of(read.getSource()); outputs = FieldOperationOutput.of(read.getOutputs()); break; case TRANSFORM: TransformOperation transform = (TransformOperation) operation; inputs = FieldOperationInput.of(transform.getInputs()); outputs = FieldOperationOutput.of(transform.getOutputs()); break; case WRITE: WriteOperation write = (WriteOperation) operation; inputs = FieldOperationInput.of(write.getInputs()); outputs = FieldOperationOutput.of(write.getDestination()); break; } return new FieldOperationInfo(operation.getName(), operation.getDescription(), inputs, outputs); }
private void populateSourcesAndDestinations() { sources = new HashSet<>(); destinations = new HashSet<>(); for (Operation operation : operations) { if (OperationType.READ == operation.getType()) { ReadOperation read = (ReadOperation) operation; sources.add(read.getSource()); } else if (OperationType.WRITE == operation.getType()) { WriteOperation write = (WriteOperation) operation; destinations.add(write.getDestination()); } } }
@Test(expected = IllegalArgumentException.class) public void testCycle() { EndPoint readEndPoint = EndPoint.of("ns", "file1"); EndPoint writeEndPoint = EndPoint.of("ns", "file2"); ReadOperation read = new ReadOperation("read", "read", readEndPoint, "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parse", Arrays.asList(InputField.of("read", "body"), InputField.of("normalize", "name")), "name", "address"); TransformOperation normalize = new TransformOperation("normalize", "normalize", Collections.singletonList(InputField.of("parse", "name")), "name"); WriteOperation write = new WriteOperation("write", "writing to another file", writeEndPoint, Arrays.asList(InputField.of("normalize", "name"), InputField.of("parse", "address"))); List<Operation> operations = new ArrayList<>(); operations.add(parse); operations.add(read); operations.add(normalize); operations.add(write); FieldLineageInfo.getTopologicallySortedOperations(new HashSet<>(operations)); }
if (!(readOperation.getSource().equals(sourceField.getEndPoint()) && readOperation.getOutputs().contains(sourceField.getField()))) { continue; for (Operation outgoingOperation : operationOutgoingConnections.get(readOperation.getName())) { InputField inputField = InputField.of(readOperation.getName(), sourceField.getField()); if (containsInputField(outgoingOperation, inputField)) { computeOutgoing(outgoingOperation, visitedOperations);
EndPoint source = read.getSource(); for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) {
@Test public void testDisjointBranches() { // read1 -----> write1 // read2 -----> write2 ReadOperation read1 = new ReadOperation("read1", "read descr", EndPoint.of("ns", "input1"), "offset", "body"); WriteOperation write1 = new WriteOperation("write1", "write descr", EndPoint.of("ns", "output"), InputField.of("read1", "offset")); ReadOperation read2 = new ReadOperation("read2", "read descr", EndPoint.of("ns", "input2"), "offset", "body"); WriteOperation write2 = new WriteOperation("write2", "write descr", EndPoint.of("ns", "output"), InputField.of("read2", "offset")); Set<Operation> operations = new LinkedHashSet<>(); operations.add(write1); operations.add(write2); operations.add(read2); operations.add(read1); List<Operation> topologicallySortedOperations = FieldLineageInfo.getTopologicallySortedOperations(operations); assertBefore(topologicallySortedOperations, read1, write1); assertBefore(topologicallySortedOperations, read2, write2); }
EndPoint source = read.getSource(); for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) {
@Test(expected = IllegalArgumentException.class) public void testCycleWithNonExistentOperationNames() { EndPoint readEndPoint = EndPoint.of("ns", "file1"); EndPoint writeEndPoint = EndPoint.of("ns", "file2"); ReadOperation read = new ReadOperation("read", "read", readEndPoint, "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parse", Arrays.asList(InputField.of("read", "body"), InputField.of("normalize", "name"), InputField.of("nop1", "field1")), "name", "address"); TransformOperation normalize = new TransformOperation("normalize", "normalize", Arrays.asList(InputField.of("parse", "name"), InputField.of("nop2", "field2")), "name"); WriteOperation write = new WriteOperation("write", "writing to another file", writeEndPoint, Arrays.asList(InputField.of("normalize", "name"), InputField.of("parse", "address"), InputField.of("nop3", "field3"))); List<Operation> operations = new ArrayList<>(); operations.add(parse); operations.add(read); operations.add(normalize); operations.add(write); FieldLineageInfo.getTopologicallySortedOperations(new HashSet<>(operations)); }
@Test public void testLinearTopologicalSort() { ReadOperation read = new ReadOperation("read", "read descr", EndPoint.of("ns", "input"), "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parse descr", Collections.singletonList(InputField.of("read", "body")), "name",
@Test public void testValidOperations() { ReadOperation read = new ReadOperation("read", "some read", EndPoint.of("endpoint1"), "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parse body", Collections.singletonList(InputField.of("read", "body")),
ReadOperation read = new ReadOperation("read", "read descr", EndPoint.of("ns", "input"), "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parse descr", Collections.singletonList(InputField.of("read", "body")), "name",
EndPoint location = EndPoint.of("ns", "location"); ReadOperation read = new ReadOperation("read", "Reading from file", source, "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parsing body", Collections.singletonList(InputField.of("read", "body")),
@Test public void testInvalidOperations() { ReadOperation read = new ReadOperation("read", "some read", EndPoint.of("endpoint1"), "offset", "body"); TransformOperation parse = new TransformOperation("parse", "parse body", Collections.singletonList(InputField.of("read", "body")),