protected void registerNode(Node n) { _graph.addVertex(n); if(n.stateInfo!=null) { String id = n.stateInfo.id; if(!_colocate.containsKey(id)) { _colocate.put(id, new ArrayList()); } _colocate.get(id).add(n); } }
@Override public void populateBlackListedEdges(DirectedWeightedMultigraph<ServiceNode, FlowEdge> graph) { for (ServiceNode node: serviceNodes) { if (graph.containsVertex(node)) { blacklistedEdges.addAll(graph.incomingEdgesOf(node)); blacklistedEdges.addAll(graph.outgoingEdgesOf(node)); } else { log.info("The graph " + graph + " doesn't contains node " + node.toString()); } } for( FlowEdge flowEdge: flowEdges) { if (graph.containsEdge(flowEdge)) { blacklistedEdges.add(flowEdge); } else { log.info("The graph " + graph + "doesn't contains edge " + flowEdge.toString()); } } } }
protected void registerSourcedNode(List<Stream> sources, Node newNode) { registerNode(newNode); int streamIndex = 0; for(Stream s: sources) { _graph.addEdge(s._node, newNode, new IndexedEdge(s._node, newNode, streamIndex)); streamIndex++; } }
public TridentTopology() { this(new DefaultDirectedGraph<Node, IndexedEdge>(new ErrorEdgeFactory()), new LinkedHashMap<String, List<Node>>(), new UniqueIdGen()); }
DefaultDirectedGraph<Node, IndexedEdge> graph = (DefaultDirectedGraph) _graph.clone(); for(Node n: graph.vertexSet()) { if(n instanceof SpoutNode) { spoutNodes.add((SpoutNode) n); for(IndexedEdge<Node> e: new HashSet<>(graph.edgeSet())) { if(!(e.source instanceof PartitionNode) && !(e.target instanceof PartitionNode)) { Group g1 = grouper.nodeGroup(e.source); throw new RuntimeException("Planner exception: Null source group must indicate a spout node at this phase of planning"); if(g1==null || !g1.equals(g2)) { graph.removeEdge(e); PartitionNode pNode = makeIdentityPartition(e.source); graph.addVertex(pNode); graph.addEdge(e.source, pNode, new IndexedEdge(e.source, pNode, 0)); graph.addEdge(pNode, e.target, new IndexedEdge(pNode, e.target, e.index)); Node newPartitionNode = new PartitionNode(idNode.streamId, n.name, idNode.allOutputFields, n.thriftGrouping); Node parentNode = TridentUtils.getParent(graph, n); Set<IndexedEdge> outgoing = graph.outgoingEdgesOf(n); graph.removeVertex(n); graph.addVertex(idNode); graph.addVertex(newPartitionNode); addEdge(graph, parentNode, idNode, 0); addEdge(graph, idNode, newPartitionNode, 0);
private void weightGraphGenerateHelper(TopologySpec topologySpec) { try { Map<ServiceNode, ServiceNode> capabilities = topologySpec.getSpecExecutor().getCapabilities().get(); for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) { BaseServiceNodeImpl sourceNode = new BaseServiceNodeImpl(capability.getKey().getNodeName()); BaseServiceNodeImpl targetNode = new BaseServiceNodeImpl(capability.getValue().getNodeName()); if (!weightedGraph.containsVertex(sourceNode)) { weightedGraph.addVertex(sourceNode); } if (!weightedGraph.containsVertex(targetNode)) { weightedGraph.addVertex(targetNode); } FlowEdge flowEdge = new LoadBasedFlowEdgeImpl(sourceNode, targetNode, defaultFlowEdgeProps, topologySpec.getSpecExecutor()); // In Multi-Graph if flowEdge existed, just skip it. if (!weightedGraph.containsEdge(flowEdge)) { weightedGraph.addEdge(sourceNode, targetNode, flowEdge); } } } catch (InterruptedException | ExecutionException e) { Instrumented.markMeter(this.flowCompilationFailedMeter); throw new RuntimeException("Cannot determine topology capabilities", e); } }
private boolean userSpecifiedPathVerificator(Map<Spec, SpecExecutor> specExecutorInstanceMap, FlowSpec flowSpec) { Map<Spec, SpecExecutor> tmpSpecExecutorInstanceMap = new HashMap<>(); List<String> userSpecfiedPath = Arrays.asList(optionalUserSpecifiedPath.get().split(",")); for (int i = 0; i < userSpecfiedPath.size() - 1; i++) { ServiceNode sourceNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i)); ServiceNode targetNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i + 1)); if (weightedGraph.containsVertex(sourceNode) && weightedGraph.containsVertex(targetNode) && weightedGraph.containsEdge(sourceNode, targetNode)) { tmpSpecExecutorInstanceMap.put(convertHopToJobSpec(sourceNode, targetNode, flowSpec), (((LoadBasedFlowEdgeImpl) weightedGraph.getEdge(sourceNode, targetNode)).getSpecExecutorInstance())); } else { log.error("User Specified Path is invalid"); return false; } } specExecutorInstanceMap.putAll(tmpSpecExecutorInstanceMap); return true; }
/** * Retrieves array of simple class names of nodes within a particular flow step. * * @param step step in main flow. * @return list step's graph's vertex names. */ protected String[] getNodeFeatures(BaseFlowStep step) { SimpleDirectedGraph graph = step.getGraph(); Object[] vertices = graph.vertexSet().toArray(); String[] returnedFeatures = new String[vertices.length]; for (int i = 0; i < returnedFeatures.length; i++) { returnedFeatures[i] = vertices[i].getClass().getSimpleName(); } return returnedFeatures; }
private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) { UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class); for(Group g: groups) { equivs.addVertex(g);
DirectedSubgraph<Node, Object> subgraph = new DirectedSubgraph(_graph, _nodes, null); TopologicalOrderIterator it = new TopologicalOrderIterator<Node, Object>(subgraph); int stateIndex = 0;
/** * As the base implementation here, all templates will be considered for each edge. */ @Override protected void populateEdgeTemplateMap() { if (templateCatalog.isPresent()) { for (FlowEdge flowEdge : this.weightedGraph.edgeSet()) { edgeTemplateMap.put(flowEdge.getEdgeIdentity(), templateCatalog.get(). getAllTemplates(). stream().map(jobTemplate -> jobTemplate.getUri()).collect(Collectors.toList())); } } }
Set<FlowEdge> outgoingEdges = weightedGraph.outgoingEdgesOf(node.getNode()); for (FlowEdge outGoingEdge:outgoingEdges) { DistancedNode adjacentNode = new DistancedNode(weightedGraph.getEdgeTarget(outGoingEdge)); if (shortestDist.containsKey(adjacentNode)) { adjacentNode.setDistToSrc(shortestDist.get(adjacentNode));
@Test (dependsOnMethods = "testWeightedGraphConstruction") public void testDijkstraPathFinding(){ FlowSpec flowSpec = initFlowSpec(); TopologySpec topologySpec_1 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME); TopologySpec topologySpec_2 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR_SECOND, TEST_SOURCE_NAME, TEST_HOP_NAME_B, TEST_HOP_NAME_C, TEST_SINK_NAME); this.compilerWithTemplateCalague.onAddSpec(topologySpec_1); this.compilerWithTemplateCalague.onAddSpec(topologySpec_2); // Get the edge -> Change the weight -> Materialized the edge change back to graph -> compile again -> Assertion this.compilerWithTemplateCalague.compileFlow(flowSpec); DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph(); FlowEdge a2b= weightedGraph.getEdge(vertexHopA, vertexHopB); FlowEdge b2c = weightedGraph.getEdge(vertexHopB, vertexHopC); FlowEdge c2s = weightedGraph.getEdge(vertexHopC, vertexSink); weightedGraph.setEdgeWeight(a2b, 1.99); weightedGraph.setEdgeWeight(b2c, 0.1); weightedGraph.setEdgeWeight(c2s, 0.2); // Best route: Src - B(1) - C(0.1) - sink (0.2) this.compilerWithTemplateCalague.compileFlow(flowSpec); List<FlowEdge> edgeList = dijkstraBasedPathFindingHelper(vertexSource, vertexSink, weightedGraph); FlowEdge src2b = weightedGraph.getEdge(vertexSource, vertexHopB); FlowEdge b2C = weightedGraph.getEdge(vertexHopB, vertexHopC); FlowEdge c2sink = weightedGraph.getEdge(vertexHopC, vertexSink); Assert.assertEquals(edgeList.get(0).getEdgeIdentity(), src2b.getEdgeIdentity()); Assert.assertEquals(edgeList.get(1).getEdgeIdentity(), b2C.getEdgeIdentity()); Assert.assertEquals(edgeList.get(2).getEdgeIdentity(), c2sink.getEdgeIdentity()); this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_1.getUri(), ""); this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_2.getUri(), ""); }
/** * @return Transform a set of {@link TopologySpec} into a instance of {@link org.jgrapht.graph.WeightedMultigraph} * and filter out connections between blacklisted vertices that user specified. * The output of this function only stays in memory, so each time a logical flow is compiled, the multigraph will * be re-calculated. * */ private void inMemoryWeightGraphGenerator() { for (TopologySpec topologySpec : topologySpecMap.values()) { weightGraphGenerateHelper(topologySpec); } // Filter out connection appearing in servicePolicy. // This is where servicePolicy is enforced. servicePolicy.populateBlackListedEdges(this.weightedGraph); if (servicePolicy.getBlacklistedEdges().size() > 0) { for (FlowEdge toDeletedEdge : servicePolicy.getBlacklistedEdges()) { weightedGraph.removeEdge(toDeletedEdge); } } }
/** * A naive implementation of resolving templates in each JobSpec among Multi-hop FlowSpec. * Handle the case when edge is not specified. * Always select the first available template. */ private JobSpec convertHopToJobSpec (ServiceNode sourceNode, ServiceNode targetNode, FlowSpec flowSpec) { FlowEdge flowEdge = weightedGraph.getAllEdges(sourceNode, targetNode).iterator().next(); URI templateURI = getTemplateURI (sourceNode, targetNode, flowSpec, flowEdge); return buildJobSpec(sourceNode, targetNode, templateURI, flowSpec); }
private void populateTemplateMap(WeightedMultigraph<ServiceNode, FlowEdge> weightedGraph, URI exempliedURI){ this.edgeTemplateMap.clear(); Set<FlowEdge> allEdges = weightedGraph.edgeSet(); for ( FlowEdge edge : allEdges ) { this.edgeTemplateMap.put(edge.getEdgeIdentity(), Arrays.asList(exempliedURI)) ; } }
DirectedGraph<PlanFragmentId, DefaultEdge> graph = new DefaultDirectedGraph<>(DefaultEdge.class); fragments.forEach(fragment -> graph.addVertex(fragment.getId())); DirectedGraph<Set<PlanFragmentId>, DefaultEdge> componentGraph = new DefaultDirectedGraph<>(DefaultEdge.class); components.forEach(componentGraph::addVertex); for (DefaultEdge edge : graph.edgeSet()) {
@Test public void testWeightedGraphConstruction(){ FlowSpec flowSpec = initFlowSpec(); TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME); this.compilerWithTemplateCalague.onAddSpec(topologySpec); // invocation of compileFlow trigger the weighedGraph construction this.compilerWithTemplateCalague.compileFlow(flowSpec); DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph(); Assert.assertTrue(weightedGraph.containsVertex(vertexSource)); Assert.assertTrue(weightedGraph.containsVertex(vertexHopA)); Assert.assertTrue(weightedGraph.containsVertex(vertexHopB)); Assert.assertTrue(weightedGraph.containsVertex(vertexSink)); FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor()); FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor()); FlowEdge edgeB2Sink = new LoadBasedFlowEdgeImpl(vertexHopB, vertexSink, topologySpec.getSpecExecutor()); Assert.assertTrue(weightedGraph.containsEdge(edgeSrc2A)); Assert.assertTrue(weightedGraph.containsEdge(edgeA2B)); Assert.assertTrue(weightedGraph.containsEdge(edgeB2Sink)); Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexSource, vertexHopA), edgeSrc2A)); Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopA, vertexHopB), edgeA2B)); Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopB, vertexSink), edgeB2Sink)); this.compilerWithTemplateCalague.onDeleteSpec(topologySpec.getUri(), ""); }
/** * Converts the flowStep that generated from cascading to a Map of DAGNode and its name to be used * to build Ambrose Graph. */ public void convert() { // returns a set of the nodes contained in this graph Set vertices = jobsGraph.vertexSet(); // create ambrose nodes for (Object vertex : vertices) { BaseFlowStep step = (BaseFlowStep) vertex; CascadingJob job = new CascadingJob(); job.setFeatures(getNodeFeatures(step)); String name = step.getName(); DAGNode<CascadingJob> node = new DAGNode<CascadingJob>(name, job); dagNamesMap.put(name, node); } // loop again to set the successors for each node after nodes are created for (Object vertex : vertices) { BaseFlowStep step = (BaseFlowStep) vertex; String name = step.getName(); DAGNode<CascadingJob> node = dagNamesMap.get(name); node.setSuccessors(getNodeSuccessors(vertex)); } }
DirectedGraph graph = new DefaultDirectedGraph<BaseFlowStep, FlowGraphEdge>( new EdgeFactory<BaseFlowStep, FlowGraphEdge>() { @Override