private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, Set<Set<ActivityId>> eqSets) { for (Set<ActivityId> eqSet : eqSets) { for (ActivityId t : eqSet) { List<IConnectorDescriptor> inputList = jag.getActivityInputMap().get(t); if (inputList != null) { for (IConnectorDescriptor conn : inputList) { ActivityId inTask = jag.getProducerActivity(conn.getConnectorId()); if (!eqSet.contains(inTask)) { return Pair.of(t, inTask); } } } List<IConnectorDescriptor> outputList = jag.getActivityOutputMap().get(t); if (outputList != null) { for (IConnectorDescriptor conn : outputList) { ActivityId outTask = jag.getConsumerActivity(conn.getConnectorId()); if (!eqSet.contains(outTask)) { return Pair.of(t, outTask); } } } } } return null; }
@Override public void addActivity(IOperatorDescriptor op, IActivity task) { activityOperatorMap.put(task.getActivityId(), op); ActivityId activityId = task.getActivityId(); jag.getActivityMap().put(activityId, task); }
public JobActivityGraphBuilder(JobSpecification jobSpec, Set<JobFlag> jobFlags) { activityOperatorMap = new HashMap<>(); jag = new JobActivityGraph(); this.jobSpec = jobSpec; connectorProducerMap = new HashMap<>(); connectorConsumerMap = new HashMap<>(); }
public void addConnector(IConnectorDescriptor conn) { jag.getConnectorMap().put(conn.getConnectorId(), conn); jag.getConnectorRecordDescriptorMap().put(conn.getConnectorId(), jobSpec.getConnectorRecordDescriptor(conn)); }
DotFormatBuilder.Node toNode; final Set<Pair<ActivityId, ActivityId>> activitiesPairedSet = new HashSet<>(); final Map<ActivityId, IActivity> activityMap = jobActivityGraph.getActivityMap(); final Map<ActivityId, List<IConnectorDescriptor>> activityInputMap = jobActivityGraph.getActivityInputMap(); final Map<ActivityId, List<IConnectorDescriptor>> activityOutputMap = jobActivityGraph.getActivityOutputMap(); if (connectors != null) { for (IConnectorDescriptor connector : connectors) { fromActivityId = jobActivityGraph.getProducerActivity(connector.getConnectorId()); activity = activityMap.get(fromActivityId); fromFullClassName = activity.getClass().getName(); if (connectors != null) { for (IConnectorDescriptor connector : connectors) { toActivityId = jobActivityGraph.getConsumerActivity(connector.getConnectorId()); activity = activityMap.get(toActivityId); toFullClassName = activity.getClass().getName(); final Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jobActivityGraph.getBlocked2BlockerMap(); IActivity blockedActivity; for (Map.Entry<ActivityId, Set<ActivityId>> entry : blocked2BlockerMap.entrySet()) {
for (ActivityId taskId : jag.getActivityMap().keySet()) { Set<ActivityId> eqSet = new HashSet<ActivityId>(); eqSet.add(taskId); Map<ActivityId, ActivityCluster> acMap = new HashMap<ActivityId, ActivityCluster>(); int acCounter = 0; Map<ActivityId, IActivity> activityNodeMap = jag.getActivityMap(); List<ActivityCluster> acList = new ArrayList<ActivityCluster>(); for (Set<ActivityId> stage : stages) { IActivity activity = activityNodeMap.get(aid); ActivityCluster ac = acMap.get(aid); List<IConnectorDescriptor> aOutputs = jag.getActivityOutputMap().get(aid); if (aOutputs == null || aOutputs.isEmpty()) { ac.addRoot(activity); ac.addConnector(conn); Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> pcPair = jag.getConnectorActivityMap().get(conn.getConnectorId()); ac.connect(conn, activity, i, pcPair.getRight().getLeft(), pcPair.getRight().getRight(), jag.getConnectorRecordDescriptorMap().get(conn.getConnectorId())); Map<ActivityId, Set<ActivityId>> blocked2BlockerMap = jag.getBlocked2BlockerMap(); for (ActivityCluster s : acList) { Map<ActivityId, Set<ActivityId>> acBlocked2BlockerMap = s.getBlocked2BlockerMap();
@Override public void addBlockingEdge(IActivity blocker, IActivity blocked) { addToValueSet(jag.getBlocked2BlockerMap(), blocked.getActivityId(), blocker.getActivityId()); }
@Override public void addTargetEdge(int operatorOutputIndex, IActivity task, int taskOutputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding target edge: " + task.getActivityId() + ":" + operatorOutputIndex + " -> " + task.getActivityId() + ":" + taskOutputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getOutputConnectorDescriptor(op, operatorOutputIndex); insertIntoIndexedMap(jag.getActivityOutputMap(), task.getActivityId(), taskOutputIndex, conn); connectorProducerMap.put(conn.getConnectorId(), Pair.of(task, taskOutputIndex)); }
@Override public void addSourceEdge(int operatorInputIndex, IActivity task, int taskInputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding source edge: " + task.getActivityId() + ":" + operatorInputIndex + " -> " + task.getActivityId() + ":" + taskInputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getInputConnectorDescriptor(op, operatorInputIndex); insertIntoIndexedMap(jag.getActivityInputMap(), task.getActivityId(), taskInputIndex, conn); connectorConsumerMap.put(conn.getConnectorId(), Pair.of(task, taskInputIndex)); }
public void finish() { Map<ConnectorDescriptorId, Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>>> caMap = jag.getConnectorActivityMap(); connectorProducerMap .forEach((cdId, producer) -> caMap.put(cdId, Pair.of(producer, connectorConsumerMap.get(cdId)))); }