public void addConnector(IConnectorDescriptor connector) { connectors.put(connector.getConnectorId(), connector); }
@Override public void visit(IConnectorDescriptor conn) { conn.contributeSchedulingConstraints(acceptor, acg.getConnectorMap().get(conn.getConnectorId()), ccServiceCtx); } });
ConnectorDescriptorId cdId = c.getConnectorId(); ActivityId a2 = ac.getConsumerActivity(cdId); Task[] ac2TaskStates = taskMap.get(a2).getTasks(); if (c.allProducersToAllConsumers()) { for (int i = 0; i < nProducers; ++i) { fanouts[i] = nConsumers; c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap); fanouts[i] = targetBitmap.cardinality();
for (int i = 0; i < inputs.size(); ++i) { IConnectorDescriptor conn = inputs.get(i); IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId()); LOGGER.trace("input: {}: {}", i, conn.getConnectorId()); RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); IPartitionCollector collector = createPartitionCollector(td, partition, task, i, conn, recordDesc, cPolicy); for (int i = 0; i < outputs.size(); ++i) { final IConnectorDescriptor conn = outputs.get(i); RecordDescriptor recordDesc = ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId()); LOGGER.trace("input: {}: {}", i, conn.getConnectorId()); IFrameWriter writer = conn.createPartitioner(task, recordDesc, pwFactory, partition, td.getPartitionCount(), td.getOutputPartitionCounts()[i]); writer = enforce ? EnforceFrameWriter.enforce(writer) : writer;
private IPartitionCollector createPartitionCollector(TaskAttemptDescriptor td, final int partition, Task task, int i, IConnectorDescriptor conn, RecordDescriptor recordDesc, IConnectorPolicy cPolicy) throws HyracksDataException { IPartitionCollector collector = conn.createPartitionCollector(task, recordDesc, partition, td.getInputPartitionCounts()[i], td.getPartitionCount()); if (cPolicy.materializeOnReceiveSide()) { return new ReceiveSideMaterializingCollector(task, ncs.getPartitionManager(), collector, task.getTaskAttemptId(), ncs.getExecutor()); } else { return collector; } }
ConnectorDescriptorId cdId = c.getConnectorId(); ActivityId ac2 = ac.getConsumerActivity(cdId); Task[] ac2TaskStates = activityPlanMap.get(ac2).getTasks(); int nConsumers = ac2TaskStates.length; if (c.allProducersToAllConsumers()) { List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = new ArrayList<>(); for (int j = 0; j < nConsumers; j++) { c.indicateTargetPartitions(nProducers, nConsumers, i, targetBitmap); List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(ac1TaskStates[i].getTaskId());
public int getProducerOutputIndex(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getLeft().getRight(); }
public IOperatorDescriptor getProducer(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getLeft().getLeft(); }
public int getConsumerInputIndex(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getRight().getRight(); }
public IOperatorDescriptor getConsumer(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getRight().getLeft(); }
public RecordDescriptor getConnectorRecordDescriptor(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getLeft().getLeft().getOutputRecordDescriptors()[connInfo.getLeft().getRight()]; }
public void addConnector(IConnectorDescriptor conn) { jag.getConnectorMap().put(conn.getConnectorId(), conn); jag.getConnectorRecordDescriptorMap().put(conn.getConnectorId(), jobSpec.getConnectorRecordDescriptor(conn)); }
@Override public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) { ActivityCluster ac = acg.getActivityMap().get(aid); IConnectorDescriptor conn = ac.getActivityOutputMap().get(aid).get(outputIndex); return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); }
@Override public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) { ActivityCluster ac = acg.getActivityMap().get(aid); IConnectorDescriptor conn = ac.getActivityInputMap().get(aid).get(inputIndex); return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); } };
@Override public String toString() { StringBuilder buffer = new StringBuilder(); opMap.forEach((key, value) -> { buffer.append(key.getId()).append(" : ").append(value.toString()).append("\n"); List<IConnectorDescriptor> inputs = opInputMap.get(key); if (inputs != null && !inputs.isEmpty()) { buffer.append(" Inputs:\n"); for (IConnectorDescriptor c : inputs) { buffer.append(" ").append(c.getConnectorId().getId()).append(" : ").append(c.toString()) .append("\n"); } } List<IConnectorDescriptor> outputs = opOutputMap.get(key); if (outputs != null && !outputs.isEmpty()) { buffer.append(" Outputs:\n"); for (IConnectorDescriptor c : outputs) { buffer.append(" ").append(c.getConnectorId().getId()).append(" : ").append(c.toString()) .append("\n"); } } }); buffer.append("\n").append("Constraints:\n").append(userConstraints); return buffer.toString(); }
public void connect(IConnectorDescriptor conn, IOperatorDescriptor producerOp, int producerPort, IOperatorDescriptor consumerOp, int consumerPort) { insertIntoIndexedMap(opInputMap, consumerOp.getOperatorId(), consumerPort, conn); insertIntoIndexedMap(opOutputMap, producerOp.getOperatorId(), producerPort, conn); connectorOpMap.put(conn.getConnectorId(), Pair.of(Pair.of(producerOp, producerPort), Pair.of(consumerOp, consumerPort))); }
public void connect(IConnectorDescriptor connector, IActivity producerActivity, int producerPort, IActivity consumerActivity, int consumerPort, RecordDescriptor recordDescriptor) { if (!activities.containsKey(producerActivity.getActivityId()) || !activities.containsKey(consumerActivity.getActivityId())) { throw new IllegalStateException("Connected Activities belong to different Activity Clusters: " + producerActivity.getActivityId() + " and " + consumerActivity.getActivityId()); } insertIntoIndexedMap(activityInputMap, consumerActivity.getActivityId(), consumerPort, connector); insertIntoIndexedMap(activityOutputMap, producerActivity.getActivityId(), producerPort, connector); connectorActivityMap.put(connector.getConnectorId(), Pair.of(Pair.of(producerActivity, producerPort), Pair.of(consumerActivity, consumerPort))); connectorRecordDescriptorMap.put(connector.getConnectorId(), recordDescriptor); }
private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, Set<Set<ActivityId>> eqSets) { for (Set<ActivityId> eqSet : eqSets) { for (ActivityId t : eqSet) { List<IConnectorDescriptor> inputList = jag.getActivityInputMap().get(t); if (inputList != null) { for (IConnectorDescriptor conn : inputList) { ActivityId inTask = jag.getProducerActivity(conn.getConnectorId()); if (!eqSet.contains(inTask)) { return Pair.of(t, inTask); } } } List<IConnectorDescriptor> outputList = jag.getActivityOutputMap().get(t); if (outputList != null) { for (IConnectorDescriptor conn : outputList) { ActivityId outTask = jag.getConsumerActivity(conn.getConnectorId()); if (!eqSet.contains(outTask)) { return Pair.of(t, outTask); } } } } } return null; }
private void setPartitionConstraintsTopdown(OperatorDescriptorId opId, Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp) { List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId); AlgebricksPartitionConstraint opConstraint; IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId); if (opInputs != null) { for (IConnectorDescriptor conn : opInputs) { ConnectorDescriptorId cid = conn.getConnectorId(); org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec.getConnectorOperatorMap().get(cid); IOperatorDescriptor src = p.getLeft().getLeft(); TargetConstraint constraint = tgtConstraints.get(conn); if (constraint != null) { if (constraint == TargetConstraint.SAME_COUNT) { opConstraint = partitionConstraintMap.get(opDesc); if (partitionConstraintMap.get(src) == null) { if (opConstraint != null) { partitionConstraintMap.put(src, opConstraint); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, src, opConstraint); } } } } // Post Order DFS setPartitionConstraintsTopdown(src.getOperatorId(), tgtConstraints, opDesc); } } }
@Override public void addSourceEdge(int operatorInputIndex, IActivity task, int taskInputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding source edge: " + task.getActivityId() + ":" + operatorInputIndex + " -> " + task.getActivityId() + ":" + taskInputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getInputConnectorDescriptor(op, operatorInputIndex); insertIntoIndexedMap(jag.getActivityInputMap(), task.getActivityId(), taskInputIndex, conn); connectorConsumerMap.put(conn.getConnectorId(), Pair.of(task, taskInputIndex)); }