public void addActivity(IActivity activity) { activities.put(activity.getActivityId(), activity); }
entry.getValue().createPushRuntime(ctx, recordDescProvider, partition, nPartitions); operatorNodePushablesBFSOrder.add(opPushable); operatorNodePushables.put(entry.getKey(), opPushable); ActivityId sourceId = channel.getLeft().getLeft().getActivityId(); int outputChannel = channel.getLeft().getRight(); ActivityId destId = channel.getRight().getLeft().getActivityId(); int inputChannel = channel.getRight().getRight(); IOperatorNodePushable sourceOp = operatorNodePushables.get(sourceId); IOperatorNodePushable destOp = operatorNodePushables.get(destId); if (destOp == null) { destOp = channel.getRight().getLeft().createPushRuntime(ctx, recordDescProvider, partition, nPartitions); operatorNodePushablesBFSOrder.add(destOp);
public FeedMetaComputeNodePushable(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions, IOperatorDescriptor coreOperator, FeedConnectionId feedConnectionId, Map<String, String> feedPolicyProperties, FeedMetaOperatorDescriptor feedMetaOperatorDescriptor) throws HyracksDataException { this.ctx = ctx; this.coreOperator = (AbstractUnaryInputUnaryOutputOperatorNodePushable) ((IActivity) coreOperator) .createPushRuntime(ctx, recordDescProvider, partition, nPartitions); this.policyAccessor = new FeedPolicyAccessor(feedPolicyProperties); this.partition = partition; this.connectionId = feedConnectionId; this.feedManager = (ActiveManager) ((INcApplicationContext) ctx.getJobletContext().getServiceContext() .getApplicationContext()).getActiveManager(); this.message = new VSizeFrame(ctx); TaskUtil.put(HyracksConstants.KEY_MESSAGE, message, ctx); this.opDesc = feedMetaOperatorDescriptor; this.recordDescProvider = recordDescProvider; }
task = new Task(joblet, flags, taId, han.getClass().getName(), ncs.getExecutor(), ncs, createInputChannels(td, inputs)); IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount()); List<IPartitionCollector> collectors = new ArrayList<>(); if (inputs != null) {
@Override public void addActivity(IOperatorDescriptor op, IActivity task) { activityOperatorMap.put(task.getActivityId(), op); ActivityId activityId = task.getActivityId(); jag.getActivityMap().put(activityId, task); }
public FeedMetaStoreNodePushable(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions, IOperatorDescriptor coreOperator, FeedConnectionId feedConnectionId, Map<String, String> feedPolicyProperties, FeedMetaOperatorDescriptor feedMetaOperatorDescriptor) throws HyracksDataException { this.ctx = ctx; this.insertOperator = (AbstractUnaryInputUnaryOutputOperatorNodePushable) ((IActivity) coreOperator) .createPushRuntime(ctx, recordDescProvider, partition, nPartitions); this.policyAccessor = new FeedPolicyAccessor(feedPolicyProperties); this.partition = partition; this.connectionId = feedConnectionId; this.feedManager = (ActiveManager) ((INcApplicationContext) ctx.getJobletContext().getServiceContext() .getApplicationContext()).getActiveManager(); this.message = new VSizeFrame(ctx); TaskUtil.put(HyracksConstants.KEY_MESSAGE, message, ctx); this.recordDescProvider = recordDescProvider; this.opDesc = feedMetaOperatorDescriptor; tracer = ctx.getJobletContext().getServiceContext().getTracer(); traceCategory = tracer.getRegistry().get(TraceUtils.STORAGE); }
public ActivityId getProducerActivity(ConnectorDescriptorId cdId) { Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId); return connEdge.getLeft().getLeft().getActivityId(); }
public ActivityId getConsumerActivity(ConnectorDescriptorId cdId) { Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId); return connEdge.getRight().getLeft().getActivityId(); }
public ActivityId getConsumerActivity(ConnectorDescriptorId cdId) { Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId); return connEdge.getRight().getLeft().getActivityId(); }
public ActivityId getProducerActivity(ConnectorDescriptorId cdId) { Pair<Pair<IActivity, Integer>, Pair<IActivity, Integer>> connEdge = connectorActivityMap.get(cdId); return connEdge.getLeft().getLeft().getActivityId(); }
public void connect(IConnectorDescriptor connector, IActivity producerActivity, int producerPort, IActivity consumerActivity, int consumerPort, RecordDescriptor recordDescriptor) { if (!activities.containsKey(producerActivity.getActivityId()) || !activities.containsKey(consumerActivity.getActivityId())) { throw new IllegalStateException("Connected Activities belong to different Activity Clusters: " + producerActivity.getActivityId() + " and " + consumerActivity.getActivityId()); } insertIntoIndexedMap(activityInputMap, consumerActivity.getActivityId(), consumerPort, connector); insertIntoIndexedMap(activityOutputMap, producerActivity.getActivityId(), producerPort, connector); connectorActivityMap.put(connector.getConnectorId(), Pair.of(Pair.of(producerActivity, producerPort), Pair.of(consumerActivity, consumerPort))); connectorRecordDescriptorMap.put(connector.getConnectorId(), recordDescriptor); }
@Override public void addSourceEdge(int operatorInputIndex, IActivity task, int taskInputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding source edge: " + task.getActivityId() + ":" + operatorInputIndex + " -> " + task.getActivityId() + ":" + taskInputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getInputConnectorDescriptor(op, operatorInputIndex); insertIntoIndexedMap(jag.getActivityInputMap(), task.getActivityId(), taskInputIndex, conn); connectorConsumerMap.put(conn.getConnectorId(), Pair.of(task, taskInputIndex)); }
@Override public void addBlockingEdge(IActivity blocker, IActivity blocked) { addToValueSet(jag.getBlocked2BlockerMap(), blocked.getActivityId(), blocker.getActivityId()); }
@Override public void addTargetEdge(int operatorOutputIndex, IActivity task, int taskOutputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding target edge: " + task.getActivityId() + ":" + operatorOutputIndex + " -> " + task.getActivityId() + ":" + taskOutputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getOutputConnectorDescriptor(op, operatorOutputIndex); insertIntoIndexedMap(jag.getActivityOutputMap(), task.getActivityId(), taskOutputIndex, conn); connectorProducerMap.put(conn.getConnectorId(), Pair.of(task, taskOutputIndex)); }
for (IActivity an : activities.values()) { ObjectNode jan = om.createObjectNode(); jan.put("id", an.getActivityId().toString()); jan.put("java-class", an.getClass().getName()); List<IConnectorDescriptor> inputs = activityInputMap.get(an.getActivityId()); if (inputs != null) { ArrayNode jInputs = om.createArrayNode(); List<IConnectorDescriptor> outputs = activityOutputMap.get(an.getActivityId()); if (outputs != null) { ArrayNode jOutputs = om.createArrayNode(); Set<ActivityId> blockers = getBlocked2BlockerMap().get(an.getActivityId()); if (blockers != null) { ArrayNode jDeps = om.createArrayNode();
Map<ActivityId, ActivityId> invertedAid2SuperAidMap = new HashMap<>(); invertedActivitySuperActivityMap .forEach((key, value) -> invertedAid2SuperAidMap.put(key.getActivityId(), value.getActivityId())); Map<ActivityId, Set<ActivityId>> replacedBlocked2BlockerMap = new HashMap<>(); blocked2BlockerMap.forEach((blocked, blockers) -> {
List<IConnectorDescriptor> outputConnectors = activityOutputMap.get(expendingActivity.getActivityId()); if (outputConnectors != null) { for (IConnectorDescriptor outputConn : outputConnectors) { invertedActivitySuperActivityMap, newActivity.getActivityId(), newActivity); producerSuperActivity.setClusterOutputIndex(producerSAPort, producerActivity.getActivityId(), producerPort); consumerSuperActivity.setClusterInputIndex(consumerSAPort, consumerActivity.getActivityId(), consumerPort); acg.getConnectorMap().put(connectorId, newActivityCluster);
toFullClassName = entry.getValue().getClass().getName(); toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); toActivityId = entry.getValue().getActivityId(); toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName));