public AbstractSingleActivityOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity) { super(spec, inputArity, outputArity); activityNodeId = new ActivityId(odId, 0); }
@Override public final void setOperatorId(OperatorDescriptorId id) { super.setOperatorId(id); if (activityNodeId != null && !activityNodeId.getOperatorDescriptorId().equals(odId)) { activityNodeId.setOperatorDescriptorId(odId); } }
@Override public void readFields(DataInput input) throws IOException { activityId = ActivityId.create(input); partition = input.readInt(); } }
public static ActivityId create(DataInput dis) throws IOException { ActivityId activityId = new ActivityId(); activityId.readFields(dis); return activityId; }
private static void writeTaskAttemptId(DataOutputStream dos, TaskAttemptId taId) throws IOException { TaskId tid = taId.getTaskId(); ActivityId aid = tid.getActivityId(); OperatorDescriptorId odId = aid.getOperatorDescriptorId(); dos.writeInt(odId.getId()); dos.writeInt(aid.getLocalId()); dos.writeInt(tid.getPartition()); dos.writeInt(taId.getAttempt()); }
Set<LValueConstraintExpression> lValues = new HashSet<>(); for (ActivityId anId : ac.getActivityMap().keySet()) { lValues.add(new PartitionCountExpression(anId.getOperatorDescriptorId())); int nParts = nPartMap.get(anId.getOperatorDescriptorId()); int[] nInputPartitions = null; List<IConnectorDescriptor> inputs = ac.getActivityInputMap().get(anId); ConnectorDescriptorId cdId = inputs.get(i).getConnectorId(); ActivityId aid = ac.getProducerActivity(cdId); Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId()); nInputPartitions[i] = nPartInt; ConnectorDescriptorId cdId = outputs.get(i).getConnectorId(); ActivityId aid = ac.getConsumerActivity(cdId); Integer nPartInt = nPartMap.get(aid.getOperatorDescriptorId()); nOutputPartitions[i] = nPartInt;
@Override public boolean equals(Object o) { if (!(o instanceof TaskId)) { return false; } TaskId oTaskId = (TaskId) o; return oTaskId.partition == partition && oTaskId.activityId.equals(activityId); }
toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); toActivityId = entry.getValue().getActivityId(); toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName)); fromFullClassName = activity.getClass().getName(); fromClassName = fromFullClassName.substring(fromFullClassName.lastIndexOf('.') + 1); fromNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(fromActivityId.toString()), DotFormatBuilder.StringValue.of(fromActivityId.toString() + "-" + fromClassName)); Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId); if (!activitiesPairedSet.contains(newPair)) { toFullClassName = activity.getClass().getName(); toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName)); Pair<ActivityId, ActivityId> newPair = new ImmutablePair<>(fromActivityId, toActivityId); if (!activitiesPairedSet.contains(newPair)) { toClassName = toFullClassName.substring(toFullClassName.lastIndexOf('.') + 1); toActivityId = entry.getKey(); toNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(toActivityId.toString()), DotFormatBuilder.StringValue.of(toActivityId.toString() + "-" + toClassName)); for (ActivityId blockingActivityId : entry.getValue()) { fromActivityId = blockingActivityId; fromFullClassName = activity.getClass().getName(); fromClassName = fromFullClassName.substring(fromFullClassName.lastIndexOf('.') + 1); fromNode = graphBuilder.createNode(DotFormatBuilder.StringValue.of(fromActivityId.toString()),
@Override public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, ActivityCluster ac, ICCServiceContext serviceCtx) { OperatorDescriptorId consumer = ac.getConsumerActivity(getConnectorId()).getOperatorDescriptorId(); OperatorDescriptorId producer = ac.getProducerActivity(getConnectorId()).getOperatorDescriptorId(); constraintAcceptor.addConstraint( new Constraint(new PartitionCountExpression(consumer), new PartitionCountExpression(producer))); }
@Override protected AbstractExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, IFrameWriter writer, ISorter sorter, List<GeneratedRunFileReader> runs, IBinaryComparator[] comparators, INormalizedKeyComputer nmkComputer, int necessaryFrames) { return new ExternalSortGroupByRunMerger(ctx, sorter, runs, sortFields, recordDescProvider.getInputRecordDescriptor(new ActivityId(odId, SORT_ACTIVITY_ID), 0), partialAggRecordDesc, outputRecordDesc, necessaryFrames, writer, groupFields, nmkComputer, comparators, partialAggregatorFactory, mergeAggregatorFactory, !finalStage); } };
ObjectNode json = om.createObjectNode(); json.put("activity-id", taskAttemptId.getTaskId().getActivityId().toString()); json.put("partition", taskAttemptId.getTaskId().getPartition()); json.put("attempt", taskAttemptId.getAttempt());
taskAttempt.setStatus(TaskAttempt.TaskStatus.INITIALIZED, null); locationMap.put(tid, new PartitionLocationExpression(tid.getActivityId().getOperatorDescriptorId(), tid.getPartition())); taskAttempts.put(tid, taskAttempt); taskAttemptMap.put(nodeId, tads); OperatorDescriptorId opId = tid.getActivityId().getOperatorDescriptorId(); jobRun.registerOperatorLocation(opId, tid.getPartition(), nodeId); ActivityPartitionDetails apd = ts.getActivityPlan().getActivityPartitionDetails();
public static ActivityId parse(String str) { if (str.startsWith("ANID:")) { str = str.substring(5); int idIdx = str.lastIndexOf(':'); return new ActivityId(OperatorDescriptorId.parse(str.substring(0, idIdx)), Integer.parseInt(str.substring(idIdx + 1))); } throw new IllegalArgumentException("Unable to parse: " + str); }
acp.getActivityPlanMap().forEach((key, acPlan) -> { ObjectNode entry = om.createObjectNode(); entry.put("activity-id", key.toString());
@Override public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) throws HyracksDataException { return new ExternalGroupWriteOperatorNodePushable(ctx, new TaskId(new ActivityId(getOperatorId(), AGGREGATE_ACTIVITY_ID), partition), spillableTableFactory, partialRecDesc, outRecDesc, framesLimit, keyFields, firstNormalizerFactory, comparatorFactories, intermediateAggregateFactory); }
@Override public void open() throws HyracksDataException { state = (CollectTaskState) ctx.getStateObject( new TaskId(new ActivityId(getOperatorId(), COLLECT_ACTIVITY_ID), partition)); }
@Override public void contributeActivities(IActivityGraphBuilder builder) { CollectActivity ca = new CollectActivity(new ActivityId(odId, COLLECT_ACTIVITY_ID)); SplitActivity sa = new SplitActivity(new ActivityId(odId, SPLIT_ACTIVITY_ID)); builder.addActivity(this, ca); builder.addSourceEdge(0, ca, 0); builder.addActivity(this, sa); builder.addTargetEdge(0, sa, 0); builder.addBlockingEdge(ca, sa); } }
@Override public void contributeActivities(IActivityGraphBuilder builder) { SortActivity sa = new SortActivity(new ActivityId(odId, SORT_ACTIVITY_ID)); MergeActivity ma = new MergeActivity(new ActivityId(odId, MERGE_ACTIVITY_ID)); builder.addActivity(this, sa); builder.addSourceEdge(0, sa, 0); builder.addActivity(this, ma); builder.addTargetEdge(0, ma, 0); builder.addBlockingEdge(sa, ma); }
private static TaskAttemptId readTaskAttemptId(DataInputStream dis) throws IOException { int odid = dis.readInt(); int aid = dis.readInt(); int partition = dis.readInt(); int attempt = dis.readInt(); TaskAttemptId taId = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(odid), aid), partition), attempt); return taId; }