@Override public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) { ActivityCluster ac = acg.getActivityMap().get(aid); IConnectorDescriptor conn = ac.getActivityInputMap().get(aid).get(inputIndex); return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); } };
/** * rewrite an activity cluster graph to eliminate * all one-to-one connections and merge one-to-one connected * DAGs into super activities. * * @param acg * the activity cluster graph */ public void rewrite(ActivityClusterGraph acg) { acg.getActivityMap().clear(); acg.getConnectorMap().clear(); Map<IActivity, SuperActivity> invertedActivitySuperActivityMap = new HashMap<>(); acg.getActivityClusterMap() .forEach((key, value) -> rewriteIntraActivityCluster(value, invertedActivitySuperActivityMap)); acg.getActivityClusterMap() .forEach((key, value) -> rewriteInterActivityCluster(value, invertedActivitySuperActivityMap)); invertedActivitySuperActivityMap.clear(); }
private ObjectNode createJobLogObject(final JobRun run) { ObjectMapper om = new ObjectMapper(); ObjectNode jobLogObject = om.createObjectNode(); ActivityClusterGraph acg = run.getActivityClusterGraph(); jobLogObject.set("activity-cluster-graph", acg.toJSON()); jobLogObject.set("job-run", run.toJSON()); return jobLogObject; }
ActivityClusterGraph acg = new ActivityClusterGraph(); Map<ActivityId, ActivityCluster> acMap = new HashMap<ActivityId, ActivityCluster>(); int acCounter = 0; acg.addActivityClusters(acList); LOGGER.trace(acg.toJSON().asText());
private void startRunnableActivityClusters() throws HyracksException { Set<TaskCluster> taskClusterRoots = new HashSet<>(); findRunnableTaskClusterRoots(taskClusterRoots, jobRun.getActivityClusterGraph().getActivityClusterMap().values()); if (LOGGER.isTraceEnabled()) { LOGGER.log(Level.TRACE, "Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: " + inProgressTaskClusters); } if (taskClusterRoots.isEmpty() && inProgressTaskClusters.isEmpty()) { ccs.getWorkQueue().schedule(new JobCleanupWork(ccs.getJobManager(), jobRun.getJobId(), JobStatus.TERMINATED, null, NoOpCallback.INSTANCE)); return; } startRunnableTaskClusters(taskClusterRoots); }
public Joblet(NodeControllerService nodeController, DeploymentId deploymentId, JobId jobId, INCServiceContext serviceCtx, ActivityClusterGraph acg, IJobletEventListenerFactory jobletEventListenerFactory, long jobStartTime) { this.nodeController = nodeController; this.serviceCtx = serviceCtx; this.deploymentId = deploymentId; this.jobId = jobId; this.frameManager = new FrameManager(acg.getFrameSize()); memoryAllocation = new AtomicLong(); this.acg = acg; partitionRequestMap = new HashMap<>(); env = new OperatorEnvironmentImpl(nodeController.getId()); stateObjectMap = new HashMap<>(); taskMap = new HashMap<>(); counterMap = new HashMap<>(); deallocatableRegistry = new DefaultDeallocatableRegistry(); fileFactory = new WorkspaceFileFactory(this, serviceCtx.getIoManager()); cleanupPending = false; this.jobletEventListenerFactory = jobletEventListenerFactory; if (jobletEventListenerFactory != null) { IJobletEventListener listener = jobletEventListenerFactory.createListener(this); this.jobletEventListener = listener; listener.jobletStart(); } else { jobletEventListener = null; } IGlobalJobDataFactory gjdf = acg.getGlobalJobDataFactory(); globalJobData = gjdf != null ? gjdf.createGlobalJobData(this) : null; this.jobStartTime = jobStartTime; }
TaskId tid = taid.getTaskId(); ActivityId aid = tid.getActivityId(); List<IConnectorDescriptor> inConnectors = acg.getActivityInputs(aid); int[] inPartitionCounts = tad.getInputPartitionCounts(); if (inPartitionCounts == null) { continue; ActivityId producerAid = acg.getProducerActivity(cdId); partitionLocations[i] = new NetworkAddress[inPartitionCounts[i]]; for (int j = 0; j < inPartitionCounts[i]; ++j) {
@Override public void visit(IConnectorDescriptor conn) { conn.contributeSchedulingConstraints(acceptor, acg.getConnectorMap().get(conn.getConnectorId()), ccServiceCtx); } });
private IConnectorPolicy assignConnectorPolicy(ActivityCluster ac, IConnectorDescriptor c, int nProducers, int nConsumers, int[] fanouts) { IConnectorPolicyAssignmentPolicy cpap = ac.getConnectorPolicyAssignmentPolicy(); if (cpap != null) { return cpap.getConnectorPolicyAssignment(c, nProducers, nConsumers, fanouts); } cpap = ac.getActivityClusterGraph().getConnectorPolicyAssignmentPolicy(); if (cpap != null) { return cpap.getConnectorPolicyAssignment(c, nProducers, nConsumers, fanouts); } return new PipeliningConnectorPolicy(); }
for (Entry<ActivityClusterId, ActivityCluster> entry : acg.getActivityClusterMap().entrySet()) { ActivityCluster ac = entry.getValue(); for (Entry<ActivityId, IActivity> saEntry : ac.getActivityMap().entrySet()) {
@Override public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) { ActivityCluster ac = acg.getActivityMap().get(aid); IConnectorDescriptor conn = ac.getActivityOutputMap().get(aid).get(outputIndex); return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); }
superActivities.forEach((key, value) -> { newActivityCluster.addActivity(value); acg.getActivityMap().put(key, newActivityCluster); }); consumerSuperActivity.setClusterInputIndex(consumerSAPort, consumerActivity.getActivityId(), consumerPort); acg.getConnectorMap().put(connectorId, newActivityCluster); acg.getActivityClusterMap().put(ac.getId(), newActivityCluster);
for (Entry<ActivityClusterId, ActivityCluster> entry : acg.getActivityClusterMap().entrySet()) { ActivityCluster ac = entry.getValue(); for (Entry<ActivityId, IActivity> saEntry : ac.getActivityMap().entrySet()) {
@Override protected void doRun() throws Exception { IJobManager jobManager = ccs.getJobManager(); ObjectMapper om = new ObjectMapper(); JobRun run = jobManager.get(jobId); if (run == null) { json = om.createObjectNode(); return; } json = run.getActivityClusterGraph().toJSON(); }
replacedBlockers.add(invertedAid2SuperAidMap.get(blocker)); ActivityCluster dependingAc = ac.getActivityClusterGraph().getActivityMap().get(invertedAid2SuperAidMap.get(blocker)); if (!ac.getDependencies().contains(dependingAc)) { ac.getDependencies().add(dependingAc);
for (ActivityCluster ac : acg.getActivityClusterMap().values()) { ObjectNode acJSON = om.createObjectNode();
private String findTaskLocation(TaskId tid) { ActivityId aid = tid.getActivityId(); ActivityCluster ac = jobRun.getActivityClusterGraph().getActivityMap().get(aid); Task[] tasks = getActivityClusterPlan(ac).getActivityPlanMap().get(aid).getTasks(); List<TaskClusterAttempt> tcAttempts = tasks[tid.getPartition()].getTaskCluster().getAttempts(); if (tcAttempts == null || tcAttempts.isEmpty()) { return null; } TaskClusterAttempt lastTCA = tcAttempts.get(tcAttempts.size() - 1); TaskAttempt ta = lastTCA.getTaskAttempts().get(tid); return ta == null ? null : ta.getNodeId(); }
for (ActivityCluster ac : jobRun.getActivityClusterGraph().getActivityClusterMap().values()) { if (!isPlanned(ac)) { continue;
BitSet targetBitmap = new BitSet(); for (ActivityId ac1 : activities) { ActivityCluster ac = acg.getActivityMap().get(ac1); Task[] ac1TaskStates = activityPlanMap.get(ac1).getTasks(); int nProducers = ac1TaskStates.length;
tasks[i] = new Task(tid, activityPlan); for (ActivityId danId : depAnIds) { ActivityCluster dAC = ac.getActivityClusterGraph().getActivityMap().get(danId); ActivityClusterPlan dACP = jobRun.getActivityClusterPlanMap().get(dAC.getId()); assert dACP != null : "IllegalStateEncountered: Dependent AC is being planned without a plan for "