@Override public boolean handleError() { currentActivityIndex++; if (currentActivityIndex == jobEvent.getActivities().size()) return false; List<Activity<?>> newActivityList = new ArrayList<>(); newActivityList.add(jobEvent.getActivities().get( currentActivityIndex)); getBelow().receiveJob( new DispatchJobEvent(jobEvent.getOwningProcess(), jobEvent .getIndex(), jobEvent.getContext(), jobEvent .getData(), newActivityList)); return true; } }
@Override public DispatchJobEvent popOwningProcess() throws ProcessIdentifierException { return new DispatchJobEvent(popOwner(), index, context, dataMap, activities); }
@Override public DispatchJobEvent pushOwningProcess(String localProcessName) throws ProcessIdentifierException { return new DispatchJobEvent(pushOwner(localProcessName), index, context, dataMap, activities); }
@Override public void fail(String message, Throwable t, DispatchErrorType errorType) { logger.warn("Failed (" + errorType + ") invoking " + activity + " for job " + jobEvent + ": " + message, t); monMan.deregisterNode( invocationProcessIdentifier); getAbove().receiveError( new DispatchErrorEvent(jobEvent.getOwningProcess(), jobEvent.getIndex(), jobEvent.getContext(), message, t, errorType, activity)); }
protected void invoke(final DispatchJobEvent jobEvent, final AsynchronousActivity<?> activity) { final String invocationProcessIdentifier = jobEvent.pushOwningProcess( getNextProcessID()).getOwningProcess(); monMan.registerNode(activity, invocationProcessIdentifier, new HashSet<MonitorableProperty<?>>()); InvocationContext context = jobEvent.getContext(); final ReferenceService refService = context.getReferenceService(); invocationItem.setIdentifier(UUID.randomUUID().toString()); invocationItem.setActivity(activity); invocationItem.setProcessId(jobEvent.getOwningProcess()); invocationItem.setInvocationProcessId(invocationProcessIdentifier); invocationItem.setParentId(parentItem.getIdentifier()); for (String inputName : jobEvent.getData().keySet()) { String activityInputName = activity .getInputPortMapping().get(inputName); if (activityInputName != null) inputData.put(activityInputName, jobEvent.getData() .get(inputName));
private DispatchJobEvent prepareNewJobEvent( Map<String, T2Reference> data, AbstractDispatchEvent dispatchEvent) { DispatchJobEvent dispatchJobEvent = (DispatchJobEvent) dispatchEvent; Map<String, T2Reference> newInputs = new HashMap<String, T2Reference>( dispatchJobEvent.getData()); newInputs.putAll(data); DispatchJobEvent newJobEvent = new DispatchJobEvent(dispatchEvent .getOwningProcess(), dispatchEvent.getIndex(), dispatchEvent.getContext(), newInputs, ((DispatchJobEvent) dispatchEvent).getActivities()); /* * TODO: Should this be registered as an incomingJobs? If so the * conditional could even feed to itself, and we should also keep a * list of originalJobs. */ return newJobEvent; }
/** * If the job contains errors, or collections which contain errors * themselves then bounce a result message with error documents in back up * to the layer above */ @Override public void receiveJob(DispatchJobEvent jobEvent) { Set<T2Reference> errorReferences = new HashSet<>(); for (T2Reference ei : jobEvent.getData().values()) if (ei.containsErrors()) errorReferences.add(ei); if (errorReferences.isEmpty()) // relay the message down... getBelow().receiveJob(jobEvent); else { getState(jobEvent.getOwningProcess()).incrementErrorsReflected(); sendErrorOutput(jobEvent, null, errorReferences); } }
/** * Remove the specified pending retry job from the cache * * @param owningProcess * Owning process identifier as returned by * {@link DispatchJobEvent#getOwningProcess()} * @param index * Index of the job as returned by * {@link DispatchJobEvent#getIndex()} */ protected void forget(String owningProcess, int[] index) { for (JobState jobState : getJobsCopy(owningProcess)) if (identicalIndex(jobState.jobEvent.getIndex(), index)) { removeJob(owningProcess, jobState); return; } // It could be due to pipelining activities like BioMart logger.debug("Could not forget " + owningProcess + " " + Arrays.toString(index)); }
protected void addJobToStateList(DispatchJobEvent jobEvent) { List<JobState> stateList = null; stateList = getJobsDefault(jobEvent.getOwningProcess()); synchronized (stateList) { stateList.add(getStateObject(jobEvent)); } }
DispatchJobEvent dispatchJobEvent = new DispatchJobEvent(e .getOwningProcess(), e .getIndex(), e.getContext(),
@Override public InvocationContext getContext() { return jobEvent.getContext(); }
private Map<String, T2Reference> getInData(String jobIdentifier) { AbstractDispatchEvent inEvent; synchronized (incomingJobs) { inEvent = incomingJobs.get(jobIdentifier); } Map<String, T2Reference> inData = new HashMap<>(); if (inEvent instanceof DispatchJobEvent) inData = ((DispatchJobEvent) inEvent).getData(); return inData; }
@Override public void requestRun(Runnable runMe) { String newThreadName = jobEvent.toString(); Thread thread = new Thread(runMe, newThreadName); thread.setContextClassLoader(activity.getClass() .getClassLoader()); thread.setUncaughtExceptionHandler(new UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { fail("Uncaught exception while invoking " + activity, e); } }); thread.start(); } }
/** * Receive a job from the layer above and pick the first concrete activity * from the list to invoke. Invoke this activity, creating a callback which * will wrap up the result messages in the appropriate collection depth * before sending them on (in general activities are not aware of their * invocation context and should not be responsible for providing correct * index arrays for results) * <p> * This layer will invoke the first invokable activity in the activity list, * so any sane dispatch stack will have narrowed this down to a single item * list by this point, i.e. by the insertion of a failover layer. */ @Override public void receiveJob(final DispatchJobEvent jobEvent) { for (Activity<?> activity : jobEvent.getActivities()) if (activity instanceof AsynchronousActivity) { invoke(jobEvent, (AsynchronousActivity<?>) activity); break; } }
boolean streaming = false; if (index.length == 0) newIndex = jobEvent.getIndex(); else { streaming = true; newIndex = new int[jobEvent.getIndex().length + index.length]; int i = 0; for (int indexValue : jobEvent.getIndex()) newIndex[i++] = indexValue; for (int indexValue : index) .getOwningProcess(), newIndex, jobEvent.getContext(), resultMap, streaming); if (!streaming) {
/** * If an error occurs we can either handle the error or send it to the layer * above for further processing. */ @Override public void receiveError(DispatchErrorEvent errorEvent) { String owningProcess = errorEvent.getOwningProcess(); for (JobState rs : getJobsCopy(owningProcess)) if (identicalIndex(rs.jobEvent.getIndex(), errorEvent.getIndex())) { boolean handled = rs.handleError(); if (!handled) { removeJob(owningProcess, rs); getAbove().receiveError(errorEvent); return; } } }
@Override public void receiveJob(final DispatchJobEvent jobEvent) { List<WorkflowRunIdEntity> entities = jobEvent.getContext().getEntities( WorkflowRunIdEntity.class); if (entities != null && !entities.isEmpty()) {
/** * Receive a job from the layer above, store it in the state map then relay * it to the layer below with a modified activity list containing only the * activity at index 0 */ @Override public void receiveJob(DispatchJobEvent jobEvent) { addJobToStateList(jobEvent); List<Activity<?>> newActivityList = new ArrayList<>(); newActivityList.add(jobEvent.getActivities().get(0)); getBelow().receiveJob( new DispatchJobEvent(jobEvent.getOwningProcess(), jobEvent .getIndex(), jobEvent.getContext(), jobEvent.getData(), newActivityList)); }
int[] newIndex; if (completionIndex.length == 0) newIndex = jobEvent.getIndex(); else { newIndex = new int[jobEvent.getIndex().length + completionIndex.length]; int i = 0; for (int indexValue : jobEvent.getIndex()) newIndex[i++] = indexValue; for (int indexValue : completionIndex) jobEvent.getOwningProcess(), newIndex, jobEvent .getContext()); getAbove().receiveResultCompletion(c); } else { int portDepth = op.getDepth(); emptyListMap.put(portName, refService.getListService() .registerEmptyList(portDepth, jobEvent.getContext()).getId());
String[] split = jobEvent.getOwningProcess().split(":"); provenanceItem = new ProcessProvenanceItem(); String parentDataflowId = workflowItem.getParentId(); provenanceItem.setFacadeID(split[0]); provenanceItem.setDataflowID(split[1]); provenanceItem.setProcessId(jobEvent.getOwningProcess()); provenanceItem.setIdentifier(uuid()); provenanceItem.setParentId(workflowItem.getIdentifier()); processorProvItem.setWorkflowId(parentDataflowId); processorProvItem.setProcessId(jobEvent .getOwningProcess()); processorProvItem.setIdentifier(uuid()); processorProvItem.setParentId(provenanceItem.getIdentifier()); provenanceItem.setProcessId(jobEvent.getOwningProcess()); getReporter().addProvenanceItem(provenanceItem); getReporter().addProvenanceItem(processorProvItem); iterationProvItem = new IterationProvenanceItem(); iterationProvItem.setWorkflowId(parentDataflowId); iterationProvItem.setIteration(jobEvent.getIndex()); iterationProvItem.setIdentifier(uuid()); ReferenceService referenceService = jobEvent.getContext() .getReferenceService(); inputDataItem.setDataMap(jobEvent.getData()); inputDataItem.setReferenceService(referenceService);