@Override public DispatchLayer<?> createDispatchLayer(URI uri) { DispatchLayer<?> dispatchLayer = null; if (parallelizeLayer.equals(uri)) { dispatchLayer = new Parallelize(); } else if (errorBounceLayer.equals(uri)) { dispatchLayer = new ErrorBounce(); } else if (failoverLayer.equals(uri)) { dispatchLayer = new Failover(); } else if (retryLayer.equals(uri)) { dispatchLayer = new Retry(); } else if (invokeLayer.equals(uri)) { dispatchLayer = new Invoke(); } else if (loopLayer.equals(uri)) { dispatchLayer = new Loop(); } else if (intermediateProvenanceLayer.equals(uri)) { dispatchLayer = new IntermediateProvenance(); } else if (stopLayer.equals(uri)) { dispatchLayer = new Stop(); } return dispatchLayer; }
/** * If the job contains errors, or collections which contain errors * themselves then bounce a result message with error documents in back up * to the layer above */ @Override public void receiveJob(DispatchJobEvent jobEvent) { Set<T2Reference> errorReferences = new HashSet<T2Reference>(); for (T2Reference ei : jobEvent.getData().values()) { if (ei.containsErrors()) { errorReferences.add(ei); } } if (errorReferences.isEmpty()) { // relay the message down... getBelow().receiveJob(jobEvent); } else { getState(jobEvent.getOwningProcess()) .incrementErrorsReflected(); sendErrorOutput(jobEvent, null, errorReferences); } }
/** * Always send the error document job result on receiving a failure, at * least for now! This should be configurable, in effect this is the part * that ensures the processor never sees a top level failure. */ @Override public void receiveError(DispatchErrorEvent errorEvent) { getState(errorEvent.getOwningProcess()).incrementErrorsTranslated(); sendErrorOutput(errorEvent); }
/** * Construct and send a new result message with error documents in place of * all outputs at the appropriate depth * * @param e */ private void sendErrorOutput(Event<?> e) { DataManager dm = e.getContext().getDataManager(); Processor p = dispatchStack.getProcessor(); Map<String, EntityIdentifier> outputDataMap = new HashMap<String, EntityIdentifier>(); for (OutputPort op : p.getOutputPorts()) { outputDataMap.put(op.getName(), dm.registerError(op.getDepth(), 0, "No message...")); } DispatchResultEvent dre = new DispatchResultEvent(e.getOwningProcess(), e.getIndex(), e.getContext(), outputDataMap, false); getAbove().receiveResult(dre); }
/** * Always send the error document job result on receiving a failure, at * least for now! This should be configurable, in effect this is the part * that ensures the processor never sees a top level failure. */ @Override public void receiveError(DispatchErrorEvent errorEvent) { getState(errorEvent.getOwningProcess()).incrementErrorsTranslated();this. sendErrorOutput(errorEvent, errorEvent.getCause(), null); }
getAbove().receiveResult(dre);
/** * If the job contains errors, or collections which contain errors * themselves then bounce a result message with error documents in back up * to the layer above */ @Override public void receiveJob(DispatchJobEvent jobEvent) { Set<T2Reference> errorReferences = new HashSet<T2Reference>(); for (T2Reference ei : jobEvent.getData().values()) { if (ei.containsErrors()) { errorReferences.add(ei); } } if (errorReferences.isEmpty()) { // relay the message down... getBelow().receiveJob(jobEvent); } else { getState(jobEvent.getOwningProcess()) .incrementErrorsReflected(); sendErrorOutput(jobEvent, null, errorReferences); } }
public DefaultDispatchStackEdit(Processor processor) { super(processor); DispatchStackImpl stack = ((ProcessorImpl)processor).getDispatchStack(); // Top level parallelise layer int layer = 0; List<Edit<?>> edits = new ArrayList<Edit<?>>(); edits.add(new AddDispatchLayerEdit(stack, new Parallelize(MAX_JOBS), layer++)); edits.add(new AddDispatchLayerEdit(stack, new ErrorBounce(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Failover(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Retry(MAX_RETRIES, INITIAL_DELAY, MAX_DELAY, BACKOFF_FACTOR), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Invoke(), layer++)); compoundEdit=new CompoundEdit(edits); }
/** * Always send the error document job result on receiving a failure, at * least for now! This should be configurable, in effect this is the part * that ensures the processor never sees a top level failure. */ @Override public void receiveError(DispatchErrorEvent errorEvent) { getState(errorEvent.getOwningProcess()).incrementErrorsTranslated(); sendErrorOutput(errorEvent, errorEvent.getCause(), null); }
getAbove().receiveResult(dre);
/** * If the job contains errors, or collections which contain errors * themselves then bounce a result message with error documents in back up * to the layer above */ @Override public void receiveJob(DispatchJobEvent jobEvent) { for (EntityIdentifier ei : jobEvent.getData().values()) { if (ei instanceof ErrorDocumentIdentifier || (ei instanceof EntityListIdentifier && ((EntityListIdentifier) ei) .getContainsErrors())) { getState(jobEvent.getOwningProcess()) .incrementErrorsReflected(); sendErrorOutput(jobEvent); return; } } // Got here so relay the message down... getBelow().receiveJob(jobEvent); }
public DefaultDispatchStackEdit(Processor processor) { super(processor); DispatchStackImpl stack = ((ProcessorImpl)processor).getDispatchStack(); // Top level parallelise layer int layer = 0; List<Edit<?>> edits = new ArrayList<Edit<?>>(); edits.add(new AddDispatchLayerEdit(stack, new Parallelize(MAX_JOBS), layer++)); edits.add(new AddDispatchLayerEdit(stack, new ErrorBounce(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Failover(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Retry(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Stop(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Invoke(), layer++)); compoundEdit=new CompoundEdit(edits); }
new AddDispatchLayerEdit(stack, new Parallelize(MAX_JOBS), layer++) .doEdit(); new AddDispatchLayerEdit(stack, new ErrorBounce(), layer++).doEdit(); new AddDispatchLayerEdit(stack, new Failover(), layer++).doEdit(); new AddDispatchLayerEdit(stack, new Retry(MAX_RETRIES, INITIAL_DELAY,
private void addDispatchLayers( org.embl.ebi.escience.scufl.Processor t1Processor, DispatchStack dispatchStack) throws EditException { int maxJobs = t1Processor.getWorkers(); int maxRetries = t1Processor.getRetries(); float backoffFactor = (float) t1Processor.getBackoff(); int initialDelay = t1Processor.getRetryDelay(); int maxDelay = (int) (initialDelay * (Math.pow(backoffFactor, maxRetries))); DispatchLayer<?> parallelize = new Parallelize(maxJobs); DispatchLayer<?> errorBounce = new ErrorBounce(); DispatchLayer<?> failover = new Failover(); DispatchLayer<?> retry = new Retry(maxRetries, initialDelay, maxDelay, backoffFactor); DispatchLayer<?> invoke = new Invoke(); int layer = 0; edits.getAddDispatchLayerEdit(dispatchStack, parallelize, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, errorBounce, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, failover, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, retry, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, invoke, layer++).doEdit(); }
private void addDispatchLayers( org.embl.ebi.escience.scufl.Processor t1Processor, DispatchStack dispatchStack) throws EditException { int maxJobs = t1Processor.getWorkers(); int maxRetries = t1Processor.getRetries(); float backoffFactor = (float) t1Processor.getBackoff(); int initialDelay = t1Processor.getRetryDelay(); int maxDelay = (int) (initialDelay * (Math.pow(backoffFactor, maxRetries))); DispatchLayer<?> parallelize = new Parallelize(maxJobs); DispatchLayer<?> errorBounce = new ErrorBounce(); DispatchLayer<?> failover = new Failover(); DispatchLayer<?> retry = new Retry(maxRetries, initialDelay, maxDelay, backoffFactor); DispatchLayer<?> invoke = new Invoke(); int layer = 0; edits.getAddDispatchLayerEdit(dispatchStack, parallelize, layer++) .doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, errorBounce, layer++) .doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, failover, layer++) .doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, retry, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, invoke, layer++).doEdit(); }