@Override public DispatchLayer<?> createDispatchLayer(URI uri) { DispatchLayer<?> dispatchLayer = null; if (parallelizeLayer.equals(uri)) { dispatchLayer = new Parallelize(); } else if (errorBounceLayer.equals(uri)) { dispatchLayer = new ErrorBounce(); } else if (failoverLayer.equals(uri)) { dispatchLayer = new Failover(); } else if (retryLayer.equals(uri)) { dispatchLayer = new Retry(); } else if (invokeLayer.equals(uri)) { dispatchLayer = new Invoke(); } else if (loopLayer.equals(uri)) { dispatchLayer = new Loop(); } else if (intermediateProvenanceLayer.equals(uri)) { dispatchLayer = new IntermediateProvenance(); } else if (stopLayer.equals(uri)) { dispatchLayer = new Stop(); } return dispatchLayer; }
@Override public void receiveError(DispatchErrorEvent errorEvent) { StateModel model; String owningProcess = errorEvent.getOwningProcess(); synchronized(stateMap) { model = stateMap.get(owningProcess); } getAbove().receiveError(errorEvent); if (model == null) { logger.warn("Error received for unknown owning process: " + owningProcess); return; } model.finishWith(errorEvent.getIndex()); }
public DefaultDispatchStackEdit(Processor processor) { super(processor); DispatchStackImpl stack = ((ProcessorImpl)processor).getDispatchStack(); // Top level parallelise layer int layer = 0; List<Edit<?>> edits = new ArrayList<Edit<?>>(); edits.add(new AddDispatchLayerEdit(stack, new Parallelize(MAX_JOBS), layer++)); edits.add(new AddDispatchLayerEdit(stack, new ErrorBounce(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Failover(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Retry(MAX_RETRIES, INITIAL_DELAY, MAX_DELAY, BACKOFF_FACTOR), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Invoke(), layer++)); compoundEdit=new CompoundEdit(edits); }
@Override @SuppressWarnings("unchecked") public void receiveResult(DispatchResultEvent resultEvent) { StateModel model; String owningProcess = resultEvent.getOwningProcess(); synchronized(stateMap) { model = stateMap.get(owningProcess); } DispatchLayer above = getAbove(); above.receiveResult(resultEvent); if (model == null) { logger.warn("Error received for unknown owning process: " + owningProcess); return; } model.finishWith(resultEvent.getIndex()); }
public DefaultDispatchStackEdit(Processor processor) { super(processor); DispatchStackImpl stack = ((ProcessorImpl)processor).getDispatchStack(); // Top level parallelise layer int layer = 0; List<Edit<?>> edits = new ArrayList<Edit<?>>(); edits.add(new AddDispatchLayerEdit(stack, new Parallelize(MAX_JOBS), layer++)); edits.add(new AddDispatchLayerEdit(stack, new ErrorBounce(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Failover(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Retry(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Stop(), layer++)); edits.add(new AddDispatchLayerEdit(stack, new Invoke(), layer++)); compoundEdit=new CompoundEdit(edits); }
@Override public void receiveError(DispatchErrorEvent errorEvent) { // System.out.println(sentJobsCount); StateModel model; String owningProcess = errorEvent.getOwningProcess(); synchronized(stateMap) { model = stateMap.get(owningProcess); } getAbove().receiveError(errorEvent); if (model == null) { logger.warn("Error received for unknown owning process: " + owningProcess); return; } model.finishWith(errorEvent.getIndex()); }
public void testStackConstructionAndSimpleInvocation() { System.out.println("Multiple jobs, single process identifier"); DispatchStackImpl d = new BasicDispatchStackImpl(new ArrayList<Activity<?>>()); d.addLayer(new DiagnosticLayer()); d.addLayer(new Parallelize()); d.addLayer(new DummyInvokerLayer()); for (IterationInternalEvent<?> e : generateLotsOfEvents("Process1", 10)) { d.receiveEvent(e); } try { Thread.sleep(3000); System.out.println("--------------------------------------------------\n"); } catch (InterruptedException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } }
@Override public void receiveError(DispatchErrorEvent errorEvent) { // System.out.println(sentJobsCount); StateModel model = stateMap.get(errorEvent.getOwningProcess()); getAbove().receiveError(errorEvent); model.finishWith(errorEvent.getIndex()); }
/** * Test single job with streaming */ public void testSingleJobWithStream() { System.out.println("Single job with streaming"); DispatchStackImpl d = new BasicDispatchStackImpl(new ArrayList<Activity<?>>()); d.addLayer(new DiagnosticLayer()); d.addLayer(new Parallelize()); d.addLayer(new DummyStreamingInvokerLayer()); Map<String, EntityIdentifier> dataMap = new HashMap<String, EntityIdentifier>(); dataMap.put("SingleJobInput", nextID()); d.receiveEvent(new Job("Process1:processorName", new int[] {}, dataMap, context)); try { Thread.sleep(2000); System.out.println("--------------------------------------------------\n"); } catch (InterruptedException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } }
/** * Only going to receive this if the activity invocation was streaming, in * which case we need to handle all completion events and pass them up the * stack. */ @Override public void receiveResultCompletion(DispatchCompletionEvent completionEvent) { StateModel model = stateMap.get(completionEvent.getOwningProcess()); getAbove().receiveResultCompletion(completionEvent); model.finishWith(completionEvent.getIndex()); }
/** * Test that we still get a cache purge when there are no explicit * completion events, in this case because a job is submitted with an empty * index array (i.e. no iteration) * */ public void testSingleJob() { System.out.println("Single job"); DispatchStackImpl d = new BasicDispatchStackImpl(new ArrayList<Activity<?>>()); d.addLayer(new DiagnosticLayer()); d.addLayer(new Parallelize()); d.addLayer(new DummyInvokerLayer()); Map<String, EntityIdentifier> dataMap = new HashMap<String, EntityIdentifier>(); dataMap.put("SingleJobInput", nextID()); d.receiveEvent(new Job("Process1:processorName", new int[] {}, dataMap, context)); try { Thread.sleep(1000); System.out.println("--------------------------------------------------\n"); } catch (InterruptedException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } }
@Override @SuppressWarnings("unchecked") public void receiveResult(DispatchResultEvent resultEvent) { StateModel model = stateMap.get(resultEvent.getOwningProcess()); DispatchLayer above = getAbove(); above.receiveResult(resultEvent); model.finishWith(resultEvent.getIndex()); }
/** * Test parallel jobs with streaming */ public void testParallelJobsWithStream() { System.out.println("Parallel jobs with streaming"); DispatchStackImpl d = new BasicDispatchStackImpl(new ArrayList<Activity<?>>()); d.addLayer(new DiagnosticLayer()); d.addLayer(new Parallelize()); d.addLayer(new DummyStreamingInvokerLayer()); for (IterationInternalEvent<?> e : generateLotsOfEvents("Process1", 4)) { d.receiveEvent(e); } try { Thread.sleep(3000); System.out.println("--------------------------------------------------\n"); } catch (InterruptedException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } }
@Override @SuppressWarnings("unchecked") public void receiveResult(DispatchResultEvent resultEvent) { StateModel model; String owningProcess = resultEvent.getOwningProcess(); synchronized(stateMap) { model = stateMap.get(owningProcess); } DispatchLayer above = getAbove(); above.receiveResult(resultEvent); if (model == null) { logger.warn("Error received for unknown owning process: " + owningProcess); return; } model.finishWith(resultEvent.getIndex()); }
d.addLayer(new Parallelize()); d.addLayer(new DummyStreamingInvokerLayer()); Map<String, EntityIdentifier> dataMap = new HashMap<String, EntityIdentifier>();
/** * Only going to receive this if the activity invocation was streaming, in * which case we need to handle all completion events and pass them up the * stack. */ @Override public void receiveResultCompletion(DispatchCompletionEvent completionEvent) { StateModel model; String owningProcess = completionEvent.getOwningProcess(); synchronized(stateMap) { model = stateMap.get(owningProcess); } getAbove().receiveResultCompletion(completionEvent); if (model == null) { logger.warn("Error received for unknown owning process: " + owningProcess); return; } model.finishWith(completionEvent.getIndex()); }
public void testMultipleProcessIDs() { System.out.println("Multiple jobs, multiple process identifiers"); DispatchStackImpl d = new BasicDispatchStackImpl(new ArrayList<Activity<?>>()); d.addLayer(new DiagnosticLayer()); d.addLayer(new Parallelize()); d.addLayer(new DummyInvokerLayer()); for (IterationInternalEvent<?> e : generateLotsOfEvents("Process1", 10)) { d.receiveEvent(e); } for (IterationInternalEvent<?> e : generateLotsOfEvents("Process2", 6)) { d.receiveEvent(e); } try { Thread.sleep(3000); System.out.println("--------------------------------------------------\n"); } catch (InterruptedException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } }
/** * Only going to receive this if the activity invocation was streaming, in * which case we need to handle all completion events and pass them up the * stack. */ @Override public void receiveResultCompletion(DispatchCompletionEvent completionEvent) { StateModel model; String owningProcess = completionEvent.getOwningProcess(); synchronized(stateMap) { model = stateMap.get(owningProcess); } getAbove().receiveResultCompletion(completionEvent); if (model == null) { logger.warn("Error received for unknown owning process: " + owningProcess); return; } model.finishWith(completionEvent.getIndex()); }
new AddDispatchLayerEdit(stack, new Parallelize(MAX_JOBS), layer++) .doEdit(); new AddDispatchLayerEdit(stack, new ErrorBounce(), layer++).doEdit();
private void addDispatchLayers( org.embl.ebi.escience.scufl.Processor t1Processor, DispatchStack dispatchStack) throws EditException { int maxJobs = t1Processor.getWorkers(); int maxRetries = t1Processor.getRetries(); float backoffFactor = (float) t1Processor.getBackoff(); int initialDelay = t1Processor.getRetryDelay(); int maxDelay = (int) (initialDelay * (Math.pow(backoffFactor, maxRetries))); DispatchLayer<?> parallelize = new Parallelize(maxJobs); DispatchLayer<?> errorBounce = new ErrorBounce(); DispatchLayer<?> failover = new Failover(); DispatchLayer<?> retry = new Retry(maxRetries, initialDelay, maxDelay, backoffFactor); DispatchLayer<?> invoke = new Invoke(); int layer = 0; edits.getAddDispatchLayerEdit(dispatchStack, parallelize, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, errorBounce, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, failover, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, retry, layer++).doEdit(); edits.getAddDispatchLayerEdit(dispatchStack, invoke, layer++).doEdit(); }