@Override public Void call() throws InterruptedException, IOException { if (Thread.interrupted()) { throw new InterruptedException(); } String executionId = UUID.randomUUID().toString(); LOG.debug("Generated execution ID for \"{}\": {}", flow.getId(), executionId); lock.beginFlow(flow.getId(), executionId); executeFlow(batchId, flow, executionId); lock.endFlow(flow.getId(), executionId); LOG.debug("Completing jobflow \"{}\": {}", flow.getId(), batchId); return null; } });
/** * Creates a new instance. * @param batchId ID of this batch * @param buildId the application verification code, or {@code null} if not defined * @param flows member flows * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.4.0 */ public BatchScript(String batchId, String buildId, Collection<FlowScript> flows) { if (batchId == null) { throw new IllegalArgumentException("batchId must not be null"); //$NON-NLS-1$ } if (flows == null) { throw new IllegalArgumentException("flows must not be null"); //$NON-NLS-1$ } this.id = batchId; this.buildId = buildId; SortedMap<String, FlowScript> map = new TreeMap<>(); for (FlowScript flow : flows) { map.put(flow.getId(), flow); } this.flows = Collections.unmodifiableSortedMap(map); }
private void submit(FlowScript flow) { LOG.debug("Submitting jobflow \"{}\": {}", flow.getId(), batchId); FlowScriptTask task = new FlowScriptTask(flow, doneQueue, new Callable<Void>() { @Override public Void call() throws InterruptedException, IOException { if (Thread.interrupted()) { throw new InterruptedException(); } String executionId = UUID.randomUUID().toString(); LOG.debug("Generated execution ID for \"{}\": {}", flow.getId(), executionId); lock.beginFlow(flow.getId(), executionId); executeFlow(batchId, flow, executionId); lock.endFlow(flow.getId(), executionId); LOG.debug("Completing jobflow \"{}\": {}", flow.getId(), batchId); return null; } }); YSLOG.info("I01003", batchId, flow.getId()); executor.execute(task); running.put(flow.getId(), task); }
BatchScheduler(String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) { assert batchId != null; assert batchScript != null; assert lock != null; assert executor != null; this.batchId = batchId; this.flows = new LinkedList<>(batchScript.getAllFlows()); this.lock = lock; this.executor = executor; this.running = new HashMap<>(); this.blocking = new HashSet<>(); for (FlowScript flow : flows) { blocking.add(flow.getId()); } this.doneQueue = new LinkedBlockingQueue<>(); }
@Override public String toString() { return MessageFormat.format( "Flow'{'id={0}, blockers={1}, scripts={2}'}'", getId(), getBlockerIds(), getScripts()); } }
private void waitForComplete() throws InterruptedException, IOException { LOG.debug("Waiting for running jobflows complete: {}", batchId); FlowScriptTask done = doneQueue.take(); assert done.isDone(); FlowScript flow = done.script; try { done.get(); boolean blocked = blocking.remove(flow.getId()); assert blocked; } catch (CancellationException e) { YSLOG.info(e, "I01005", batchId, flow.getId()); } catch (ExecutionException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } else if (e.getCause() instanceof InterruptedException) { throw (InterruptedException) e.getCause(); } else if (e.getCause() instanceof Error) { throw (Error) e.getCause(); } else { throw new IOException("Flow execution failed by unknown error", e); } } finally { FlowScriptTask ran = running.remove(flow.getId()); assert ran != null; } } }
private static JsonObject analyzeJobflow(FlowScript flowScript) { assert flowScript != null; JsonArray phases = new JsonArray(); for (Map.Entry<ExecutionPhase, Set<ExecutionScript>> entry : flowScript.getScripts().entrySet()) { ExecutionPhase phase = entry.getKey(); if (entry.getValue().isEmpty() == false || phase == ExecutionPhase.SETUP || phase == ExecutionPhase.CLEANUP) { phases.add(new JsonPrimitive(phase.getSymbol())); } } JsonObject jobflow = new JsonObject(); jobflow.addProperty("id", flowScript.getId()); jobflow.add("blockers", toJsonArray(flowScript.getBlockerIds())); jobflow.add("phases", phases); return jobflow; }
blockerIds.remove(script.getId()); results.add(script);
private void executePhase( String batchId, FlowScript flow, String executionId, ExecutionPhase phase) throws InterruptedException, IOException { ExecutionContext context = newContext(batchId, flow.getId(), executionId, phase); Set<ExecutionScript> scripts = flow.getScripts().get(phase); assert scripts != null; executePhase(context, scripts, flow.getEnabledScriptKinds()); }
assert flow != null; assert executionId != null; YSLOG.info("I02000", batchId, flow.getId(), executionId); long start = System.currentTimeMillis(); try { if (skipFlows.contains(flow.getId())) { YSLOG.info("I02002", batchId, flow.getId(), executionId); return; executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE); } else { YSLOG.info("I02003", batchId, flow.getId(), executionId); try { executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE); } catch (Exception e) { YSLOG.warn(e, "W02002", batchId, flow.getId(), executionId); executePhase(batchId, flow, executionId, ExecutionPhase.CLEANUP); } catch (Exception e) { YSLOG.warn(e, "W02003", batchId, flow.getId(), executionId); YSLOG.info("I02001", batchId, flow.getId(), executionId); } catch (ExitCodeException e) { YSLOG.error("E02001", batchId, flow.getId(), executionId); throw e; } catch (IOException e) { YSLOG.error(e, "E02001", batchId, flow.getId(), executionId); throw e; } catch (InterruptedException e) {
throw new IllegalArgumentException("properties must not be null"); //$NON-NLS-1$ String flowPrefix = getPrefix(getId()); properties.setProperty(flowPrefix + KEY_BLOCKERS, join(getBlockerIds())); properties.setProperty(flowPrefix + KEY_ENABLED_SCRIPT_KINDS, join(toSymbols(getEnabledScriptKinds()))); int index = 0; for (ExecutionScript script : phase.getValue()) { String scriptPrefix = getPrefix(getId(), phase.getKey(), String.format("%04d", index++)); properties.setProperty(scriptPrefix + KEY_ID, script.getId()); properties.setProperty(scriptPrefix + KEY_KIND, script.getKind().getSymbol());