@Override public String toString() { return MessageFormat.format( "Flow'{'id={0}, blockers={1}, scripts={2}'}'", getId(), getBlockerIds(), getScripts()); } }
/** * Loads a {@link FlowScript} with the specified ID. * @param properties source properties * @param flowId the target flow ID * @return the loaded script * @throws IllegalArgumentException if script is invalid, or some parameters were {@code null} * @see #extractFlowIds(Properties) */ public static FlowScript load(Properties properties, String flowId) { if (properties == null) { throw new IllegalArgumentException("properties must not be null"); //$NON-NLS-1$ } if (flowId == null) { throw new IllegalArgumentException("flowId must not be null"); //$NON-NLS-1$ } String prefix = getPrefix(flowId); LOG.debug("Loading execution scripts: {}*", prefix); NavigableMap<String, String> flowMap = PropertiesUtil.createPrefixMap(properties, prefix); Set<String> blockerIds = consumeBlockerIds(flowMap, flowId); Set<ExecutionScript.Kind> enables = consumeEnables(flowMap, flowId); Map<ExecutionPhase, List<ExecutionScript>> scripts = consumeScripts(flowMap, flowId); FlowScript script = new FlowScript(flowId, blockerIds, scripts, enables); LOG.trace("Loaded {}*: {}", prefix, script); return script; }
private static List<ExecutionScript> loadScripts( String flowId, ExecutionPhase phase, NavigableMap<String, String> contents) { assert flowId != null; assert phase != null; assert contents != null; if (contents.isEmpty()) { return Collections.emptyList(); } List<ExecutionScript> results = new ArrayList<>(); Map<String, NavigableMap<String, String>> scripts = partitioning(contents); for (Map.Entry<String, NavigableMap<String, String>> entry : scripts.entrySet()) { String scriptId = entry.getKey(); NavigableMap<String, String> scriptContents = entry.getValue(); ExecutionScript script = loadScript(flowId, phase, scriptId, scriptContents); results.add(script); } checkBlockers(flowId, phase, results); return results; }
private static Set<String> consumeBlockerIds(NavigableMap<String, String> flowMap, String flowId) { String blockersString = extract(flowMap, getPrefix(flowId), KEY_BLOCKERS); Set<String> blockerIds = parseTokens(blockersString); return blockerIds; }
private void executePhase( String batchId, FlowScript flow, String executionId, ExecutionPhase phase) throws InterruptedException, IOException { ExecutionContext context = newContext(batchId, flow.getId(), executionId, phase); Set<ExecutionScript> scripts = flow.getScripts().get(phase); assert scripts != null; executePhase(context, scripts, flow.getEnabledScriptKinds()); }
throw new IllegalArgumentException("properties must not be null"); //$NON-NLS-1$ String flowPrefix = getPrefix(getId()); properties.setProperty(flowPrefix + KEY_BLOCKERS, join(getBlockerIds())); properties.setProperty(flowPrefix + KEY_ENABLED_SCRIPT_KINDS, join(toSymbols(getEnabledScriptKinds()))); for (Map.Entry<ExecutionPhase, Set<ExecutionScript>> phase : getScripts().entrySet()) { int index = 0; for (ExecutionScript script : phase.getValue()) { String scriptPrefix = getPrefix(getId(), phase.getKey(), String.format("%04d", index++)); properties.setProperty(scriptPrefix + KEY_ID, script.getId()); properties.setProperty(scriptPrefix + KEY_KIND, script.getKind().getSymbol()); properties.setProperty(scriptPrefix + KEY_BLOCKERS, join(script.getBlockerIds())); properties.setProperty(scriptPrefix + KEY_SUPPORTED_EXTENSIONS, join(script.getSupportedExtensions())); String envPrefix = scriptPrefix + KEY_ENV_PREFIX; for (Map.Entry<String, String> entry : script.getEnvironmentVariables().entrySet()) {
FlowScript script = iter.next(); boolean blocked = false; for (String blockerId : script.getBlockerIds()) { if (blockerIds.contains(blockerId)) { blocked = true; blockerIds.remove(script.getId()); results.add(script);
@Override public Void call() throws InterruptedException, IOException { if (Thread.interrupted()) { throw new InterruptedException(); } String executionId = UUID.randomUUID().toString(); LOG.debug("Generated execution ID for \"{}\": {}", flow.getId(), executionId); lock.beginFlow(flow.getId(), executionId); executeFlow(batchId, flow, executionId); lock.endFlow(flow.getId(), executionId); LOG.debug("Completing jobflow \"{}\": {}", flow.getId(), batchId); return null; } });
throw new IllegalArgumentException("phase must not be null"); //$NON-NLS-1$ String prefix = getPrefix(flowId, phase); LOG.debug("Loading execution scripts: {}*", prefix); Set<String> availableFlowIds = extractFlowIds(properties); if (availableFlowIds.contains(flowId) == false) { throw new IllegalArgumentException(MessageFormat.format( List<ExecutionScript> scripts = loadScripts(flowId, phase, contents); LOG.debug("Loaded {} execution scripts: {}*", scripts.size(), prefix); LOG.trace("Loaded {}*: {}", prefix, scripts);
/** * Executes a target phase. * @param context the current context * @throws InterruptedException if interrupted during this execution * @throws IOException if failed to execute target phase * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.2.5 */ public void executePhase(ExecutionContext context) throws InterruptedException, IOException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } FlowScript flow = script.findFlow(context.getFlowId()); if (flow == null) { throw new IllegalArgumentException(MessageFormat.format( "Flow is undefined: batchId={0}, flowId={1}, executionId={2}", context.getBatchId(), context.getFlowId(), context.getExecutionId())); } Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase()); try (ExecutionLock lock = acquireExecutionLock(context.getBatchId())) { lock.beginFlow(context.getFlowId(), context.getExecutionId()); executePhase(context, executions, flow.getEnabledScriptKinds()); lock.endFlow(context.getFlowId(), context.getExecutionId()); } }
private boolean submit() { LOG.debug("Submitting waiting jobflows: {}", batchId); boolean submitted = false; for (Iterator<FlowScript> iter = flows.iterator(); iter.hasNext();) { FlowScript flow = iter.next(); boolean blocked = false; for (String blockerId : flow.getBlockerIds()) { if (blocking.contains(blockerId)) { blocked = true; break; } } if (blocked == false) { submit(flow); iter.remove(); submitted = true; } } return submitted; }
/** * Loads a {@link BatchScript}. * @param properties source properties * @return the loaded script * @throws IllegalArgumentException if script is invalid, or some parameters were {@code null} */ public static BatchScript load(Properties properties) { if (properties == null) { throw new IllegalArgumentException("properties must not be null"); //$NON-NLS-1$ } String version = properties.getProperty(KEY_VERSION); if (VERSION.equals(version) == false) { throw new IllegalArgumentException(MessageFormat.format( "Unsupported script version: {0}", version)); } String batchId = properties.getProperty(KEY_ID); String verificationCode = properties.getProperty(KEY_VERIFICATION_CODE); Set<String> flowIds = FlowScript.extractFlowIds(properties); List<FlowScript> flowScripts = new ArrayList<>(); for (String flowId : flowIds) { FlowScript flowScript = FlowScript.load(properties, flowId); flowScripts.add(flowScript); } return new BatchScript(batchId, verificationCode, flowScripts); } }
/** * Creates a new instance. * @param batchId ID of this batch * @param buildId the application verification code, or {@code null} if not defined * @param flows member flows * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.4.0 */ public BatchScript(String batchId, String buildId, Collection<FlowScript> flows) { if (batchId == null) { throw new IllegalArgumentException("batchId must not be null"); //$NON-NLS-1$ } if (flows == null) { throw new IllegalArgumentException("flows must not be null"); //$NON-NLS-1$ } this.id = batchId; this.buildId = buildId; SortedMap<String, FlowScript> map = new TreeMap<>(); for (FlowScript flow : flows) { map.put(flow.getId(), flow); } this.flows = Collections.unmodifiableSortedMap(map); }
assert nodeId != null; assert contents != null; String prefix = getPrefix(flowId, phase, nodeId); String scriptId = extract(contents, prefix, KEY_ID); String kindSymbol = extract(contents, prefix, KEY_KIND); ExecutionScript.Kind kind = ExecutionScript.Kind.findFromSymbol(kindSymbol); String blockersString = extract(contents, prefix, KEY_BLOCKERS); Set<String> blockers = parseTokens(blockersString); Map<String, String> environmentVariables = PropertiesUtil.createPrefixMap(contents, KEY_ENV_PREFIX); String extensionsString = contents.get(KEY_SUPPORTED_EXTENSIONS); Set<String> extensions = extensionsString == null ? Collections.emptySet() : parseTokens(extensionsString); ExecutionScript script; if (kind == ExecutionScript.Kind.COMMAND) { String profileName = extract(contents, prefix, KEY_PROFILE); String moduleName = extract(contents, prefix, KEY_MODULE); NavigableMap<String, String> commandMap = PropertiesUtil.createPrefixMap(contents, KEY_COMMAND_PREFIX); if (commandMap.isEmpty()) { extensions); } else if (kind == ExecutionScript.Kind.HADOOP) { String className = extract(contents, prefix, KEY_CLASS_NAME); Map<String, String> properties = PropertiesUtil.createPrefixMap(contents, KEY_PROP_PREFIX); script = new HadoopScript(
private static JsonObject analyzeJobflow(FlowScript flowScript) { assert flowScript != null; JsonArray phases = new JsonArray(); for (Map.Entry<ExecutionPhase, Set<ExecutionScript>> entry : flowScript.getScripts().entrySet()) { ExecutionPhase phase = entry.getKey(); if (entry.getValue().isEmpty() == false || phase == ExecutionPhase.SETUP || phase == ExecutionPhase.CLEANUP) { phases.add(new JsonPrimitive(phase.getSymbol())); } } JsonObject jobflow = new JsonObject(); jobflow.addProperty("id", flowScript.getId()); jobflow.add("blockers", toJsonArray(flowScript.getBlockerIds())); jobflow.add("phases", phases); return jobflow; }
private void submit(FlowScript flow) { LOG.debug("Submitting jobflow \"{}\": {}", flow.getId(), batchId); FlowScriptTask task = new FlowScriptTask(flow, doneQueue, new Callable<Void>() { @Override public Void call() throws InterruptedException, IOException { if (Thread.interrupted()) { throw new InterruptedException(); } String executionId = UUID.randomUUID().toString(); LOG.debug("Generated execution ID for \"{}\": {}", flow.getId(), executionId); lock.beginFlow(flow.getId(), executionId); executeFlow(batchId, flow, executionId); lock.endFlow(flow.getId(), executionId); LOG.debug("Completing jobflow \"{}\": {}", flow.getId(), batchId); return null; } }); YSLOG.info("I01003", batchId, flow.getId()); executor.execute(task); running.put(flow.getId(), task); }
BatchScheduler(String batchId, BatchScript batchScript, ExecutionLock lock, ExecutorService executor) { assert batchId != null; assert batchScript != null; assert lock != null; assert executor != null; this.batchId = batchId; this.flows = new LinkedList<>(batchScript.getAllFlows()); this.lock = lock; this.executor = executor; this.running = new HashMap<>(); this.blocking = new HashSet<>(); for (FlowScript flow : flows) { blocking.add(flow.getId()); } this.doneQueue = new LinkedBlockingQueue<>(); }
private void waitForComplete() throws InterruptedException, IOException { LOG.debug("Waiting for running jobflows complete: {}", batchId); FlowScriptTask done = doneQueue.take(); assert done.isDone(); FlowScript flow = done.script; try { done.get(); boolean blocked = blocking.remove(flow.getId()); assert blocked; } catch (CancellationException e) { YSLOG.info(e, "I01005", batchId, flow.getId()); } catch (ExecutionException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } else if (e.getCause() instanceof InterruptedException) { throw (InterruptedException) e.getCause(); } else if (e.getCause() instanceof Error) { throw (Error) e.getCause(); } else { throw new IOException("Flow execution failed by unknown error", e); } } finally { FlowScriptTask ran = running.remove(flow.getId()); assert ran != null; } } }
assert flow != null; assert executionId != null; YSLOG.info("I02000", batchId, flow.getId(), executionId); long start = System.currentTimeMillis(); try { if (skipFlows.contains(flow.getId())) { YSLOG.info("I02002", batchId, flow.getId(), executionId); return; executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE); } else { YSLOG.info("I02003", batchId, flow.getId(), executionId); try { executePhase(batchId, flow, executionId, ExecutionPhase.FINALIZE); } catch (Exception e) { YSLOG.warn(e, "W02002", batchId, flow.getId(), executionId); executePhase(batchId, flow, executionId, ExecutionPhase.CLEANUP); } catch (Exception e) { YSLOG.warn(e, "W02003", batchId, flow.getId(), executionId); YSLOG.info("I02001", batchId, flow.getId(), executionId); } catch (ExitCodeException e) { YSLOG.error("E02001", batchId, flow.getId(), executionId); throw e; } catch (IOException e) { YSLOG.error(e, "E02001", batchId, flow.getId(), executionId); throw e; } catch (InterruptedException e) {