/** * Creates a new instance. * @param script target script * @param context current execution context * @param handler handler which will attempt to resolve placeholders * @throws InterruptedException if interrupted to prepare this resolver * @throws IOException if failed to prepare replacements * @throws IllegalArgumentException if some parameters were {@code null} */ public PlaceholderResolver( ExecutionScript script, ExecutionContext context, ExecutionScriptHandler<?> handler) throws InterruptedException, IOException { if (script == null) { throw new IllegalArgumentException("script must not be null"); //$NON-NLS-1$ } if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } if (handler == null) { throw new IllegalArgumentException("handler must not be null"); //$NON-NLS-1$ } replacements = new HashMap<>(); replacements.put(PLACEHOLDER_HOME, getAsakusaHomePath(context, script, handler)); replacements.put(PLACEHOLDER_EXECUTION_ID, context.getExecutionId()); replacements.put(PLACEHOLDER_ARGUMENTS, context.getArgumentsAsString()); }
private List<String> buildExecutionCommand( ExecutionContext context, HadoopScript script) throws IOException, InterruptedException { assert context != null; assert script != null; List<String> command = new ArrayList<>(); command.add(getCommand(context, PATH_EXECUTE, script)); command.add(script.getClassName()); command.add(context.getBatchId()); command.add(context.getFlowId()); command.add(context.getExecutionId()); command.add(context.getArgumentsAsString()); Map<String, String> props = buildHadoopProperties(context, script); for (Map.Entry<String, String> entry : props.entrySet()) { command.add("-D"); command.add(MessageFormat.format("{0}={1}", entry.getKey(), entry.getValue())); } return command; }
/** * Computes the tracking ID for script-less job. * @param context target context * @return the operation ID * @since 0.5.0 */ public static String computeTrackingId(ExecutionContext context) { return MessageFormat.format( "YAESS/{0}/{1}/{2}/{3}", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); }
/** * Creates a new instance. * @param context current context * @param stepUnit report unit for each progress size (0.0~1.0) * @throws IllegalArgumentException if some parameters were {@code null} */ public LoggingExecutionMonitor(ExecutionContext context, double stepUnit) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } this.label = MessageFormat.format("{0}|{1}|{3}@{2}", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); if (stepUnit <= 0) { this.stepUnit = Double.MAX_VALUE; } else { this.stepUnit = Math.max(stepUnit, 0.01) - 0.0000000000001; } }
context.getFlowId(), context.getPhase(), context.getExecutionId(), blockers)); context.getFlowId(), context.getPhase(), context.getExecutionId()));
/** * Computes the tracking ID. * @param context target context * @param script target script * @return the operation ID * @since 0.5.0 */ public static String computeTrackingId(ExecutionContext context, ExecutionScript script) { return MessageFormat.format( "YAESS/{0}/{1}/{2}/{3}/{4}", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId()); }
/** * Executes a target phase. * @param context the current context * @throws InterruptedException if interrupted during this execution * @throws IOException if failed to execute target phase * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.2.5 */ public void executePhase(ExecutionContext context) throws InterruptedException, IOException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } FlowScript flow = script.findFlow(context.getFlowId()); if (flow == null) { throw new IllegalArgumentException(MessageFormat.format( "Flow is undefined: batchId={0}, flowId={1}, executionId={2}", context.getBatchId(), context.getFlowId(), context.getExecutionId())); } Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase()); try (ExecutionLock lock = acquireExecutionLock(context.getBatchId())) { lock.beginFlow(context.getFlowId(), context.getExecutionId()); executePhase(context, executions, flow.getEnabledScriptKinds()); lock.endFlow(context.getFlowId(), context.getExecutionId()); } }
/** * Performs as {@link #setUp(ExecutionMonitor, ExecutionContext)} that does nothing. * @param context current context * @since 0.4.0 */ protected final void voidSetUp(ExecutionContext context) { YSLOG.info("I51001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getHandlerId()); }
private void command( ExecutionMonitor monitor, ExecutionContext context, ExecutionScript script, List<String> command) throws InterruptedException, IOException { assert monitor != null; assert context != null; assert command != null; assert command.isEmpty() == false; Map<String, String> env = getEnvironmentVariables(context, script); LOG.debug("env: {}", env); LOG.debug("command: {}", command); Map<String, Blob> extensions = BlobUtil.getExtensions(context, script); LOG.debug("extensions: {}", extensions); monitor.checkCancelled(); ProcessExecutor executor = getCommandExecutor(); int exit = executor.execute(context, command, env, extensions, monitor.getOutput()); if (exit == 0) { return; } throw new ExitCodeException(MessageFormat.format( "Unexpected exit code from command job: " + "code={4} (batch={0}, flow={1}, phase={2}, exection={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), String.valueOf(exit)), exit); }
/** * Performs as {@link #cleanUp(ExecutionMonitor, ExecutionContext)} that does nothing. * @param context current context * @since 0.4.0 */ protected final void voidCleanUp(ExecutionContext context) { YSLOG.info("I51002", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getHandlerId()); } }
@Override public void cleanUp(ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { ExecutionScriptHandler<T> target; if (forceCleanUp != null) { target = delegations.get(forceCleanUp); } else { target = resolve(context, null); } assert target != null; YSLOG.info("I01003", target.getHandlerId(), context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); target.cleanUp(monitor, context); }
@Override public void setUp(ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { ExecutionScriptHandler<T> target; if (forceSetUp != null) { target = delegations.get(forceSetUp); } else { target = resolve(context, null); } assert target != null; YSLOG.info("I01001", target.getHandlerId(), context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); target.setUp(monitor, context); }
context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), job.getJobLabel(), context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), job.getJobLabel(),
context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId(), currentProfile.getPrefix(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId(), String.valueOf(exit)), exit);
@Override public void cleanUp( ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { monitor.open(1); try { if (cleanup) { YSLOG.info("I51001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), getHandlerId()); HadoopScript script = new HadoopScript( context.getPhase().getSymbol(), Collections.emptySet(), CLEANUP_STAGE_CLASS, Collections.emptyMap(), Collections.emptyMap()); execute0(monitor, context, script); } else { YSLOG.info("I51002", context.getBatchId(), context.getFlowId(), context.getExecutionId(), getHandlerId()); } } finally { monitor.close(); } }
context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId(), currentProfile.getPrefix(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId(), String.valueOf(exit)), exit);
private JobScript convert(ExecutionContext context, HadoopScript script) throws InterruptedException, IOException { assert context != null; assert script != null; JobScript result = new JobScript(); result.setBatchId(context.getBatchId()); result.setFlowId(context.getFlowId()); result.setExecutionId(context.getExecutionId()); result.setPhase(context.getPhase()); result.setArguments(new HashMap<>(context.getArguments())); result.setStageId(script.getId()); result.setMainClassName(script.getClassName()); Map<String, String> props = new HashMap<>(); props.putAll(getProperties(context, script)); props.putAll(script.getHadoopProperties()); props.put(HadoopScriptUtil.PROP_TRACKING_ID, Job.computeTrackingId(context, script)); result.setProperties(props); Map<String, String> env = new HashMap<>(); // NOTE: Handler has only dummy environment variables // env.putAll(getEnvironmentVariables(context, script)); env.putAll(context.getEnvironmentVariables()); env.putAll(script.getEnvironmentVariables()); result.setEnvironmentVariables(env); return result; }
assert executions != null; YSLOG.info("I03000", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); long start = System.currentTimeMillis(); try { if (skipFlows.contains(context.getFlowId())) { YSLOG.info("I03002", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); return; context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); } catch (ExitCodeException e) { YSLOG.error("E03001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); throw e; } catch (IOException e) { YSLOG.error(e, "E03001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); throw e; } catch (InterruptedException e) { YSLOG.warn(e, "W03001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); throw e; } finally { long end = System.currentTimeMillis(); YSLOG.info("I03999", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), end - start);
context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); long start = System.currentTimeMillis(); execute(monitor, context); YSLOG.info("I04001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); } catch (IOException e) { YSLOG.error(e, "E04001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); throw e; } catch (InterruptedException e) { YSLOG.warn(e, "W04001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context)); throw e; long end = System.currentTimeMillis(); YSLOG.info("I04999", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getJobLabel(), getServiceLabel(), getTrackingId(context), end - start);
@Override public void execute( ExecutionMonitor monitor, ExecutionContext context, T script) throws InterruptedException, IOException { ExecutionScriptHandler<T> target = resolve(context, script); assert target != null; YSLOG.info("I01002", target.getHandlerId(), context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId()); target.execute(monitor, context, script); }