/** * Computes the tracking ID for script-less job. * @param context target context * @return the operation ID * @since 0.5.0 */ public static String computeTrackingId(ExecutionContext context) { return MessageFormat.format( "YAESS/{0}/{1}/{2}/{3}", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); }
private ExecutionContext newContext(String batchId, String flowId, String executionId, ExecutionPhase phase) { return new ExecutionContext( batchId, flowId, executionId, phase, batchArguments, environmentVaritables, extensions); }
/** * Returns current argument pairs as string format. * @return string format of the execution arguments */ public String getArgumentsAsString() { StringBuilder buf = new StringBuilder(); for (Map.Entry<String, String> entry : arguments.entrySet()) { buf.append(escape(entry.getKey())); buf.append("="); buf.append(escape(entry.getValue())); buf.append(","); } if (buf.length() >= 1) { buf.deleteCharAt(buf.length() - 1); } return buf.toString(); }
private JobScript convert(ExecutionContext context, HadoopScript script) throws InterruptedException, IOException { assert context != null; assert script != null; JobScript result = new JobScript(); result.setBatchId(context.getBatchId()); result.setFlowId(context.getFlowId()); result.setExecutionId(context.getExecutionId()); result.setPhase(context.getPhase()); result.setArguments(new HashMap<>(context.getArguments())); result.setStageId(script.getId()); result.setMainClassName(script.getClassName()); Map<String, String> props = new HashMap<>(); props.putAll(getProperties(context, script)); props.putAll(script.getHadoopProperties()); props.put(HadoopScriptUtil.PROP_TRACKING_ID, Job.computeTrackingId(context, script)); result.setProperties(props); Map<String, String> env = new HashMap<>(); // NOTE: Handler has only dummy environment variables // env.putAll(getEnvironmentVariables(context, script)); env.putAll(context.getEnvironmentVariables()); env.putAll(script.getEnvironmentVariables()); result.setEnvironmentVariables(env); return result; }
private ExecutionScriptHandler<T> resolve( ExecutionContext context, ExecutionScript script) throws IOException { assert context != null; Properties batchConf = getBatchConf(context, script); String key = findKey(context, script, batchConf); if (key != null) { ExecutionScriptHandler<T> target = delegations.get(key); if (target != null) { return target; } throw new IOException(MessageFormat.format( "Invalid dispatch target for multidispatch plugin: " + "{4} (batchId={0}, flowId={1}, phase={2}, stageId={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), script == null ? LABEL_UNDEFINED : script.getId(), key)); } ExecutionScriptHandler<T> defaultTarget = delegations.get(PREFIX_DEFAULT); assert defaultTarget != null; return defaultTarget; }
private List<String> buildExecutionCommand( ExecutionContext context, HadoopScript script) throws IOException, InterruptedException { assert context != null; assert script != null; List<String> command = new ArrayList<>(); command.add(getCommand(context, PATH_EXECUTE, script)); command.add(script.getClassName()); command.add(context.getBatchId()); command.add(context.getFlowId()); command.add(context.getExecutionId()); command.add(context.getArgumentsAsString()); Map<String, String> props = buildHadoopProperties(context, script); for (Map.Entry<String, String> entry : props.entrySet()) { command.add("-D"); command.add(MessageFormat.format("{0}={1}", entry.getKey(), entry.getValue())); } return command; }
/** * Creates a new instance. * @param script target script * @param context current execution context * @param handler handler which will attempt to resolve placeholders * @throws InterruptedException if interrupted to prepare this resolver * @throws IOException if failed to prepare replacements * @throws IllegalArgumentException if some parameters were {@code null} */ public PlaceholderResolver( ExecutionScript script, ExecutionContext context, ExecutionScriptHandler<?> handler) throws InterruptedException, IOException { if (script == null) { throw new IllegalArgumentException("script must not be null"); //$NON-NLS-1$ } if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } if (handler == null) { throw new IllegalArgumentException("handler must not be null"); //$NON-NLS-1$ } replacements = new HashMap<>(); replacements.put(PLACEHOLDER_HOME, getAsakusaHomePath(context, script, handler)); replacements.put(PLACEHOLDER_EXECUTION_ID, context.getExecutionId()); replacements.put(PLACEHOLDER_ARGUMENTS, context.getArgumentsAsString()); }
private Properties loadBatchConf(ExecutionContext context, ExecutionScript script) throws IOException { assert context != null; String fileName = context.getBatchId() + SUFFIX_CONF; File file = new File(confDirectory, fileName); LOG.debug("Finding multidispatch configuration file: batchId={}, file={}", context.getBatchId(), file); if (file.isFile() == false) { LOG.debug("Missing multidispatch configuration file: batchId={}, file={}", context.getBatchId(), file); return null; } LOG.debug("Loading multidispatch configuration file: batchId={}, file={}", context.getBatchId(), file); try (InputStream in = new FileInputStream(file)) { Properties properties = new Properties(); properties.load(in); return properties; } catch (IOException e) { YSLOG.error(e, "E01001", context.getBatchId(), file.getAbsolutePath()); throw e; } }
@Override String getKey(ExecutionContext context, ExecutionScript script) { return MessageFormat.format( "{0}.{1}", context.getFlowId(), WILDCARD); } },
@Override public void cleanUp( ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { HadoopScript script = new HadoopScript( context.getPhase().getSymbol(), Collections.emptySet(), CLEANUP_STAGE_CLASS, Collections.emptyMap(), Collections.emptyMap()); run(monitor, context, script); }
private Map<String, String> buildEnvironmentVariables( ExecutionContext context, ExecutionScript script) throws InterruptedException, IOException { assert script != null; Map<String, String> env = new HashMap<>(); env.putAll(getEnvironmentVariables(context, script)); env.putAll(context.getEnvironmentVariables()); env.putAll(script.getEnvironmentVariables()); return env; }
LOG.debug("Dispatch target found (batchId={}, flowId={}, phase={}, stageId={}, key={}, value={})", new Object[] { context.getBatchId(), context.getFlowId(), context.getPhase(), script == null ? LABEL_UNDEFINED : script.getId(), key, context.getBatchId(), context.getFlowId(), context.getPhase().getSymbol(), script == null ? LABEL_UNDEFINED : script.getId(), });
private Properties getBatchConf(ExecutionContext context, ExecutionScript script) throws IOException { assert context != null; Map<String, Properties> cached = confCache.get(); if (cached != null) { String batchId = context.getBatchId(); synchronized (this) { if (cached.containsKey(batchId)) { return cached.get(batchId); } } } Properties batchConf = loadBatchConf(context, script); synchronized (this) { cached = confCache.get(); if (cached == null) { cached = new HashMap<>(); confCache = new SoftReference<>(cached); } cached.put(context.getBatchId(), batchConf); } return batchConf; }
private Map<String, String> buildEnvironmentVariables( ExecutionContext context, ExecutionScript script) throws InterruptedException, IOException { assert script != null; Map<String, String> env = new HashMap<>(); env.putAll(getEnvironmentVariables(context, script)); env.putAll(context.getEnvironmentVariables()); env.putAll(script.getEnvironmentVariables()); return env; } }
/** * Computes the tracking ID. * @param context target context * @param script target script * @return the operation ID * @since 0.5.0 */ public static String computeTrackingId(ExecutionContext context, ExecutionScript script) { return MessageFormat.format( "YAESS/{0}/{1}/{2}/{3}/{4}", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId()); }
throw new IllegalStateException(MessageFormat.format( "job kind {2} is not enabled in this flow (batch={0}, flow={1})", context.getBatchId(), context.getFlowId(), execution.getKind())); throw new IOException(MessageFormat.format( "Profile \"{5}\" is not defined (batch={0}, flow={1}, phase={2}, module={3}, id={4})", context.getBatchId(), context.getFlowId(), context.getPhase().getSymbol(), exec.getModuleName(), exec.getId(),
/** * Performs as {@link #setUp(ExecutionMonitor, ExecutionContext)} that does nothing. * @param context current context * @since 0.4.0 */ protected final void voidSetUp(ExecutionContext context) { YSLOG.info("I51001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getHandlerId()); }
/** * Performs as {@link #cleanUp(ExecutionMonitor, ExecutionContext)} that does nothing. * @param context current context * @since 0.4.0 */ protected final void voidCleanUp(ExecutionContext context) { YSLOG.info("I51002", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getHandlerId()); } }