List<ExecutablePO> allExecutable = executableDao.getJobs(); for (ExecutablePO executable : allExecutable) { long lastModified = executable.getLastModified(); ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid()); if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) { toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid()); toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid()); for (ExecutablePO task : executable.getTasks()) { toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
@Override public ExecutablePO reloadAt(String path) { try { ExecutablePO executablePO = readJobResource(path); if (executablePO == null) { logger.warn("No job found at {}, returning null", path); executableDigestMap.removeLocal(resourceName(path)); return null; } // create a digest ExecutablePO digestExecutablePO = new ExecutablePO(); digestExecutablePO.setUuid(executablePO.getUuid()); digestExecutablePO.setName(executablePO.getName()); digestExecutablePO.setLastModified(executablePO.getLastModified()); digestExecutablePO.setType(executablePO.getType()); digestExecutablePO.setParams(executablePO.getParams()); executableDigestMap.putLocal(resourceName(path), digestExecutablePO); return digestExecutablePO; } catch (Exception e) { throw new IllegalStateException("Error loading execute at " + path, e); } }
public ExecutablePO updateJob(ExecutablePO job) throws PersistentException { try { if (getJob(job.getUuid()) == null) { throw new IllegalArgumentException("job id:" + job.getUuid() + " does not exist"); } writeJobResource(pathOfJob(job), job); executableDigestMap.put(job.getId(), job); return job; } catch (IOException e) { logger.error("error update job:" + job.getUuid(), e); throw new PersistentException(e); } }
private static ExecutablePO parse(AbstractExecutable executable) { ExecutablePO result = new ExecutablePO(); result.setName(executable.getName()); result.setUuid(executable.getId()); result.setType(executable.getClass().getName()); result.setParams(executable.getParams()); if (executable instanceof ChainedExecutable) { List<ExecutablePO> tasks = Lists.newArrayList(); for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) { tasks.add(parse(task)); } result.setTasks(tasks); } if (executable instanceof CheckpointExecutable) { List<ExecutablePO> tasksForCheck = Lists.newArrayList(); for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) { tasksForCheck.add(parse(taskForCheck)); } result.setTasksForCheck(tasksForCheck); } return result; }
private static AbstractExecutable parseTo(ExecutablePO executablePO) { if (executablePO == null) { return null; } String type = executablePO.getType(); try { Class<? extends AbstractExecutable> clazz = ClassUtil.forName(type, AbstractExecutable.class); Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor(); AbstractExecutable result = constructor.newInstance(); result.setId(executablePO.getUuid()); result.setName(executablePO.getName()); result.setParams(executablePO.getParams()); List<ExecutablePO> tasks = executablePO.getTasks(); if (tasks != null && !tasks.isEmpty()) { Preconditions.checkArgument(result instanceof DefaultChainedExecutable); for (ExecutablePO subTask: tasks) { ((DefaultChainedExecutable) result).addTask(parseTo(subTask)); } } return result; } catch (ReflectiveOperationException e) { throw new IllegalArgumentException("cannot parse this job:" + executablePO.getId(), e); } }
return null; String type = executablePO.getType(); AbstractExecutable result = newExecutable(type); result.initConfig(config); result.setId(executablePO.getUuid()); result.setName(executablePO.getName()); result.setParams(executablePO.getParams()); List<ExecutablePO> tasks = executablePO.getTasks(); if (tasks != null && !tasks.isEmpty()) { Preconditions.checkArgument(result instanceof ChainedExecutable); List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck(); if (tasksForCheck != null && !tasksForCheck.isEmpty()) { Preconditions.checkArgument(result instanceof CheckpointExecutable);
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + kylinJobId); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + kylinJobId); for (ExecutablePO kylinTask : executablePO.getTasks()) { addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + kylinTask.getUuid()); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + kylinTask.getUuid()); if (includeYarnLogs) { yarnLogsResources.add(kylinTask.getUuid()); String cubeName = executablePO.getParams().get("cubeName"); if (!StringUtils.isEmpty(cubeName)) { File metaDir = new File(exportDir, "cube");
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId); for (ExecutablePO task : executablePO.getTasks()) { addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid()); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
private String pathOfJob(ExecutablePO job) { return pathOfJob(job.getUuid()); }
public void forceKillJob(String jobId) { try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks(); for (ExecutablePO task : tasks) { if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) { continue; } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) { updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } break; } if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) { jobOutput.setStatus(ExecutableState.ERROR.toString()); executableDao.updateJobOutput(jobOutput); } } catch (PersistentException e) { throw new RuntimeException(e); } }
public List<ExecutablePO> getJobDigests(long timeStart, long timeEndExclusive) { List<ExecutablePO> jobDigests = Lists.newArrayList(); for (ExecutablePO po : executableDigestMap.values()) { if (po.getLastModified() >= timeStart && po.getLastModified() < timeEndExclusive) jobDigests.add(po); } return jobDigests; }
private static AbstractExecutable parseTo(ExecutablePO executablePO) { if (executablePO == null) { return null; } String type = executablePO.getType(); try { Class<? extends AbstractExecutable> clazz = ClassUtil.forName(type, AbstractExecutable.class); Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor(); AbstractExecutable result = constructor.newInstance(); result.setId(executablePO.getUuid()); result.setName(executablePO.getName()); result.setParams(executablePO.getParams()); List<ExecutablePO> tasks = executablePO.getTasks(); if (tasks != null && !tasks.isEmpty()) { Preconditions.checkArgument(result instanceof ChainedExecutable); for (ExecutablePO subTask : tasks) { ((ChainedExecutable) result).addTask(parseTo(subTask)); } } return result; } catch (ReflectiveOperationException e) { throw new IllegalArgumentException("cannot parse this job:" + executablePO.getId(), e); } }
private static ExecutablePO parse(AbstractExecutable executable) { ExecutablePO result = new ExecutablePO(); result.setName(executable.getName()); result.setUuid(executable.getId()); result.setType(executable.getClass().getName()); result.setParams(executable.getParams()); if (executable instanceof DefaultChainedExecutable) { List<ExecutablePO> tasks = Lists.newArrayList(); for (AbstractExecutable task : ((DefaultChainedExecutable) executable).getTasks()) { tasks.add(parse(task)); } result.setTasks(tasks); } return result; }
return null; String type = executablePO.getType(); AbstractExecutable result = newExecutable(type); result.initConfig(config); result.setId(executablePO.getUuid()); result.setName(executablePO.getName()); result.setParams(executablePO.getParams()); List<ExecutablePO> tasks = executablePO.getTasks(); if (tasks != null && !tasks.isEmpty()) { Preconditions.checkArgument(result instanceof ChainedExecutable); List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck(); if (tasksForCheck != null && !tasksForCheck.isEmpty()) { Preconditions.checkArgument(result instanceof CheckpointExecutable);
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + kylinJobId); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + kylinJobId); for (ExecutablePO kylinTask : executablePO.getTasks()) { addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + kylinTask.getUuid()); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + kylinTask.getUuid()); if (includeYarnLogs) { yarnLogsResources.add(kylinTask.getUuid()); String cubeName = executablePO.getParams().get("cubeName"); if (!StringUtils.isEmpty(cubeName)) { File metaDir = new File(exportDir, "cube");
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId); for (ExecutablePO task : executablePO.getTasks()) { addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid()); addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
private String pathOfJob(ExecutablePO job) { return pathOfJob(job.getUuid()); } private String pathOfJob(String uuid) {
public void forceKillJob(String jobId) { try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks(); for (ExecutablePO task : tasks) { if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) { continue; } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) { updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } break; } if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) { jobOutput.setStatus(ExecutableState.ERROR.toString()); executableDao.updateJobOutput(jobOutput); } } catch (PersistentException e) { throw new RuntimeException(e); } }
public List<ExecutablePO> getJobDigests(long timeStart, long timeEndExclusive) { List<ExecutablePO> jobDigests = Lists.newArrayList(); for (ExecutablePO po : executableDigestMap.values()) { if (po.getLastModified() >= timeStart && po.getLastModified() < timeEndExclusive) jobDigests.add(po); } return jobDigests; }
@Override public ExecutablePO reloadAt(String path) { try { ExecutablePO executablePO = readJobResource(path); if (executablePO == null) { logger.warn("No job found at {}, returning null", path); executableDigestMap.removeLocal(resourceName(path)); return null; } // create a digest ExecutablePO digestExecutablePO = new ExecutablePO(); digestExecutablePO.setUuid(executablePO.getUuid()); digestExecutablePO.setName(executablePO.getName()); digestExecutablePO.setLastModified(executablePO.getLastModified()); digestExecutablePO.setType(executablePO.getType()); digestExecutablePO.setParams(executablePO.getParams()); executableDigestMap.putLocal(resourceName(path), digestExecutablePO); return digestExecutablePO; } catch (Exception e) { throw new IllegalStateException("Error loading execute at " + path, e); } }