@Override public ExecutableOutputPO reloadAt(String path) { try { ExecutableOutputPO executableOutputPO = readJobOutputResource(path); if (executableOutputPO == null) { logger.warn("No job output found at {}, returning null", path); executableOutputDigestMap.removeLocal(resourceName(path)); return null; } // create a digest ExecutableOutputPO digestExecutableOutputPO = new ExecutableOutputPO(); digestExecutableOutputPO.setUuid(executableOutputPO.getUuid()); digestExecutableOutputPO.setLastModified(executableOutputPO.getLastModified()); digestExecutableOutputPO.setStatus(executableOutputPO.getStatus()); executableOutputDigestMap.putLocal(resourceName(path), digestExecutableOutputPO); return digestExecutableOutputPO; } catch (Exception e) { throw new IllegalStateException("Error loading execute at " + path, e); } }
private DefaultOutput parseOutput(ExecutableOutputPO jobOutput) { final DefaultOutput result = new DefaultOutput(); result.setExtra(jobOutput.getInfo()); result.setState(ExecutableState.valueOf(jobOutput.getStatus())); result.setVerboseMsg(jobOutput.getContent()); result.setLastModified(jobOutput.getLastModified()); return result; }
public Map<String, ExecutableOutputPO> getAllOutputDigests(long timeStartInMillis, long timeEndInMillis) { final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputDigests(timeStartInMillis, timeEndInMillis); HashMap<String, ExecutableOutputPO> result = Maps.newHashMap(); for (ExecutableOutputPO jobOutput : jobOutputs) { result.put(jobOutput.getId(), jobOutput); } return result; }
public void addJobOutput(ExecutableOutputPO output) throws PersistentException { try { output.setLastModified(0); writeJobOutputResource(pathOfJobOutput(output.getUuid()), output); if (!isTaskExecutableOutput(output.getUuid())) executableOutputDigestMap.put(output.getUuid(), output); } catch (IOException e) { logger.error("error update job output id:" + output.getUuid(), e); throw new PersistentException(e); } }
public ExecutableOutputPO getJobOutput(String uuid) throws PersistentException { ExecutableOutputPO result = null; try { result = readJobOutputResource(pathOfJobOutput(uuid)); if (result == null) { result = new ExecutableOutputPO(); result.setUuid(uuid); return result; } return result; } catch (IOException e) { logger.error("error get job output id:" + uuid, e); if (e.getCause() instanceof FileNotFoundException) { result = new ExecutableOutputPO(); result.setUuid(uuid); result.setStatus(ExecutableState.SUCCEED.name()); return result; } else { throw new PersistentException(e); } } }
public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) { try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId); ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus()); if (newStatus != null && oldStatus != newStatus) { if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) { throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus); } jobOutput.setStatus(newStatus.toString()); } if (info != null) { jobOutput.setInfo(info); } if (output != null) { jobOutput.setContent(output); } executableDao.updateJobOutput(jobOutput); logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus); } catch (PersistentException e) { logger.error("error change job:" + jobId + " to " + newStatus.toString()); throw new RuntimeException(e); } }
public void updateAllRunningJobsToError() { try { final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs(); for (ExecutableOutputPO executableOutputPO : jobOutputs) { if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) { executableOutputPO.setStatus(ExecutableState.ERROR.toString()); executableDao.updateJobOutput(executableOutputPO); } } } catch (PersistentException e) { logger.error("error reset job status from RUNNING to ERROR", e); throw new RuntimeException(e); } }
public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) { // when if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Current thread is interruptted, aborting"); } try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId); ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus()); if (newStatus != null && oldStatus != newStatus) { if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) { throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus + ", job id: " + jobId); } jobOutput.setStatus(newStatus.toString()); } if (info != null) { jobOutput.getInfo().putAll(info); } if (output != null) { jobOutput.setContent(output); } executableDao.updateJobOutput(jobOutput); logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus); } catch (PersistentException e) { logger.error("error change job:" + jobId + " to " + newStatus); throw new RuntimeException(e); } }
@Override public boolean apply(CheckpointExecutable executable) { try { ExecutableOutputPO executableOutputPO = allExecutableOutputPO .get(executable.getId()); ExecutableState state = ExecutableState.valueOf(executableOutputPO.getStatus()); return statusList.contains(state); } catch (Exception e) { throw e; } } }, new Predicate<CheckpointExecutable>() {
private void addJobOutput(AbstractExecutable executable) throws PersistentException { ExecutableOutputPO executableOutputPO = new ExecutableOutputPO(); executableOutputPO.setUuid(executable.getId()); executableDao.addJobOutput(executableOutputPO); if (executable instanceof DefaultChainedExecutable) { for (AbstractExecutable subTask : ((DefaultChainedExecutable) executable).getTasks()) { addJobOutput(subTask); } } }
private void extractJobInfo(String taskId, File destDir) throws Exception { final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo(); if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) { String mrJobId = jobInfo.get(ExecutableConstants.MR_JOB_ID); FileUtils.forceMkdir(destDir); String[] mrJobArgs = { "-mrJobId", mrJobId, "-destDir", destDir.getAbsolutePath(), "-compress", "false", "-submodule", "true" }; new MrJobInfoExtractor().execute(mrJobArgs); } }
public void resetJobOutput(String jobId, ExecutableState state, String output) { try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); jobOutput.setStatus(state.toString()); if (output != null) { jobOutput.setContent(output); } executableDao.updateJobOutput(jobOutput); } catch (PersistentException e) { throw new RuntimeException(e); } }
public List<ExecutableOutputPO> getJobOutputDigests(long timeStart, long timeEndExclusive) { List<ExecutableOutputPO> jobOutputDigests = Lists.newArrayList(); for (ExecutableOutputPO po : executableOutputDigestMap.values()) { if (po.getLastModified() >= timeStart && po.getLastModified() < timeEndExclusive) jobOutputDigests.add(po); } return jobOutputDigests; }
public void updateJobOutput(ExecutableOutputPO output) throws PersistentException { try { writeJobOutputResource(pathOfJobOutput(output.getUuid()), output); if (!isTaskExecutableOutput(output.getUuid())) executableOutputDigestMap.put(output.getUuid(), output); } catch (IOException e) { logger.error("error update job output id:" + output.getUuid(), e); throw new PersistentException(e); } }
public void updateJobOutput(String jobId, ExecutableState newStatus, Map<String, String> info, String output) { // when if (Thread.currentThread().isInterrupted()) { throw new RuntimeException("Current thread is interruptted, aborting"); } try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + jobId); ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus()); if (newStatus != null && oldStatus != newStatus) { if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) { throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus + ", job id: " + jobId); } jobOutput.setStatus(newStatus.toString()); } if (info != null) { jobOutput.setInfo(info); } if (output != null) { jobOutput.setContent(output); } executableDao.updateJobOutput(jobOutput); logger.info("job id:" + jobId + " from " + oldStatus + " to " + newStatus); } catch (PersistentException e) { logger.error("error change job:" + jobId + " to " + newStatus); throw new RuntimeException(e); } }
public void resumeAllRunningJobs() { try { final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs(); for (ExecutableOutputPO executableOutputPO : jobOutputs) { if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) { executableOutputPO.setStatus(ExecutableState.READY.toString()); executableDao.updateJobOutput(executableOutputPO); } } } catch (PersistentException e) { logger.error("error reset job status from RUNNING to READY", e); throw new RuntimeException(e); } }
@Override public boolean apply(CubingJob executable) { try { ExecutableOutputPO executableOutputPO = allExecutableOutputPO .get(executable.getId()); ExecutableState state = ExecutableState.valueOf(executableOutputPO.getStatus()); return statusList.contains(state); } catch (Exception e) { throw e; } } }, new Predicate<CubingJob>() {
public void updateJobOutput(ExecutableOutputPO output) throws PersistentException { try { final long ts = writeJobOutputResource(pathOfJobOutput(output.getUuid()), output); output.setLastModified(ts); } catch (IOException e) { logger.error("error update job output id:" + output.getUuid(), e); throw new PersistentException(e); } }
public ExecutableOutputPO getJobOutput(String uuid) throws PersistentException { try { ExecutableOutputPO result = readJobOutputResource(pathOfJobOutput(uuid)); if (result == null) { result = new ExecutableOutputPO(); result.setUuid(uuid); return result; } return result; } catch (IOException e) { logger.error("error get job output id:" + uuid, e); throw new PersistentException(e); } }
private void extractJobLog(String taskId, File destDir, boolean onlyFail) throws Exception { final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo(); FileUtils.forceMkdir(destDir); if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) { String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job", "application"); if (!onlyFail || !isYarnAppSucc(applicationId)) { File destFile = new File(destDir, applicationId + ".log"); String yarnCmd = "yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath(); logger.debug(yarnCmd); try { kylinConfig.getCliCommandExecutor().execute(yarnCmd); } catch (Exception ex) { logger.warn("Failed to get yarn logs. ", ex); } } } }