public Map<String, JobExecutionAPIEntity> recoverYarnApp(String appId) throws Exception { Map<String, Pair<Map<String, String>, AppInfo>> result = this.runningJobManager.recoverYarnApp(appId); Map<String, JobExecutionAPIEntity> jobs = new HashMap<>(); for (String jobId : result.keySet()) { Pair<Map<String, String>, AppInfo> job = result.get(jobId); JobExecutionAPIEntity jobExecutionAPIEntity = new JobExecutionAPIEntity(); jobExecutionAPIEntity.setTags(job.getLeft()); jobExecutionAPIEntity.setAppInfo(job.getRight()); jobExecutionAPIEntity.setTimestamp(job.getRight().getStartedTime()); jobs.put(jobId, jobExecutionAPIEntity); } return jobs; }
public Map<String, SparkAppEntity> recoverYarnApp(String appId) throws Exception { Map<String, Pair<Map<String, String>, AppInfo>> result = this.runningJobManager.recoverYarnApp(appId); Map<String, SparkAppEntity> apps = new HashMap<>(); for (String jobId : result.keySet()) { Pair<Map<String, String>, AppInfo> job = result.get(jobId); SparkAppEntity sparkAppEntity = new SparkAppEntity(); sparkAppEntity.setTags(job.getLeft()); sparkAppEntity.setAppInfo(job.getRight()); sparkAppEntity.setTimestamp(job.getRight().getStartedTime()); apps.put(jobId, sparkAppEntity); } return apps; }
public Map<String, Map<String, Pair<Map<String, String>, AppInfo>>> recover() { //we need read from zookeeper, path looks like /apps/x/running/yarnAppId/jobId/ //content of path /apps/x/running/yarnAppId/jobId is Pair<Map<String, String>, AppInfo> //Pair is entity tags and AppInfo //as we know, a yarn application may contains many mr jobs or spark applications //so, the returned results is a Map-Map //<yarnAppId, <jobId, Pair<<Map<String, String>, AppInfo>>> Map<String, Map<String, Pair<Map<String, String>, AppInfo>>> result = new HashMap<>(); try { lock.acquire(); List<String> yarnAppIds = curator.getChildren().forPath(this.zkRoot); for (String yarnAppId : yarnAppIds) { if (!result.containsKey(yarnAppId)) { result.put(yarnAppId, new HashMap<>()); } result.put(yarnAppId, recoverYarnApp(yarnAppId)); } } catch (Exception e) { LOG.error("fail to recover", e); throw new RuntimeException(e); } finally { try { lock.release(); } catch (Exception e) { LOG.error("fail releasing lock", e); } } return result; }