public void showFinalStatus() { YarnApplicationState state = report.getYarnApplicationState(); if (state == YarnApplicationState.FAILED || state == YarnApplicationState.FINISHED) { FinalApplicationStatus status = report.getFinalApplicationStatus(); System.out.println("Final status: " + status.toString()); if (status != FinalApplicationStatus.SUCCEEDED) { String diag = report.getDiagnostics(); if (!DoYUtil.isBlank(diag)) { System.out.println("Diagnostics: " + diag); } } } }
@Subscribe public void handleApplicationReportArrivalEvent(ApplicationReportArrivalEvent applicationReportArrivalEvent) { ApplicationReport applicationReport = applicationReportArrivalEvent.getApplicationReport(); YarnApplicationState appState = applicationReport.getYarnApplicationState(); LOGGER.info("Gobblin Yarn application state: " + appState.toString()); // Reset the count on failures to get the ApplicationReport when there's one success this.getApplicationReportFailureCount.set(0); if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED) { applicationCompleted = true; LOGGER.info("Gobblin Yarn application finished with final status: " + applicationReport.getFinalApplicationStatus().toString()); if (applicationReport.getFinalApplicationStatus() == FinalApplicationStatus.FAILED) { LOGGER.error("Gobblin Yarn application failed for the following reason: " + applicationReport.getDiagnostics()); } try { GobblinYarnAppLauncher.this.stop(); } catch (IOException ioe) { LOGGER.error("Failed to close the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe); } catch (TimeoutException te) { LOGGER.error("Timeout in stopping the service manager", te); } finally { if (this.emailNotificationOnShutdown) { sendEmailOnShutdown(Optional.of(applicationReport)); } } } }
+ ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false;
} else { LOG.info("Application has completed failed with YarnApplicationState=" + yarnApplicationState.toString() + " and FinalApplicationStatus=" + finalApplicationStatus.toString()); return false; isRunning.set(false); LOG.info("Application has completed with YarnApplicationState=" + yarnApplicationState.toString() + " and FinalApplicationStatus=" + finalApplicationStatus.toString()); return false;
+ ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false;
private String clairfyAppFinalStatus(FinalApplicationStatus status) { if (status == FinalApplicationStatus.UNDEFINED) { return "Application has not completed yet."; } return status.toString(); }
private String clairfyAppFinalStatus(FinalApplicationStatus status) { if (status == FinalApplicationStatus.UNDEFINED) { return "Application has not completed yet."; } return status.toString(); }
private String clairfyAppFinalStatus(FinalApplicationStatus status) { if (status == FinalApplicationStatus.UNDEFINED) { return "Application has not completed yet."; } return status.toString(); }
private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) { TimelineEntity entity = createAppAttemptEntity(event.getApplicationAttemptId()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); Map<String, Object> eventInfo = new HashMap<String, Object>(); eventInfo.put( AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, event.getTrackingUrl()); eventInfo.put( AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, event.getOriginalTrackingURL()); eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, event.getDiagnosticsInfo()); eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, event.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event.getYarnApplicationAttemptState().toString()); if (event.getMasterContainerId() != null) { eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event.getMasterContainerId().toString()); } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); }
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { TimelineEntity entity = createApplicationEntity(event.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( ApplicationMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); Map<String, Object> eventInfo = new HashMap<String, Object>(); eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, event.getDiagnosticsInfo()); eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event.getFinalApplicationStatus().toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event.getYarnApplicationState().toString()); if (event.getLatestApplicationAttemptId() != null) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, event.getLatestApplicationAttemptId().toString()); } RMAppMetrics appMetrics = event.getAppMetrics(); entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); }
public void testNotify() throws Exception { Configuration conf = setupEmbeddedContainer(QueryServlet.class, "/count/*", "/count/?status=$jobStatus", null); LauncherAMCallbackNotifier cn = new LauncherAMCallbackNotifier(conf); assertNull(QueryServlet.lastQueryString); cn.notifyURL(OozieActionResult.SUCCEEDED); waitForCallbackAndCheckResult(FinalApplicationStatus.SUCCEEDED.toString()); }
public void testNotifyBackgroundActionWhenSubmitFailsWithFailed() throws Exception { Configuration conf = setupEmbeddedContainer(QueryServlet.class, "/count/*", "/count/?status=$jobStatus", null); LauncherAMCallbackNotifier cn = new LauncherAMCallbackNotifier(conf); assertNull(QueryServlet.lastQueryString); cn.notifyURL(OozieActionResult.FAILED); waitForCallbackAndCheckResult(FinalApplicationStatus.FAILED.toString()); }
@SuppressWarnings("unchecked") @Override public void appAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) { ApplicationAttemptId attemptId = appAttempt.getAppAttemptId(); ApplicationAttemptEntity entity = createAppAttemptEntity(appAttempt.getAppAttemptId()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(finishedTime); entity.addEvent(tEvent); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO, appAttempt.getDiagnostics()); // app will get the final status from app attempt, or create one // based on app state if it doesn't exist entityInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO, app.getFinalApplicationStatus().toString()); entityInfo.put(AppAttemptMetricsConstants.STATE_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); entity.setInfo(entityInfo); entity.setIdPrefix( TimelineServiceHelper.invertLong(attemptId.getAttemptId())); getDispatcher().getEventHandler().handle( new TimelineV2PublishEvent(SystemMetricsEventType.PUBLISH_ENTITY, entity, appAttempt.getAppAttemptId().getApplicationId())); }
public SerializedApplicationReport(ApplicationReport report) { this.applicationId = report.getApplicationId().toString(); ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; this.name = report.getName(); this.applicationType = report.getApplicationType(); this.user = report.getUser(); this.queue = report.getQueue(); this.host = report.getHost(); this.rpcPort = report.getRpcPort(); this.state = report.getYarnApplicationState().toString(); this.diagnostics = report.getDiagnostics(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); this.finalStatus = appStatus == null ? "" : appStatus.toString(); this.progress = report.getProgress(); this.url = report.getTrackingUrl(); this.origTrackingUrl= report.getOriginalTrackingUrl(); }
public SerializedApplicationReport(ApplicationReport report) { this.applicationId = report.getApplicationId().toString(); ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; this.name = report.getName(); this.applicationType = report.getApplicationType(); this.user = report.getUser(); this.queue = report.getQueue(); this.host = report.getHost(); this.rpcPort = report.getRpcPort(); this.state = report.getYarnApplicationState().toString(); this.diagnostics = report.getDiagnostics(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); this.finalStatus = appStatus == null ? "" : appStatus.toString(); this.progress = report.getProgress(); this.url = report.getTrackingUrl(); this.origTrackingUrl= report.getOriginalTrackingUrl(); }
@Test public void testAppsQueryFinalStatusNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("finalStatus", FinalApplicationStatus.KILLED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); assertEquals("apps is not null", new JSONObject().toString(), json.get("apps").toString()); rm.stop(); }
@Test public void testAppsQueryFinalStatus() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); System.out.println(json.toString()); JSONObject apps = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); verifyAppInfo(array.getJSONObject(0), app1, false); rm.stop(); }
@Test public void testAppsQueryFinalStatusNone() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("finalStatus", FinalApplicationStatus.KILLED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); rm.stop(); }
@Test public void testAppsQueryFinalStatus() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); System.out.println(json.toString()); JSONObject apps = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); verifyAppInfo(array.getJSONObject(0), app1); rm.stop(); }
@Test public void testSingleApp() throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") .path(appId.toString()) .queryParam("user.name", USERS[round]) .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject app = json.getJSONObject("app"); assertEquals(appId.toString(), app.getString("appId")); assertEquals("test app", app.get("name")); assertEquals(round == 0 ? "test diagnostics info" : "", app.get("diagnosticsInfo")); assertEquals("test queue", app.get("queue")); assertEquals("user1", app.get("user")); assertEquals("test app type", app.get("type")); assertEquals(FinalApplicationStatus.UNDEFINED.toString(), app.get("finalAppStatus")); assertEquals(YarnApplicationState.FINISHED.toString(), app.get("appState")); }