/** * Write a summary to the json document * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} * used to write the json document */ protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { jsonWriter.name("job name").value(this.getJobName()).name("job id").value(this.getJobId()).name("job state") .value(this.getState().name()).name("start time").value(this.getStartTime()).name("end time") .value(this.getEndTime()).name("duration").value(this.getDuration()).name("tasks").value(this.getTaskCount()) .name("completed tasks").value(this.getCompletedTasks()); }
private void shutDown() throws IOException { _log.info("Shutting down " + getClass().getSimpleName()); if (null != _jobContext) { switch (_jobContext.getJobState().getState()) { case PENDING: case SUCCESSFUL: case RUNNING: { // We have to pass another listener instance as launcher does not store the listener used // in launchJob() cancel(false); break; } case FAILED: case COMMITTED: case CANCELLED: { // Nothing to do break; } } } _closer.close(); }
/** * Submits an event for the given {@link JobState}. */ private void submitJobStateEvent(JobState jobState) { ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_START_TIME, Long.toString(jobState.getStartTime())); jobMetadataBuilder.put(METADATA_JOB_END_TIME, Long.toString(jobState.getEndTime())); jobMetadataBuilder.put(METADATA_JOB_STATE, jobState.getState().toString()); jobMetadataBuilder.put(METADATA_JOB_LAUNCHED_TASKS, Integer.toString(jobState.getTaskCount())); jobMetadataBuilder.put(METADATA_JOB_COMPLETED_TASKS, Integer.toString(jobState.getCompletedTasks())); jobMetadataBuilder.put(METADATA_JOB_LAUNCHER_TYPE, jobState.getLauncherType().toString()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); jobMetadataBuilder.put(EventSubmitter.EVENT_TYPE, JOB_STATE); this.eventSubmitter.submit(JOB_STATE, jobMetadataBuilder.build()); }
/** * From {@link org.apache.gobblin.runtime.AbstractJobLauncher#launchJob(JobListener)}, the final * job state should only be FAILED or COMMITTED. This means the completed jobs metrics covers * both failed jobs and committed jobs. */ @Override public void onJobCompletion(JobContext jobContext) throws Exception { super.onJobCompletion(jobContext); long startTime = jobContext.getJobState().getPropAsLong(JOB_START_TIME); jobLauncherMetrics.totalJobsCompleted.incrementAndGet(); Instrumented.updateTimer(Optional.of(jobLauncherMetrics.timeForCompletedJobs), System.nanoTime() - startTime, TimeUnit.NANOSECONDS); if (jobContext.getJobState().getState() == JobState.RunningState.FAILED) { jobLauncherMetrics.totalJobsFailed.incrementAndGet(); Instrumented.updateTimer(Optional.of(jobLauncherMetrics.timeForFailedJobs), System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } else { jobLauncherMetrics.totalJobsCommitted.incrementAndGet(); Instrumented.updateTimer(Optional.of(jobLauncherMetrics.timeForCommittedJobs), System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } }
@Override public void onJobCompletion(JobContext jobContext) throws Exception { Preconditions.checkArgument(jobContext.getJobState().getState() == RunningState.SUCCESSFUL || jobContext.getJobState().getState() == RunningState.COMMITTED || jobContext.getJobState().getState() == RunningState.FAILED, "Unexpected state: " + jobContext.getJobState().getState() + " in " + jobContext); super.onJobCompletion(jobContext); if (_instrumentationEnabled && null != _launcherMetrics) { _launcherMetrics.getNumJobsCompleted().inc(); } if (jobContext.getJobState().getState() == RunningState.FAILED) { if (_instrumentationEnabled && null != _launcherMetrics) { _launcherMetrics.getNumJobsFailed().inc(); } _jobState.switchToFailed(); } else { // TODO Remove next line once the JobLauncher starts sending notifications for success _jobState.switchToSuccessful(); _jobState.switchToCommitted(); if (_instrumentationEnabled && null != _launcherMetrics) { _launcherMetrics.getNumJobsCommitted().inc(); } } }
@Test public void testSubmitJobExecutionEvents() { JobState mockJobState = mock(JobState.class, Mockito.RETURNS_SMART_NULLS); when(mockJobState.getState()).thenReturn(JobState.RunningState.SUCCESSFUL); when(mockJobState.getLauncherType()).thenReturn(LauncherTypeEnum.$UNKNOWN); when(mockJobState.getTrackingURL()).thenReturn(Optional.<String> absent()); TaskState mockTaskState1 = createMockTaskState(); TaskState mockTaskState2 = createMockTaskState(); when(mockJobState.getTaskStates()).thenReturn(Lists.newArrayList(mockTaskState1, mockTaskState2)); this.jobExecutionEventSubmitter.submitJobExecutionEvents(mockJobState); verify(this.mockEventSubmitter, atLeastOnce()).submit(any(String.class), any(Map.class)); }
LOG.info("Current job state is: " + this.jobContext.getJobState().getState()); if (this.jobContext.getJobState().getState() != JobState.RunningState.COMMITTED && ( this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_SUCCESSFUL_TASKS || this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_PARTIAL_SUCCESS)) {
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean alertEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, Boolean.toString(false))); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); // Send out alert email if the maximum number of consecutive failures is reached if (jobState.getState() == JobState.RunningState.FAILED) { int failures = jobState.getPropAsInt(ConfigurationKeys.JOB_FAILURES_KEY, 0); int maxFailures = jobState.getPropAsInt(ConfigurationKeys.JOB_MAX_FAILURES_KEY, ConfigurationKeys.DEFAULT_JOB_MAX_FAILURES); if (alertEmailEnabled && failures >= maxFailures) { try { EmailUtils.sendJobFailureAlertEmail(jobState.getJobName(), jobState.toString(), failures, jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job failure alert email for job " + jobState.getJobId(), ee); } return; } } if (notificationEmailEnabled) { try { EmailUtils.sendJobCompletionEmail( jobState.getJobId(), jobState.toString(), jobState.getState().toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job completion notification email for job " + jobState.getJobId(), ee); } } }
if (jobState.getState() == JobState.RunningState.RUNNING) { jobState.setState(JobState.RunningState.SUCCESSFUL);
Assert.assertEquals(jobContext.getJobState().getState(), JobState.RunningState.COMMITTED);
Assert.assertEquals(jobContext.getJobState().getState(), JobState.RunningState.COMMITTED);
private void verifyJobState(int run) throws IOException { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(JOB_NAME, "current.jst"); Assert.assertEquals(datasetStateList.size(), 1); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskStates().size(), DummySource.NUM_WORK_UNITS); for (TaskState taskState : jobState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getProp(FOO), BAR); // Check if the low watermark is properly kept track of int expectedLowWatermark = (run - 1) * DummySource.NUM_WORK_UNITS * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + taskState.getPropAsInt(WORK_UNIT_INDEX_KEY) * DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR + 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY), expectedLowWatermark); // Check if the high watermark is properly kept track of int expectedHighWatermark = expectedLowWatermark + DummySource.NUM_RECORDS_TO_EXTRACT_PER_EXTRACTOR - 1; Assert.assertEquals(taskState.getPropAsInt(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY), expectedHighWatermark); Assert.assertEquals(taskState.getPropAsInt(LAST_READ_RECORD_KEY), expectedHighWatermark); } }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
Assert.assertEquals(jobContext.getJobState().getState(), JobState.RunningState.FAILED);
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskCount(), 3); Assert.assertEquals(jobState.getProp("foo"), "bar");
.submit(CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, "JOB_" + jobState.getState())); if (jobState.getState() == JobState.RunningState.CANCELLED) { LOG.info(String.format("Job %s has been cancelled, aborting now", jobId)); return; if (jobState.getState() == JobState.RunningState.FAILED) { notifyListeners(this.jobContext, jobListener, TimingEvent.LauncherTimings.JOB_FAILED, new JobListenerAction() { @Override