@Override public void onJobCancellation(JobContext jobContext) { this.jobExecutionEventSubmitter.submitJobExecutionEvents(jobContext.getJobState()); } }
@Override public void onJobCompletion(JobContext jobContext) { this.jobExecutionEventSubmitter.submitJobExecutionEvents(jobContext.getJobState()); }
@Override public String toString() { return Objects.toStringHelper(JobContext.class.getSimpleName()).add("jobName", getJobName()) .add("jobId", getJobId()).add("jobState", getJobState()).toString(); } }
@Override public void onJobPrepare(JobContext jobContext) throws Exception { super.onJobPrepare(jobContext); jobContext.getJobState().setProp(JOB_START_TIME, Long.toString(System.nanoTime())); jobLauncherMetrics.totalJobsLaunched.incrementAndGet(); }
public Map<String, String> getMetadata(JobContext jobContext, EventName eventName) { List<TaskState> taskStates = jobContext.getJobState().getTaskStates(); switch (eventName) { case JOB_COMPLETE: return ImmutableMap.of(PROCESSED_COUNT_KEY, Long.toString(EventMetadataUtils.getProcessedCount(taskStates))); case JOB_FAILED: return ImmutableMap.of(MESSAGE_KEY, EventMetadataUtils.getTaskFailureExceptions(taskStates)); default: break; } return ImmutableMap.of(); } }
void checkForUnpublishedWUHandling(String datasetUrn, JobState.DatasetState datasetState, Class<? extends DataPublisher> dataPublisherClass, Closer closer) throws ReflectiveOperationException, IOException { if (UnpublishedHandling.class.isAssignableFrom(dataPublisherClass)) { // pass in jobstate to retrieve properties DataPublisher publisher = closer.register(DataPublisher.getInstance(dataPublisherClass, this.jobContext.getJobState())); log.info(String.format("Calling publisher to handle unpublished work units for dataset %s of job %s.", datasetUrn, this.jobContext.getJobId())); ((UnpublishedHandling) publisher).handleUnpublishedWorkUnits(datasetState.getTaskStatesAsWorkUnitStates()); } }
private void shutDown() throws IOException { _log.info("Shutting down " + getClass().getSimpleName()); if (null != _jobContext) { switch (_jobContext.getJobState().getState()) { case PENDING: case SUCCESSFUL: case RUNNING: { // We have to pass another listener instance as launcher does not store the listener used // in launchJob() cancel(false); break; } case FAILED: case COMMITTED: case CANCELLED: { // Nothing to do break; } } } _closer.close(); }
/** * From {@link org.apache.gobblin.runtime.AbstractJobLauncher#launchJob(JobListener)}, the final * job state should only be FAILED or COMMITTED. This means the completed jobs metrics covers * both failed jobs and committed jobs. */ @Override public void onJobCompletion(JobContext jobContext) throws Exception { super.onJobCompletion(jobContext); long startTime = jobContext.getJobState().getPropAsLong(JOB_START_TIME); jobLauncherMetrics.totalJobsCompleted.incrementAndGet(); Instrumented.updateTimer(Optional.of(jobLauncherMetrics.timeForCompletedJobs), System.nanoTime() - startTime, TimeUnit.NANOSECONDS); if (jobContext.getJobState().getState() == JobState.RunningState.FAILED) { jobLauncherMetrics.totalJobsFailed.incrementAndGet(); Instrumented.updateTimer(Optional.of(jobLauncherMetrics.timeForFailedJobs), System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } else { jobLauncherMetrics.totalJobsCommitted.incrementAndGet(); Instrumented.updateTimer(Optional.of(jobLauncherMetrics.timeForCommittedJobs), System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); if (!jobState.contains(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)) { LOG.error("Job configuration file path not found in job state of job " + jobState.getJobId()); return; } String jobConfigFile = jobState.getProp(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY); // Rename the config file so we won't run this job when the worker is bounced try { Files.move(new File(jobConfigFile), new File(jobConfigFile + ".done")); } catch (IOException ioe) { LOG.error("Failed to rename job configuration file for job " + jobState.getJobName(), ioe); } } }
@Override public void run() { synchronized (AbstractJobLauncher.this.cancellationRequest) { try { while (!AbstractJobLauncher.this.cancellationRequested) { // Wait for a cancellation request to arrive AbstractJobLauncher.this.cancellationRequest.wait(); } LOG.info("Cancellation has been requested for job " + AbstractJobLauncher.this.jobContext.getJobId()); executeCancellation(); LOG.info("Cancellation has been executed for job " + AbstractJobLauncher.this.jobContext.getJobId()); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } } synchronized (AbstractJobLauncher.this.cancellationExecution) { AbstractJobLauncher.this.cancellationExecuted = true; AbstractJobLauncher.this.jobContext.getJobState().setState(JobState.RunningState.CANCELLED); // Notify that the cancellation has been executed AbstractJobLauncher.this.cancellationExecution.notifyAll(); } } });
@Override public void onJobCancellation(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); if (notificationEmailEnabled) { try { EmailUtils.sendJobCancellationEmail(jobState.getJobId(), jobState.toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job cancellation notification email for job " + jobState.getJobId(), ee); } } } }
/** * FIXME this method is provided for backwards compatibility in the LocalJobLauncher since it does * not access the task state store. This should be addressed as all task executions should be * updating the task state. */ public static GobblinMultiTaskAttempt runWorkUnits(JobContext jobContext, Iterator<WorkUnit> workUnits, TaskStateTracker taskStateTracker, TaskExecutor taskExecutor, CommitPolicy multiTaskAttemptCommitPolicy) throws IOException, InterruptedException { GobblinMultiTaskAttempt multiTaskAttempt = new GobblinMultiTaskAttempt(workUnits, jobContext.getJobId(), jobContext.getJobState(), taskStateTracker, taskExecutor, Optional.<String>absent(), Optional.<StateStore<TaskState>>absent(), jobContext.getJobBroker()); multiTaskAttempt.runAndOptionallyCommitTaskAttempt(multiTaskAttemptCommitPolicy); return multiTaskAttempt; }
@SuppressWarnings("unchecked") private Optional<CommitSequence.Builder> generateCommitSequenceBuilder(JobState.DatasetState datasetState, Collection<TaskState> taskStates) { try (Closer closer = Closer.create()) { Class<? extends CommitSequencePublisher> dataPublisherClass = (Class<? extends CommitSequencePublisher>) Class .forName(datasetState .getProp(ConfigurationKeys.DATA_PUBLISHER_TYPE, ConfigurationKeys.DEFAULT_DATA_PUBLISHER_TYPE)); CommitSequencePublisher publisher = (CommitSequencePublisher) closer .register(DataPublisher.getInstance(dataPublisherClass, this.jobContext.getJobState())); publisher.publish(taskStates); return publisher.getCommitSequenceBuilder(); } catch (Throwable t) { log.error("Failed to generate commit sequence", t); setTaskFailureException(datasetState.getTaskStates(), t); throw Throwables.propagate(t); } }
@Override public void onJobCompletion(JobContext jobContext) throws Exception { Preconditions.checkArgument(jobContext.getJobState().getState() == RunningState.SUCCESSFUL || jobContext.getJobState().getState() == RunningState.COMMITTED || jobContext.getJobState().getState() == RunningState.FAILED, "Unexpected state: " + jobContext.getJobState().getState() + " in " + jobContext); super.onJobCompletion(jobContext); if (_instrumentationEnabled && null != _launcherMetrics) { _launcherMetrics.getNumJobsCompleted().inc(); } if (jobContext.getJobState().getState() == RunningState.FAILED) { if (_instrumentationEnabled && null != _launcherMetrics) { _launcherMetrics.getNumJobsFailed().inc(); } _jobState.switchToFailed(); } else { // TODO Remove next line once the JobLauncher starts sending notifications for success _jobState.switchToSuccessful(); _jobState.switchToCommitted(); if (_instrumentationEnabled && null != _launcherMetrics) { _launcherMetrics.getNumJobsCommitted().inc(); } } }
public LocalJobLauncher(Properties jobProps, SharedResourcesBroker<GobblinScopeTypes> instanceBroker, List<? extends Tag<?>> metadataTags) throws Exception { super(jobProps, metadataTags, instanceBroker); log.debug("Local job launched with properties: {}", jobProps); TimingEvent jobLocalSetupTimer = this.eventSubmitter.getTimingEvent(TimingEvent.RunJobTimings.JOB_LOCAL_SETUP); this.taskExecutor = new TaskExecutor(jobProps); this.taskStateTracker = new LocalTaskStateTracker(jobProps, this.jobContext.getJobState(), this.taskExecutor, this.eventBus); this.serviceManager = new ServiceManager(Lists.newArrayList( // The order matters due to dependencies between services this.taskExecutor, this.taskStateTracker)); // Start all dependent services this.serviceManager.startAsync().awaitHealthy(5, TimeUnit.SECONDS); startCancellationExecutor(); jobLocalSetupTimer.stop(); }
@Override public void close() throws IOException { try { this.cancellationExecutor.shutdownNow(); try { this.jobContext.getSource().shutdown(this.jobContext.getJobState()); } finally { if (GobblinMetrics.isEnabled(this.jobProps)) { GobblinMetricsRegistry.getInstance().remove(this.jobContext.getJobId()); } } } finally { unlockJob(); } }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean alertEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, Boolean.toString(false))); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); // Send out alert email if the maximum number of consecutive failures is reached if (jobState.getState() == JobState.RunningState.FAILED) { int failures = jobState.getPropAsInt(ConfigurationKeys.JOB_FAILURES_KEY, 0); int maxFailures = jobState.getPropAsInt(ConfigurationKeys.JOB_MAX_FAILURES_KEY, ConfigurationKeys.DEFAULT_JOB_MAX_FAILURES); if (alertEmailEnabled && failures >= maxFailures) { try { EmailUtils.sendJobFailureAlertEmail(jobState.getJobName(), jobState.toString(), failures, jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job failure alert email for job " + jobState.getJobId(), ee); } return; } } if (notificationEmailEnabled) { try { EmailUtils.sendJobCompletionEmail( jobState.getJobId(), jobState.toString(), jobState.getState().toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job completion notification email for job " + jobState.getJobId(), ee); } } }
private void notifyListeners(JobContext jobContext, JobListener jobListener, String timerEventName, JobListenerAction action) throws JobException { TimingEvent timer = this.eventSubmitter.getTimingEvent(timerEventName); try (CloseableJobListener parallelJobListener = getParallelCombinedJobListener(this.jobContext.getJobState(), jobListener)) { action.apply(parallelJobListener, jobContext); } catch (Exception e) { throw new JobException("Failed to execute all JobListeners", e); } finally { timer.stop(this.eventMetadataGenerator.getMetadata(this.jobContext, EventName.getEnumFromEventId(timerEventName))); } }
jobState.addTaskState(taskState2); Mockito.when(jobContext.getJobState()).thenReturn(jobState);
@Test public void testProcessedCount() throws Exception { JobContext jobContext = Mockito.mock(JobContext.class); JobState jobState = new JobState("jobName", "1234"); TaskState taskState1 = new TaskState(); TaskState taskState2 = new TaskState(); taskState1.setTaskId("1"); taskState1.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "1"); taskState2.setTaskId("2"); taskState2.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "22"); jobState.addTaskState(taskState1); jobState.addTaskState(taskState2); Mockito.when(jobContext.getJobState()).thenReturn(jobState); ClusterEventMetadataGenerator metadataGenerator = new ClusterEventMetadataGenerator(); Map<String, String> metadata; // processed count is not in job cancel event metadata = metadataGenerator.getMetadata(jobContext, EventName.JOB_CANCEL); Assert.assertEquals(metadata.get("processedCount"), null); // processed count is in job complete event metadata = metadataGenerator.getMetadata(jobContext, EventName.getEnumFromEventId("JobCompleteTimer")); Assert.assertEquals(metadata.get("processedCount"), "23"); }