/** * Get a new {@link GobblinMetrics} instance for a given job. * * @param jobName job name * @param jobId job ID * @return a new {@link GobblinMetrics} instance for the given job */ public static JobMetrics get(String jobName, String jobId) { return get(new JobState(jobName, jobId)); }
/** * Write a summary to the json document * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} * used to write the json document */ protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { jsonWriter.name("job name").value(this.getJobName()).name("job id").value(this.getJobId()).name("job state") .value(this.getState().name()).name("start time").value(this.getStartTime()).name("end time") .value(this.getEndTime()).name("duration").value(this.getDuration()).name("tasks").value(this.getTaskCount()) .name("completed tasks").value(this.getCompletedTasks()); }
/** * Finalize the {@link JobState} before committing the job. */ void finalizeJobStateBeforeCommit() { this.jobState.setEndTime(System.currentTimeMillis()); this.jobState.setDuration(this.jobState.getEndTime() - this.jobState.getStartTime()); for (TaskState taskState : this.jobState.getTaskStates()) { // Set fork.branches explicitly here so the rest job flow can pick it up this.jobState .setProp(ConfigurationKeys.FORK_BRANCHES_KEY, taskState.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1)); } }
public JobState(State properties, Map<String, JobState.DatasetState> previousDatasetStates, String jobName, String jobId) { super(properties, previousDatasetStates, workUnitStatesFromDatasetStates(previousDatasetStates.values())); this.jobName = jobName; this.jobId = jobId; this.setId(jobId); }
@Test public void testPersistJobState() throws IOException { JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID); jobState.setId(TEST_JOB_ID); jobState.setProp("foo", "bar"); jobState.setState(JobState.RunningState.COMMITTED); jobState.setStartTime(this.startTime); jobState.setEndTime(this.startTime + 1000); jobState.setDuration(1000); for (int i = 0; i < 3; i++) { TaskState taskState = new TaskState(); taskState.setJobId(TEST_JOB_ID); taskState.setTaskId(TEST_TASK_ID_PREFIX + i); taskState.setId(TEST_TASK_ID_PREFIX + i); taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); jobState.addTaskState(taskState); } this.fsJobStateStore.put(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, jobState); }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
private void doAsserts(JobState jobState, boolean considerTaskStates, boolean initial) { Assert.assertEquals(jobState.getJobName(), "TestJob"); Assert.assertEquals(jobState.getJobId(), "TestJob-1"); Assert.assertEquals(jobState.getId(), "TestJob-1"); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskCount(), 3); Assert.assertEquals(jobState.getProp("foo"), "bar"); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), "TestJob-1"); Assert.assertEquals(taskState.getStartTime(), this.startTime); Assert.assertEquals(taskStateIds, Lists.newArrayList("TestTask-0", "TestTask-1", "TestTask-2")); Set<String> sortedDatasetUrns = Sets.newTreeSet(jobState.createDatasetStatesByUrns().keySet()); Assert.assertEquals(sortedDatasetUrns.size(), jobState.getCompletedTasks()); Map<String, JobState.DatasetState> datasetStatesByUrns = jobState.createDatasetStatesByUrns(); int index = 0; for (String dataSetUrn : sortedDatasetUrns) {
executeUnfinishedCommitSequences(jobState.getJobName()); jobState.setState(JobState.RunningState.FAILED); throw new JobException("Failed to get work units for job " + jobId); this.eventSubmitter.submit(JobEvent.WORK_UNITS_EMPTY); LOG.warn("No work units have been created for job " + jobId); jobState.setState(JobState.RunningState.COMMITTED); isWorkUnitsEmpty = true; return; jobState.setStartTime(startTime); jobState.setState(JobState.RunningState.RUNNING); if (jobState.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_ENABLE_TRACKING_LOGS)) { workUnitStream = workUnitStream.transform(new Function<WorkUnit, WorkUnit>() { @Nullable .submit(CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, "JOB_" + jobState.getState())); if (jobState.getState() == JobState.RunningState.CANCELLED) { LOG.info(String.format("Job %s has been cancelled, aborting now", jobId)); return; } finally { long endTime = System.currentTimeMillis(); jobState.setEndTime(endTime); jobState.setDuration(endTime - jobState.getStartTime());
/** * Submits an event for the given {@link JobState}. */ private void submitJobStateEvent(JobState jobState) { ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_START_TIME, Long.toString(jobState.getStartTime())); jobMetadataBuilder.put(METADATA_JOB_END_TIME, Long.toString(jobState.getEndTime())); jobMetadataBuilder.put(METADATA_JOB_STATE, jobState.getState().toString()); jobMetadataBuilder.put(METADATA_JOB_LAUNCHED_TASKS, Integer.toString(jobState.getTaskCount())); jobMetadataBuilder.put(METADATA_JOB_COMPLETED_TASKS, Integer.toString(jobState.getCompletedTasks())); jobMetadataBuilder.put(METADATA_JOB_LAUNCHER_TYPE, jobState.getLauncherType().toString()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); jobMetadataBuilder.put(EventSubmitter.EVENT_TYPE, JOB_STATE); this.eventSubmitter.submit(JOB_STATE, jobMetadataBuilder.build()); }
@Test public void testSetAndGet() { this.jobState.setId(this.jobState.getJobId()); this.startTime = System.currentTimeMillis(); this.jobState.setStartTime(this.startTime); this.jobState.setEndTime(this.startTime + 1000); this.jobState.setDuration(1000); this.jobState.setState(JobState.RunningState.COMMITTED); this.jobState.setTaskCount(3); this.jobState.setProp("foo", "bar"); for (int i = 0; i < 3; i++) { WorkUnit workUnit = WorkUnit.createEmpty(); taskState.setProp("foo", "bar"); this.jobState.addTaskState(taskState);
"A job must have a job name specified by job.name"); this.jobName = JobState.getJobNameFromProps(jobProps); this.jobId = jobProps.containsKey(ConfigurationKeys.JOB_ID_KEY) ? jobProps.getProperty(ConfigurationKeys.JOB_ID_KEY) : JobLauncherUtils.newJobId(this.jobName); jobPropsState.addAll(jobProps); this.jobState = new JobState(jobPropsState, this.jobName, this.jobId); this.jobState.setBroker(this.jobBroker); this.jobState.setWorkUnitAndDatasetStateFunctional(new CombinedWorkUnitAndDatasetStateGenerator(this.datasetStateStore, this.jobName)); stagingDirProvided = this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR); outputDirProvided = this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR); this.jobState.setProp(Instrumented.METRIC_CONTEXT_NAME_KEY, this.jobMetricsOptional.get().getName()); } else { this.jobMetricsOptional = Optional.absent(); this.parallelizeCommit = this.jobState.getPropAsBoolean(ConfigurationKeys.PARALLELIZE_DATASET_COMMIT, ConfigurationKeys.DEFAULT_PARALLELIZE_DATASET_COMMIT); this.parallelCommits = this.parallelizeCommit ? this.jobState .getPropAsInt(ConfigurationKeys.DATASET_COMMIT_THREADS, ConfigurationKeys.DEFAULT_DATASET_COMMIT_THREADS) : 1;
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean alertEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, Boolean.toString(false))); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); // Send out alert email if the maximum number of consecutive failures is reached if (jobState.getState() == JobState.RunningState.FAILED) { int failures = jobState.getPropAsInt(ConfigurationKeys.JOB_FAILURES_KEY, 0); int maxFailures = jobState.getPropAsInt(ConfigurationKeys.JOB_MAX_FAILURES_KEY, ConfigurationKeys.DEFAULT_JOB_MAX_FAILURES); if (alertEmailEnabled && failures >= maxFailures) { try { EmailUtils.sendJobFailureAlertEmail(jobState.getJobName(), jobState.toString(), failures, jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job failure alert email for job " + jobState.getJobId(), ee); } return; } } if (notificationEmailEnabled) { try { EmailUtils.sendJobCompletionEmail( jobState.getJobId(), jobState.toString(), jobState.getState().toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job completion notification email for job " + jobState.getJobId(), ee); } } }
/** * Submits an event for each {@link TaskState} in the given {@link JobState}. */ private void submitTaskStateEvents(JobState jobState) { // Build Job Metadata applicable for TaskStates ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); Map<String, String> jobMetadata = jobMetadataBuilder.build(); // Submit event for each TaskState for (TaskState taskState : jobState.getTaskStates()) { submitTaskStateEvent(taskState, jobMetadata); } }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); if (!jobState.contains(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)) { LOG.error("Job configuration file path not found in job state of job " + jobState.getJobId()); return; } String jobConfigFile = jobState.getProp(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY); // Rename the config file so we won't run this job when the worker is bounced try { Files.move(new File(jobConfigFile), new File(jobConfigFile + ".done")); } catch (IOException ioe) { LOG.error("Failed to rename job configuration file for job " + jobState.getJobName(), ioe); } } }
private static List<Tag<?>> tagsForJob(JobState jobState) { List<Tag<?>> tags = Lists.newArrayList(); tags.add(new Tag<>(JobEvent.METADATA_JOB_NAME, jobState.getJobName() == null ? "" : jobState.getJobName())); tags.add(new Tag<>(JobEvent.METADATA_JOB_ID, jobState.getJobId())); tags.addAll(getCustomTagsFromState(jobState)); return tags; }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
jobExecutionInfo.setState(JobStateEnum.valueOf(this.state.name())); jobExecutionInfo.setLaunchedTasks(this.taskCount); jobExecutionInfo.setCompletedTasks(this.getCompletedTasks()); jobExecutionInfo.setLauncherType(getLauncherType()); if (getTrackingURL().isPresent()) { jobExecutionInfo.setTrackingUrl(getTrackingURL().get()); for (TaskState taskState : this.getTaskStates()) { taskExecutionInfos.add(taskState.toTaskExecutionInfo()); for (String name : this.getPropertyNames()) { String value = this.getProp(name); if (!Strings.isNullOrEmpty(value)) { jobProperties.put(name, value);
/** * If {@link ConfigurationKeys#WRITER_STAGING_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_STAGING_DIR_NAME}. */ private void setTaskStagingDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_STAGING_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState .setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(workingDir, TASK_STAGING_DIR_NAME).toString()); LOG.info(String.format("Writer Staging Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR))); } }
@Test public void testSubmitJobExecutionEvents() { JobState mockJobState = mock(JobState.class, Mockito.RETURNS_SMART_NULLS); when(mockJobState.getState()).thenReturn(JobState.RunningState.SUCCESSFUL); when(mockJobState.getLauncherType()).thenReturn(LauncherTypeEnum.$UNKNOWN); when(mockJobState.getTrackingURL()).thenReturn(Optional.<String> absent()); TaskState mockTaskState1 = createMockTaskState(); TaskState mockTaskState2 = createMockTaskState(); when(mockJobState.getTaskStates()).thenReturn(Lists.newArrayList(mockTaskState1, mockTaskState2)); this.jobExecutionEventSubmitter.submitJobExecutionEvents(mockJobState); verify(this.mockEventSubmitter, atLeastOnce()).submit(any(String.class), any(Map.class)); }