private static List<Tag<?>> tagsForJob(JobState jobState) { List<Tag<?>> tags = Lists.newArrayList(); tags.add(new Tag<>(JobEvent.METADATA_JOB_NAME, jobState.getJobName() == null ? "" : jobState.getJobName())); tags.add(new Tag<>(JobEvent.METADATA_JOB_ID, jobState.getJobId())); tags.addAll(getCustomTagsFromState(jobState)); return tags; }
/** * Write a summary to the json document * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} * used to write the json document */ protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { jsonWriter.name("job name").value(this.getJobName()).name("job id").value(this.getJobId()).name("job state") .value(this.getState().name()).name("start time").value(this.getStartTime()).name("end time") .value(this.getEndTime()).name("duration").value(this.getDuration()).name("tasks").value(this.getTaskCount()) .name("completed tasks").value(this.getCompletedTasks()); }
/** * Submits an event for each {@link TaskState} in the given {@link JobState}. */ private void submitTaskStateEvents(JobState jobState) { // Build Job Metadata applicable for TaskStates ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); Map<String, String> jobMetadata = jobMetadataBuilder.build(); // Submit event for each TaskState for (TaskState taskState : jobState.getTaskStates()) { submitTaskStateEvent(taskState, jobMetadata); } }
protected JobMetrics(JobState job, MetricContext parentContext) { super(name(job), parentContext, tagsForJob(job)); this.jobName = job.getJobName(); }
/** * Staging data cannot be cleaned if exactly once semantics is used, and the job has unfinished * commit sequences. */ private boolean canCleanStagingData(JobState jobState) throws IOException { return this.jobContext.getSemantics() != DeliverySemantics.EXACTLY_ONCE || !this.jobContext.getCommitSequenceStore() .get().exists(jobState.getJobName()); }
/** * Submits an event for the given {@link JobState}. */ private void submitJobStateEvent(JobState jobState) { ImmutableMap.Builder<String, String> jobMetadataBuilder = new ImmutableMap.Builder<>(); jobMetadataBuilder.put(METADATA_JOB_ID, jobState.getJobId()); jobMetadataBuilder.put(METADATA_JOB_NAME, jobState.getJobName()); jobMetadataBuilder.put(METADATA_JOB_START_TIME, Long.toString(jobState.getStartTime())); jobMetadataBuilder.put(METADATA_JOB_END_TIME, Long.toString(jobState.getEndTime())); jobMetadataBuilder.put(METADATA_JOB_STATE, jobState.getState().toString()); jobMetadataBuilder.put(METADATA_JOB_LAUNCHED_TASKS, Integer.toString(jobState.getTaskCount())); jobMetadataBuilder.put(METADATA_JOB_COMPLETED_TASKS, Integer.toString(jobState.getCompletedTasks())); jobMetadataBuilder.put(METADATA_JOB_LAUNCHER_TYPE, jobState.getLauncherType().toString()); jobMetadataBuilder.put(METADATA_JOB_TRACKING_URL, jobState.getTrackingURL().or(UNKNOWN_VALUE)); jobMetadataBuilder.put(EventSubmitter.EVENT_TYPE, JOB_STATE); this.eventSubmitter.submit(JOB_STATE, jobMetadataBuilder.build()); }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); if (!jobState.contains(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)) { LOG.error("Job configuration file path not found in job state of job " + jobState.getJobId()); return; } String jobConfigFile = jobState.getProp(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY); // Rename the config file so we won't run this job when the worker is bounced try { Files.move(new File(jobConfigFile), new File(jobConfigFile + ".done")); } catch (IOException ioe) { LOG.error("Failed to rename job configuration file for job " + jobState.getJobName(), ioe); } } }
private SharedResourcesBroker<GobblinScopeTypes> getJobBroker(JobState jobState, SharedResourcesBroker<GobblinScopeTypes> globalBroker) { return globalBroker.newSubscopedBuilder(new JobScopeInstance(jobState.getJobName(), jobState.getJobId())).build(); }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean alertEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, Boolean.toString(false))); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); // Send out alert email if the maximum number of consecutive failures is reached if (jobState.getState() == JobState.RunningState.FAILED) { int failures = jobState.getPropAsInt(ConfigurationKeys.JOB_FAILURES_KEY, 0); int maxFailures = jobState.getPropAsInt(ConfigurationKeys.JOB_MAX_FAILURES_KEY, ConfigurationKeys.DEFAULT_JOB_MAX_FAILURES); if (alertEmailEnabled && failures >= maxFailures) { try { EmailUtils.sendJobFailureAlertEmail(jobState.getJobName(), jobState.toString(), failures, jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job failure alert email for job " + jobState.getJobId(), ee); } return; } } if (notificationEmailEnabled) { try { EmailUtils.sendJobCompletionEmail( jobState.getJobId(), jobState.toString(), jobState.getState().toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job completion notification email for job " + jobState.getJobId(), ee); } } }
LOG.error("Job " + jobState.getJobName() + " has unfinished commit sequences. Will not clean up staging data."); return;
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = zkJobStateStore.get(TEST_JOB_NAME, zkDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + zkDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = this.fsDatasetStateStore.get(TEST_JOB_NAME, FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } }
LOG.error("Job " + jobState.getJobName() + " has unfinished commit sequences. Will not clean up staging data."); return;
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
private void doAsserts(JobState jobState, boolean considerTaskStates, boolean initial) { Assert.assertEquals(jobState.getJobName(), "TestJob"); Assert.assertEquals(jobState.getJobId(), "TestJob-1"); Assert.assertEquals(jobState.getId(), "TestJob-1");
GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = globalBroker.newSubscopedBuilder(new JobScopeInstance(this.jobState.getJobName(), this.jobState.getJobId())) .build();
executeUnfinishedCommitSequences(jobState.getJobName());
private static List<Tag<?>> tagsForJob(JobState jobState) { List<Tag<?>> tags = Lists.newArrayList(); tags.add(new Tag<>(JobEvent.METADATA_JOB_NAME, jobState.getJobName() == null ? "" : jobState.getJobName())); tags.add(new Tag<>(JobEvent.METADATA_JOB_ID, jobState.getJobId())); tags.addAll(getCustomTagsFromState(jobState)); return tags; }
/** * Write a summary to the json document * * @param jsonWriter a {@link com.google.gson.stream.JsonWriter} * used to write the json document */ protected void writeStateSummary(JsonWriter jsonWriter) throws IOException { jsonWriter.name("job name").value(this.getJobName()).name("job id").value(this.getJobId()).name("job state") .value(this.getState().name()).name("start time").value(this.getStartTime()).name("end time") .value(this.getEndTime()).name("duration").value(this.getDuration()).name("tasks").value(this.getTaskCount()) .name("completed tasks").value(this.getCompletedTasks()); }
/** * Staging data cannot be cleaned if exactly once semantics is used, and the job has unfinished * commit sequences. */ private boolean canCleanStagingData(JobState jobState) throws IOException { return this.jobContext.getSemantics() != DeliverySemantics.EXACTLY_ONCE || !this.jobContext.getCommitSequenceStore() .get().exists(jobState.getJobName()); }