public String getDatasetUrn() { return super.getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN); }
protected void propsToJson(JsonWriter jsonWriter) throws IOException { jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean alertEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, Boolean.toString(false))); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); // Send out alert email if the maximum number of consecutive failures is reached if (jobState.getState() == JobState.RunningState.FAILED) { int failures = jobState.getPropAsInt(ConfigurationKeys.JOB_FAILURES_KEY, 0); int maxFailures = jobState.getPropAsInt(ConfigurationKeys.JOB_MAX_FAILURES_KEY, ConfigurationKeys.DEFAULT_JOB_MAX_FAILURES); if (alertEmailEnabled && failures >= maxFailures) { try { EmailUtils.sendJobFailureAlertEmail(jobState.getJobName(), jobState.toString(), failures, jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job failure alert email for job " + jobState.getJobId(), ee); } return; } } if (notificationEmailEnabled) { try { EmailUtils.sendJobCompletionEmail( jobState.getJobId(), jobState.toString(), jobState.getState().toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job completion notification email for job " + jobState.getJobId(), ee); } } }
/** * Get the tracking URL for this {@link JobState}. */ public Optional<String> getTrackingURL() { return Optional.fromNullable(this.getProp(ConfigurationKeys.JOB_TRACKING_URL_KEY)); }
String value = this.getProp(name); if (!Strings.isNullOrEmpty(value)) { jobProperties.put(name, value);
public int getJobFailures() { return Integer.parseInt(super.getProp(ConfigurationKeys.JOB_FAILURES_KEY)); }
protected void propsToJson(JsonWriter jsonWriter) throws IOException { jsonWriter.beginObject(); for (String key : this.getPropertyNames()) { jsonWriter.name(key).value(this.getProp(key)); } jsonWriter.endObject(); }
private final int getTaskCommitThreadPoolSize() { return Integer.parseInt(this.jobState.getProp(ConfigurationKeys.TASK_EXECUTOR_THREADPOOL_SIZE_KEY, Integer.toString(ConfigurationKeys.DEFAULT_TASK_EXECUTOR_THREADPOOL_SIZE))); }
public void incrementJobFailures() { super.setProp(ConfigurationKeys.JOB_FAILURES_KEY, Integer.parseInt(super.getProp(ConfigurationKeys.JOB_FAILURES_KEY, "0")) + 1); }
protected Optional<CommitSequenceStore> createCommitSequenceStore() throws IOException { if (this.semantics != DeliverySemantics.EXACTLY_ONCE) { return Optional.<CommitSequenceStore>absent(); } Preconditions.checkState(this.jobState.contains(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)); try (FileSystem fs = FileSystem.get(URI.create(this.jobState .getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_FS_URI, ConfigurationKeys.LOCAL_FS_URI)), HadoopUtils.getConfFromState(this.jobState))) { return Optional.<CommitSequenceStore>of(new FsCommitSequenceStore(fs, new Path(this.jobState.getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)))); } }
/** * If {@link ConfigurationKeys#WRITER_OUTPUT_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_OUTPUT_DIR_NAME}. */ private void setTaskOutputDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_OUTPUT_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(workingDir, TASK_OUTPUT_DIR_NAME).toString()); LOG.info(String .format("Writer Output Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR))); } }
/** * If {@link ConfigurationKeys#WRITER_STAGING_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_STAGING_DIR_NAME}. */ private void setTaskStagingDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_STAGING_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState .setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(workingDir, TASK_STAGING_DIR_NAME).toString()); LOG.info(String.format("Writer Staging Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR))); } }
/** * Get the {@link LauncherTypeEnum} for this {@link JobState}. */ public LauncherTypeEnum getLauncherType() { return Enums.getIfPresent(LauncherTypeEnum.class, this.getProp(ConfigurationKeys.JOB_LAUNCHER_TYPE_KEY, JobLauncherFactory.JobLauncherType.LOCAL.name())) .or(LauncherTypeEnum.LOCAL); }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); if (!jobState.contains(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)) { LOG.error("Job configuration file path not found in job state of job " + jobState.getJobId()); return; } String jobConfigFile = jobState.getProp(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY); // Rename the config file so we won't run this job when the worker is bounced try { Files.move(new File(jobConfigFile), new File(jobConfigFile + ".done")); } catch (IOException ioe) { LOG.error("Failed to rename job configuration file for job " + jobState.getJobName(), ioe); } } }
@Override public void onJobCancellation(JobContext jobContext) { JobState jobState = jobContext.getJobState(); boolean notificationEmailEnabled = Boolean.valueOf(jobState.getProp(ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, Boolean.toString(false))); if (notificationEmailEnabled) { try { EmailUtils.sendJobCancellationEmail(jobState.getJobId(), jobState.toString(), jobState); } catch (EmailException ee) { LOGGER.error("Failed to send job cancellation notification email for job " + jobState.getJobId(), ee); } } } }
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
@Test(dependsOnMethods = "testPersistJobState") public void testGetJobState() throws IOException { JobState jobState = dbJobStateStore.get(TEST_JOB_NAME, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertNotEquals(jobState.getProp("lower"), "case"); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getStartTime(), this.startTime); Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000); Assert.assertEquals(jobState.getDuration(), 1000); Assert.assertEquals(jobState.getCompletedTasks(), 3); for (int i = 0; i < jobState.getCompletedTasks(); i++) { TaskState taskState = jobState.getTaskStates().get(i); Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID); Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); } jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER, dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX, TEST_JOB_ID); Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER); Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID); Assert.assertEquals(jobState.getProp("foo"), "bar"); Assert.assertEquals(jobState.getProp("lower"), "case"); }
Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getTaskCount(), 3); Assert.assertEquals(jobState.getProp("foo"), "bar");
/** * Get the tracking URL for this {@link JobState}. */ public Optional<String> getTrackingURL() { return Optional.fromNullable(this.getProp(ConfigurationKeys.JOB_TRACKING_URL_KEY)); }
/** * Get the {@link LauncherTypeEnum} for this {@link JobState}. */ public LauncherTypeEnum getLauncherType() { return Enums.getIfPresent(LauncherTypeEnum.class, this.getProp(ConfigurationKeys.JOB_LAUNCHER_TYPE_KEY, JobLauncherFactory.JobLauncherType.LOCAL.name())) .or(LauncherTypeEnum.LOCAL); }