protected Optional<CommitSequenceStore> createCommitSequenceStore() throws IOException { if (this.semantics != DeliverySemantics.EXACTLY_ONCE) { return Optional.<CommitSequenceStore>absent(); } Preconditions.checkState(this.jobState.contains(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)); try (FileSystem fs = FileSystem.get(URI.create(this.jobState .getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_FS_URI, ConfigurationKeys.LOCAL_FS_URI)), HadoopUtils.getConfFromState(this.jobState))) { return Optional.<CommitSequenceStore>of(new FsCommitSequenceStore(fs, new Path(this.jobState.getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)))); } }
/** * If {@link ConfigurationKeys#WRITER_OUTPUT_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_OUTPUT_DIR_NAME}. */ private void setTaskOutputDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_OUTPUT_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(workingDir, TASK_OUTPUT_DIR_NAME).toString()); LOG.info(String .format("Writer Output Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR))); } }
/** * If {@link ConfigurationKeys#WRITER_STAGING_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_STAGING_DIR_NAME}. */ private void setTaskStagingDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_STAGING_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState .setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(workingDir, TASK_STAGING_DIR_NAME).toString()); LOG.info(String.format("Writer Staging Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR))); } }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); if (!jobState.contains(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)) { LOG.error("Job configuration file path not found in job state of job " + jobState.getJobId()); return; } String jobConfigFile = jobState.getProp(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY); // Rename the config file so we won't run this job when the worker is bounced try { Files.move(new File(jobConfigFile), new File(jobConfigFile + ".done")); } catch (IOException ioe) { LOG.error("Failed to rename job configuration file for job " + jobState.getJobName(), ioe); } } }
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
if (!jobState.contains(ConfigurationKeys.JOB_TRACKING_URL_KEY)) { jobState.setProp(ConfigurationKeys.JOB_TRACKING_URL_KEY, this.job.getTrackingURL());
protected Optional<CommitSequenceStore> createCommitSequenceStore() throws IOException { if (this.semantics != DeliverySemantics.EXACTLY_ONCE) { return Optional.<CommitSequenceStore>absent(); } Preconditions.checkState(this.jobState.contains(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)); try (FileSystem fs = FileSystem.get(URI.create(this.jobState .getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_FS_URI, ConfigurationKeys.LOCAL_FS_URI)), HadoopUtils.getConfFromState(this.jobState))) { return Optional.<CommitSequenceStore>of(new FsCommitSequenceStore(fs, new Path(this.jobState.getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)))); } }
/** * If {@link ConfigurationKeys#WRITER_OUTPUT_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_OUTPUT_DIR_NAME}. */ private void setTaskOutputDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_OUTPUT_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, new Path(workingDir, TASK_OUTPUT_DIR_NAME).toString()); LOG.info(String .format("Writer Output Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR))); } }
/** * If {@link ConfigurationKeys#WRITER_STAGING_DIR} (which is deprecated) is specified, use its value. * * Otherwise, if {@link ConfigurationKeys#TASK_DATA_ROOT_DIR_KEY} is specified, use its value * plus {@link #TASK_STAGING_DIR_NAME}. */ private void setTaskStagingDir() { if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { LOG.warn(String.format("Property %s is deprecated. No need to use it if %s is specified.", ConfigurationKeys.WRITER_STAGING_DIR, ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)); } else { String workingDir = this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY); this.jobState .setProp(ConfigurationKeys.WRITER_STAGING_DIR, new Path(workingDir, TASK_STAGING_DIR_NAME).toString()); LOG.info(String.format("Writer Staging Directory is set to %s.", this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR))); } }
@Override public void onJobCompletion(JobContext jobContext) { JobState jobState = jobContext.getJobState(); if (!jobState.contains(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY)) { LOG.error("Job configuration file path not found in job state of job " + jobState.getJobId()); return; } String jobConfigFile = jobState.getProp(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY); // Rename the config file so we won't run this job when the worker is bounced try { Files.move(new File(jobConfigFile), new File(jobConfigFile + ".done")); } catch (IOException ioe) { LOG.error("Failed to rename job configuration file for job " + jobState.getJobName(), ioe); } } }
protected void setTaskStagingAndOutputDirs() { // Add jobId to writer staging dir if (this.jobState.contains(ConfigurationKeys.WRITER_STAGING_DIR)) { String writerStagingDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_STAGING_DIR), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, writerStagingDirWithJobId); } // Add jobId to writer output dir if (this.jobState.contains(ConfigurationKeys.WRITER_OUTPUT_DIR)) { String writerOutputDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR), this.getJobName()), this.jobId) .toString(); this.jobState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, writerOutputDirWithJobId); } // Add jobId to task data root dir if (this.jobState.contains(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY)) { String taskDataRootDirWithJobId = new Path(getJobDir(this.jobState.getProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY), this.getJobName()), this.jobId).toString(); this.jobState.setProp(ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY, taskDataRootDirWithJobId); setTaskStagingDir(); setTaskOutputDir(); } else { LOG.warn("Property " + ConfigurationKeys.TASK_DATA_ROOT_DIR_KEY + " is missing."); } }
if (!jobState.contains(ConfigurationKeys.JOB_TRACKING_URL_KEY)) { jobState.setProp(ConfigurationKeys.JOB_TRACKING_URL_KEY, this.job.getTrackingURL());