public JobLauncher buildJobLauncher(Properties jobProps) throws Exception { return JobLauncherFactory.newJobLauncher(this.properties, jobProps); }
/** * Create a new {@link JobLauncher}. * * <p> * This method will never return a {@code null}. * </p> * * @param sysProps system configuration properties * @param jobProps job configuration properties * @return newly created {@link JobLauncher} */ public static @Nonnull JobLauncher newJobLauncher(Properties sysProps, Properties jobProps) throws Exception { return newJobLauncher(sysProps, jobProps, null); }
/** * Creates a new instance for a JobLauncher with a given type * @param sysProps the system/environment properties * @param jobProps the job properties * @param launcherTypeValue the type of the launcher; either a {@link JobLauncherType} value or * the name of the class that extends {@link AbstractJobLauncher} and has a constructor * that has a single Properties parameter.. * @return the JobLauncher instance * @throws RuntimeException if the instantiation fails */ public static JobLauncher newJobLauncher(Properties sysProps, Properties jobProps, String launcherTypeValue, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { return newJobLauncher(sysProps, jobProps, launcherTypeValue, instanceBroker, ImmutableList.of()); }
/** * Create a new {@link JobLauncher}. * * <p> * This method will never return a {@code null}. * </p> * * @param sysProps system configuration properties * @param jobProps job configuration properties * @param instanceBroker * @return newly created {@link JobLauncher} */ public static @Nonnull JobLauncher newJobLauncher(Properties sysProps, Properties jobProps, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) throws Exception { return newJobLauncher(sysProps, jobProps, instanceBroker, ImmutableList.of()); }
/** * Create a new {@link JobLauncher}. * * <p> * This method will never return a {@code null}. * </p> * * @param sysProps system configuration properties * @param jobProps job configuration properties * @param instanceBroker * @param metadataTags * @return newly created {@link JobLauncher} */ public static @Nonnull JobLauncher newJobLauncher(Properties sysProps, Properties jobProps, SharedResourcesBroker<GobblinScopeTypes> instanceBroker, List<? extends Tag<?>> metadataTags) throws Exception { String launcherTypeValue = sysProps.getProperty(ConfigurationKeys.JOB_LAUNCHER_TYPE_KEY, JobLauncherType.LOCAL.name()); return newJobLauncher(sysProps, jobProps, launcherTypeValue, instanceBroker, metadataTags); }
private static JobLauncher createLauncher(Configurable _sysConfig, JobSpec _jobSpec, Logger _log, Optional<String> jobLauncherType, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { if (jobLauncherType.isPresent()) { return JobLauncherFactory.newJobLauncher(_sysConfig.getConfigAsProperties(), _jobSpec.getConfigAsProperties(), jobLauncherType.get(), instanceBroker); } else { _log.info("Creating auto jobLauncher for " + _jobSpec); try { return JobLauncherFactory.newJobLauncher(_sysConfig.getConfigAsProperties(), _jobSpec.getConfigAsProperties(), instanceBroker); } catch (Exception e) { throw new RuntimeException("JobLauncher creation failed: " + e, e); } } }
@Test public void testTarGzCopy() throws Exception { Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); String file1Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR) + "/LogData/sub1/sub2/text1.txt"; String file2Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR) + "/LogData/sub1/sub2/text2.txt"; FileSystem fs = FileSystem.getLocal(new Configuration()); Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file1Path)))), "text1"); Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file2Path)))), "text2"); } finally { closer.close(); } }
this.jobLauncher = this.closer.register(JobLauncherFactory.newJobLauncher(jobProps, jobProps, null, metadataTags));
public void runTestWithCancellation(final Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { final JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); final AtomicBoolean isCancelled = new AtomicBoolean(false); // This thread will cancel the job after some time Thread thread = new Thread(new Runnable() { @Override public void run() { try { Thread.sleep(500); jobLauncher.cancelJob(null); isCancelled.set(true); } catch (Exception je) { // Ignored } } }); thread.start(); jobLauncher.launchJob(null); Assert.assertTrue(isCancelled.get()); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); Assert.assertTrue(datasetStateList.isEmpty()); }
jobProps.setProperty("publish.at.job.level","false"); totalSuccessful = 5; // number of records in partition 1 JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); jobProps.setProperty("writer.kafka.producerConfig.flaky.errorType", "nth"); jobProps.setProperty("writer.kafka.producerConfig.flaky.errorEvery", "" + errorEvery); JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); totalSuccessful = totalRecords / errorEvery + totalRecords%errorEvery;
public void runTestWithFork(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); try (JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)) { jobLauncher.launchJob(null); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); FileSystem lfs = FileSystem.getLocal(new Configuration()); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Path path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_0")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".0"), TestExtractor.TOTAL_RECORDS); path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR), new Path(taskState.getExtract().getOutputFilePath(), "fork_1")); Assert.assertTrue(lfs.exists(path)); Assert.assertEquals(lfs.listStatus(path).length, 2); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".1"), TestExtractor.TOTAL_RECORDS); } }
public void runTestWithCommitSuccessfulTasksPolicy(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.FALSE.toString()); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "successful"); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, TestSourceWithFaultyExtractor.class.getName()); jobProps.setProperty(ConfigurationKeys.MAX_TASK_RETRIES_KEY, "0"); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); JobState jobState = datasetStateList.get(0); Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(jobState.getCompletedTasks(), 4); for (TaskState taskState : jobState.getTaskStates()) { if (taskState.getTaskId().endsWith("0")) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.FAILED); } else { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally {
public void runTestWithPullLimit(Properties jobProps, long limit) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED), limit); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN), limit); } }
public void runTestWithMultipleDatasets(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName).toString(); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); jobProps.setProperty(ConfigurationKeys.SOURCE_CLASS_KEY, MultiDatasetTestSource.class.getName()); Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } finally { closer.close(); } for (int i = 0; i < 4; i++) { List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, "Dataset" + i + "-current.jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getDatasetUrn(), "Dataset" + i); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 1); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getProp(ConfigurationKeys.DATASET_URN_KEY), "Dataset" + i); Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } } }
public void runTest(Properties jobProps) throws Exception { String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY); String jobId = JobLauncherUtils.newJobId(jobName); jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId); JobContext jobContext = null; Closer closer = Closer.create(); try { JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); jobContext = ((AbstractJobLauncher) jobLauncher).getJobContext(); } finally { closer.close(); } Assert.assertTrue(jobContext.getJobMetricsOptional().isPresent()); String jobMetricContextTags = jobContext.getJobMetricsOptional().get().getMetricContext().getTags().toString(); Assert.assertTrue(jobMetricContextTags.contains(ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME), ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME + " tag missing in job metric context tags."); List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED); Assert.assertEquals(datasetState.getCompletedTasks(), 4); Assert.assertEquals(datasetState.getJobFailures(), 0); for (TaskState taskState : datasetState.getTaskStates()) { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); } }
JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)); jobLauncher.launchJob(null); } catch (JobException je) {
public JobLauncher buildJobLauncher(Properties jobProps) throws Exception { return JobLauncherFactory.newJobLauncher(this.properties, jobProps); }
/** * Create a new {@link JobLauncher}. * * <p> * This method will never return a {@code null}. * </p> * * @param sysProps system configuration properties * @param jobProps job configuration properties * @return newly created {@link JobLauncher} */ public static @Nonnull JobLauncher newJobLauncher(Properties sysProps, Properties jobProps) throws Exception { return newJobLauncher(sysProps, jobProps, null); }
/** * Creates a new instance for a JobLauncher with a given type * @param sysProps the system/environment properties * @param jobProps the job properties * @param launcherTypeValue the type of the launcher; either a {@link JobLauncherType} value or * the name of the class that extends {@link AbstractJobLauncher} and has a constructor * that has a single Properties parameter.. * @return the JobLauncher instance * @throws RuntimeException if the instantiation fails */ public static JobLauncher newJobLauncher(Properties sysProps, Properties jobProps, String launcherTypeValue, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { return newJobLauncher(sysProps, jobProps, launcherTypeValue, instanceBroker, ImmutableList.of()); }