/** * Create the {@link Job} to run in this task. * @return the {@link Job} to run. If this method returns null, no job will be run and the task will be marked as successful. */ protected Job createJob() throws IOException { Job job = Job.getInstance(new Configuration()); for (Map.Entry<Object, Object> entry : this.taskContext.getTaskState().getProperties().entrySet()) { if (entry.getKey() instanceof String && ((String) entry.getKey()).startsWith(JOB_CONFIGURATION_PREFIX)) { String actualKey = ((String) entry.getKey()).substring(JOB_CONFIGURATION_PREFIX.length()); job.getConfiguration().set(actualKey, (String) entry.getValue()); } } return job; }
@Override public void run() { try { DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(new Configuration()), this.taskContext.getTaskState().getProperties()); datasetCleaner.clean(); } catch (IOException e) { throw new RuntimeException(e); } } }
Config config; try { config = ConfigUtils.propertiesToConfig(taskState.getProperties()); } catch (Exception e) { LOG.warn("Failed to deserialize taskState into Config.. continuing with an empty config", e);
HiveConvertPublisher.setDestLineageInfo(taskState, lineageInfo); Properties props = taskState.getProperties();
/** * Create the {@link Job} to run in this task. * @return the {@link Job} to run. If this method returns null, no job will be run and the task will be marked as successful. */ protected Job createJob() throws IOException { Job job = Job.getInstance(new Configuration()); for (Map.Entry<Object, Object> entry : this.taskContext.getTaskState().getProperties().entrySet()) { if (entry.getKey() instanceof String && ((String) entry.getKey()).startsWith(JOB_CONFIGURATION_PREFIX)) { String actualKey = ((String) entry.getKey()).substring(JOB_CONFIGURATION_PREFIX.length()); job.getConfiguration().set(actualKey, (String) entry.getValue()); } } return job; }
@Override public void run() { try { DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(new Configuration()), this.taskContext.getTaskState().getProperties()); datasetCleaner.clean(); } catch (IOException e) { throw new RuntimeException(e); } } }
@VisibleForTesting protected DistributeJobResult getResultFromUserContent() { String planningId = getPlanningJobId(this.jobPlanningProps); try { TaskState taskState = this.stateStores.getTaskStateStore().get(planningId, planningId, planningId); return new DistributeJobResult(Optional.of(taskState.getProperties()), Optional.empty()); } catch (IOException e) { return new DistributeJobResult(Optional.empty(), Optional.of(e)); } }
Config config; try { config = ConfigUtils.propertiesToConfig(taskState.getProperties()); } catch (Exception e) { LOG.warn("Failed to deserialize taskState into Config.. continuing with an empty config", e);