private void getInfoFromTaskConfig() { Map<String, String> configMap = this.taskConfig.getConfigMap(); this.jobName = configMap.get(ConfigurationKeys.JOB_NAME_KEY); this.jobId = configMap.get(ConfigurationKeys.JOB_ID_KEY); this.jobKey = Long.toString(Id.parse(this.jobId).getSequence()); this.taskId = configMap.get(ConfigurationKeys.TASK_ID_KEY); this.workUnitFilePath = new Path(configMap.get(GobblinClusterConfigurationKeys.WORK_UNIT_FILE_PATH)); }
/** * Create a new task ID for the job with the given job ID. * * @param jobId job ID * @param sequence task sequence number * @return new task ID */ public static String newTaskId(String jobId, int sequence) { return Id.Task.create(Id.parse(jobId).get(Id.Parts.INSTANCE_NAME), sequence).toString(); }
/** * Create an ID for a new multi-task (corresponding to a {@link org.apache.gobblin.source.workunit.MultiWorkUnit}) * for the job with the given job ID. * * @param jobId job ID * @param sequence multi-task sequence number * @return new multi-task ID */ public static String newMultiTaskId(String jobId, int sequence) { return Id.MultiTask.create(Id.parse(jobId).get(Id.Parts.INSTANCE_NAME), sequence).toString(); }
private void prepareWorkUnit(WorkUnit workUnit) { workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, TestHelper.TEST_TASK_ID); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, Long.toString(Id.parse(TestHelper.TEST_TASK_ID).getSequence())); workUnit.setProp(ConfigurationKeys.SOURCE_CLASS_KEY, SimpleJsonSource.class.getName()); workUnit.setProp(ConfigurationKeys.CONVERTER_CLASSES_KEY, SimpleJsonConverter.class.getName()); workUnit.setProp(ConfigurationKeys.WRITER_OUTPUT_FORMAT_KEY, WriterOutputFormat.AVRO.toString()); workUnit.setProp(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, Destination.DestinationType.HDFS.toString()); workUnit.setProp(ConfigurationKeys.WRITER_STAGING_DIR, this.appWorkDir.toString() + Path.SEPARATOR + "staging"); workUnit.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, this.taskOutputDir.toString()); workUnit.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestHelper.WRITER_FILE_NAME); workUnit.setProp(ConfigurationKeys.WRITER_FILE_PATH, TestHelper.REL_WRITER_FILE_PATH); workUnit.setProp(ConfigurationKeys.WRITER_BUILDER_CLASS, AvroDataWriterBuilder.class.getName()); workUnit.setProp(ConfigurationKeys.SOURCE_SCHEMA, TestHelper.SOURCE_SCHEMA); } }
taskConfigMap.put(ConfigurationKeys.JOB_NAME_KEY, TestHelper.TEST_JOB_NAME); taskConfigMap.put(ConfigurationKeys.JOB_ID_KEY, TestHelper.TEST_JOB_ID); taskConfigMap.put(ConfigurationKeys.TASK_KEY_KEY, Long.toString(Id.parse(TestHelper.TEST_JOB_ID).getSequence()));
private void getInfoFromTaskConfig() { Map<String, String> configMap = this.taskConfig.getConfigMap(); this.jobName = configMap.get(ConfigurationKeys.JOB_NAME_KEY); this.jobId = configMap.get(ConfigurationKeys.JOB_ID_KEY); this.jobKey = Long.toString(Id.parse(this.jobId).getSequence()); this.taskId = configMap.get(ConfigurationKeys.TASK_ID_KEY); this.workUnitFilePath = new Path(configMap.get(GobblinClusterConfigurationKeys.WORK_UNIT_FILE_PATH)); }
/** * Create an ID for a new multi-task (corresponding to a {@link org.apache.gobblin.source.workunit.MultiWorkUnit}) * for the job with the given job ID. * * @param jobId job ID * @param sequence multi-task sequence number * @return new multi-task ID */ public static String newMultiTaskId(String jobId, int sequence) { return Id.MultiTask.create(Id.parse(jobId).get(Id.Parts.INSTANCE_NAME), sequence).toString(); }
/** * Create a new task ID for the job with the given job ID. * * @param jobId job ID * @param sequence task sequence number * @return new task ID */ public static String newTaskId(String jobId, int sequence) { return Id.Task.create(Id.parse(jobId).get(Id.Parts.INSTANCE_NAME), sequence).toString(); }