/** * Set the specified key, value pair in this {@link MultiWorkUnit} only, but do not propagate it to all the inner * {@link WorkUnit}s. * * @param key property key * @param value property value */ public void setPropExcludeInnerWorkUnits(String key, Object value) { super.setProp(key, value); }
/** * This method will allow a work unit to be skipped if needed. */ public void skip() { this.setProp(ConfigurationKeys.WORK_UNIT_SKIP_KEY, true); }
/** * Set the low watermark of this {@link WorkUnit}. * * @param lowWaterMark low watermark * @deprecated use {@link #setWatermarkInterval(WatermarkInterval)}. */ @Deprecated public void setLowWaterMark(long lowWaterMark) { setProp(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY, lowWaterMark); }
/** * Set the high watermark of this {@link WorkUnit}. * * @param highWaterMark high watermark * @deprecated use {@link #setWatermarkInterval(WatermarkInterval)}. */ @Deprecated public void setHighWaterMark(long highWaterMark) { setProp(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY, highWaterMark); }
private void setLimiterReportKeyListToWorkUnits(List<WorkUnit> workUnits, List<String> keyNameList) { if (keyNameList.isEmpty()) { return; } String keyList = Joiner.on(',').join(keyNameList.iterator()); for (WorkUnit workUnit : workUnits) { workUnit.setProp(LimiterConfigurationKeys.LIMITER_REPORT_KEY_LIST, keyList); } }
/** * Set {@link WatermarkInterval} for a {@link WorkUnit}. */ public void setWatermarkInterval(WatermarkInterval watermarkInterval) { setProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY, watermarkInterval.toJson().toString()); }
private void addDatasetUrnOptionally(WorkUnit workUnit) { if (!this.shouldEnableDatasetStateStore) { return; } workUnit.setProp(ConfigurationKeys.DATASET_URN_KEY, workUnit.getProp(TOPIC_NAME)); }
private static void setWorkUnitWeight(WorkUnit workUnit, CopyEntity copyEntity, long minWeight) { long weight = 0; if (copyEntity instanceof CopyableFile) { weight = ((CopyableFile) copyEntity).getOrigin().getLen(); } weight = Math.max(weight, minWeight); workUnit.setProp(WORK_UNIT_WEIGHT, Long.toString(weight)); }
private void addSchemaFile(PartitionAwareFileRetriever.FileInfo dataFile, WorkUnit workUnit) throws IOException { Path schemaFile = new Path(new Path(dataFile.getFilePath()).getParent(), workUnit.getProp(ConfigurationKeys.SCHEMA_FILENAME, ConfigurationKeys.DEFAULT_SCHEMA_FILENAME)); if (fs.exists(schemaFile)) { workUnit.setProp(ConfigurationKeys.SOURCE_SCHEMA, schemaFile.toString()); } else { throw new IOException("Schema file " + schemaFile + " does not exist."); } }
@Override protected void forWorkUnit(WorkUnit workUnit) { workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, this.jobId); String taskId = JobLauncherUtils.newTaskId(this.jobId, this.taskIdSequence++); workUnit.setId(taskId); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, taskId); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, Long.toString(Id.Task.parse(taskId).getSequence())); } }
@Test public void testSerializeState() throws IOException { WorkUnit workUnit1 = WorkUnit.createEmpty(); workUnit1.setProp("foo", "bar"); workUnit1.setProp("a", 10); SerializationUtils.serializeState(this.fs, new Path(this.outputPath, "wu1"), workUnit1); WorkUnit workUnit2 = WorkUnit.createEmpty(); workUnit2.setProp("foo", "baz"); workUnit2.setProp("b", 20); SerializationUtils.serializeState(this.fs, new Path(this.outputPath, "wu2"), workUnit2); }
protected WorkUnit createWorkUnitForFailure (Dataset dataset, String reason) throws IOException { WorkUnit workUnit = new FailedTask.FailedWorkUnit(); workUnit.setProp(CompactionVerifier.COMPACTION_VERIFICATION_FAIL_REASON, reason); TaskUtils.setTaskFactoryClass(workUnit, CompactionFailedTask.CompactionFailedTaskFactory.class); suite.save (dataset, workUnit); return workUnit; }
public void serialize(WorkUnit workUnit) { workUnit.setWatermarkInterval( new WatermarkInterval(new LongWatermark(lowWatermark), new LongWatermark(highWatermark))); if (hasUserSpecifiedHighWatermark) { workUnit.setProp(Partition.HAS_USER_SPECIFIED_HIGH_WATERMARK, true); } if (isLastPartition) { workUnit.setProp(Partition.IS_LAST_PARTIITON, true); } }
private TaskContext getTaskContextForRun(WorkUnit workUnit) { workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, "job123"); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, "task123"); workUnit.setProp(HiveConverterUtils.HIVE_DATASET_DESTINATION_SKIP_SETGROUP, Boolean.toString(true)); HiveTask.disableHiveWatermarker(workUnit); JobState jobState = new JobState("job", "job123"); return new TaskContext(new WorkUnitState(workUnit, jobState)); }
public WorkUnit getWorkUnitWithWeight(long weight) { WorkUnit workUnit = new WorkUnit(new Extract(Extract.TableType.APPEND_ONLY, "", "")); workUnit.setProp(WEIGHT, Long.toString(weight)); return workUnit; }
@BeforeClass public void setUp() throws Exception { WorkUnit workUnit = WorkUnit.createEmpty(); Properties properties = new Properties(); properties.load(new StringReader(TEST_JOB_CONFIG)); workUnit.addAll(properties); workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, JobLauncherUtils.newJobId("GobblinTest1")); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, JobLauncherUtils.newTaskId(workUnit.getProp(ConfigurationKeys.JOB_ID_KEY), 0)); this.taskContext = new TaskContext(new WorkUnitState(workUnit)); }
TaskState getEmptyTestTaskState(String taskId) { // Create a TaskState WorkUnit workUnit = WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName())); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, "taskKey"); TaskState taskState = new TaskState(new WorkUnitState(workUnit)); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setTaskId(taskId); taskState.setJobId("1234"); return taskState; }
TaskState getEmptyTestTaskState(String taskId) { // Create a TaskState WorkUnit workUnit = WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName())); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, "taskKey"); TaskState taskState = new TaskState(new WorkUnitState(workUnit)); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setTaskId(taskId); taskState.setJobId("1234"); return taskState; }
private WorkUnit createWorkUnit(String... names) { if (names.length == 1) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(WORK_UNIT_NAME, names[0]); return workUnit; } MultiWorkUnit mwu = new MultiWorkUnit(); for (String name : names) { mwu.addWorkUnit(createWorkUnit(name)); } return mwu; }