/** * Factory method. * * @return An empty {@link WorkUnit}. */ public static WorkUnit createEmpty() { return new WorkUnit(); }
/** * Factory method. * * @param extract {@link Extract} * @return A {@link WorkUnit} with the given {@link Extract} */ public static WorkUnit create(Extract extract) { return new WorkUnit(null, extract); }
/** * Factory method. * * @param extract {@link Extract} * @param watermarkInterval {@link WatermarkInterval} * @return A {@link WorkUnit} with the given {@link Extract} and {@link WatermarkInterval} */ public static WorkUnit create(Extract extract, WatermarkInterval watermarkInterval) { return new WorkUnit(null, extract, watermarkInterval); }
/** * Factory method. * * @param other a {@link WorkUnit} instance * @return A copy of the given {@link WorkUnit} instance */ public static WorkUnit copyOf(WorkUnit other) { return new WorkUnit(other); }
/** * Create a new {@link WorkUnit} instance from a given {@link Extract}. * * @param extract given {@link Extract} * @return a new {@link WorkUnit} instance * * @deprecated Properties in SourceState should not added to a WorkUnit. Having each WorkUnit contain a copy of * SourceState is a waste of memory. Use {@link WorkUnit#create(Extract)}. */ @Deprecated public WorkUnit createWorkUnit(Extract extract) { return new WorkUnit(this, extract); }
/** * @return A {@link WorkUnit} that will run a {@link NoopTask}. */ public static WorkUnit noopWorkunit() { WorkUnit workUnit = new WorkUnit(); TaskUtils.setTaskFactoryClass(workUnit, Factory.class); return workUnit; }
@Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); for (int i = 0; i < state.getPropAsInt(NUM_WORK_UNITS, 1); i++) { workUnits.add(new WorkUnit()); } return workUnits; }
@Override public List<WorkUnit> getWorkunits(SourceState state) { int numWorkUnits = state.getPropAsInt(NUM_WORK_UNITS_KEY, DEFAULT_NUM_WORK_UNITS); Extract extract = new Extract(TableType.APPEND_ONLY, StressTestingSource.class.getPackage().getName(), StressTestingSource.class.getSimpleName()); List<WorkUnit> wus = new ArrayList<>(numWorkUnits); for (int i = 1; i <= numWorkUnits; ++i) { WorkUnit wu = new WorkUnit(extract); wus.add(wu); } return wus; }
protected WorkUnit createWorkUnit (Dataset dataset) throws IOException { WorkUnit workUnit = new WorkUnit(); TaskUtils.setTaskFactoryClass(workUnit, MRCompactionTaskFactory.class); suite.save (dataset, workUnit); return workUnit; }
@Override public List<WorkUnit> getWorkunits(SourceState state) { WorkUnit wu = new WorkUnit(); TaskUtils.setTaskFactoryClass(wu, FailsWithExceptionTaskFactory.class); return Collections.singletonList(wu); }
@Test public void testEquals() { SourceState sourceState = new SourceState(); sourceState.setProp("testKey", "true"); WorkUnitState workUnitState = new WorkUnitState(new WorkUnit(sourceState, null)); Assert.assertEquals(workUnitState, workUnitState); } }
/** * Create a {@link WorkUnitState} with a {@link org.apache.gobblin.broker.iface.SharedResourcesBroker} for running unit tests of * constructs. */ public static WorkUnitState createTestWorkUnitState() { return new WorkUnitState(new WorkUnit(), new State(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()). newSubscopedBuilder(new JobScopeInstance("jobName", "testJob"))); }
@Override protected WorkUnit workUnitForDataset(Dataset dataset) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(DATASET_URN, dataset.getUrn()); return workUnit; }
public WorkUnit getWorkUnitWithWeight(long weight) { WorkUnit workUnit = new WorkUnit(new Extract(Extract.TableType.APPEND_ONLY, "", "")); workUnit.setProp(WEIGHT, Long.toString(weight)); return workUnit; }
protected WorkUnit createWorkUnit(int wuNumber, String eventBusId) { WorkUnit workUnit = new WorkUnit(); TaskUtils.setTaskFactoryClass(workUnit, EventBusPublishingTaskFactory.class); workUnit.setProp(EVENTBUS_ID_KEY, eventBusId); workUnit.setProp(TASK_ID_KEY, wuNumber); return workUnit; }
public boolean addFiles(ConcurrentBoundedWorkUnitList list, String fileSetName, int fileNumber) throws IOException { FileSet.Builder<CopyEntity> partitionBuilder = new FileSet.Builder<>(fileSetName, new DummyDataset(new Path("/path"))); List<WorkUnit> workUnits = Lists.newArrayList(); for (int i = 0; i < fileNumber; i++) { CopyEntity cf = createCopyableFile(i); partitionBuilder.add(cf); WorkUnit workUnit = new WorkUnit(); CopySource.serializeCopyEntity(workUnit, cf); workUnits.add(workUnit); } return list.addFileSet(partitionBuilder.build(), workUnits); }
@BeforeClass public static void setUp() { WorkUnit workUnit = new WorkUnit(new SourceState(), new Extract(new SourceState(), Extract.TableType.SNAPSHOT_ONLY, "namespace", "dummy_table")); state = new WorkUnitState(workUnit); Type listType = new TypeToken<JsonObject>() { }.getType(); Gson gson = new Gson(); testData = gson.fromJson(new InputStreamReader( JsonElementConversionFactoryTest.class.getResourceAsStream("/converter/JsonElementConversionFactoryTest.json")), listType); }
private WorkUnit createWorkUnit(String... names) { if (names.length == 1) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(WORK_UNIT_NAME, names[0]); return workUnit; } MultiWorkUnit mwu = new MultiWorkUnit(); for (String name : names) { mwu.addWorkUnit(createWorkUnit(name)); } return mwu; }
@Override protected WorkUnit workUnitForDatasetPartition(PartitionableDataset.DatasetPartition partition) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(DATASET_URN, partition.getDataset().getUrn()); workUnit.setProp(PARTITION_URN, partition.getUrn()); return workUnit; }
public static WorkUnitState getWorkUnitState1() { WorkUnit wu = new WorkUnit(new Extract(Extract.TableType.APPEND_ONLY, "namespace", "table")); wu.setWatermarkInterval( new WatermarkInterval(new LongWatermark(20160101235959L), new LongWatermark(20160102235959L))); State js = new State(); return new WorkUnitState(wu, js); } }