/** * Get the {@link org.apache.gobblin.source.workunit.Extract} associated with the {@link WorkUnit}. * * @return {@link org.apache.gobblin.source.workunit.Extract} associated with the {@link WorkUnit} */ public Extract getExtract() { return new Extract(this.workUnit.getExtract()); }
/** * Copy constructor. * * @param other the other {@link WorkUnit} instance * * @deprecated Use {@link #copyOf(WorkUnit)} */ @Deprecated public WorkUnit(WorkUnit other) { super.addAll(other); this.extract = other.getExtract(); }
public ImmutableWorkUnit(WorkUnit workUnit) { super(workUnit.getExtract()); // Only copy the specProperties from the given workUnit. Properties specificPropertiesCopy = new Properties(); specificPropertiesCopy.putAll(workUnit.getSpecProperties()); super.setProps(workUnit.getCommonProperties(), specificPropertiesCopy); }
@Test public void testGetDefaultWriterFilePath() { String namespace = "gobblin.test"; String tableName = "test-table"; SourceState sourceState = new SourceState(); WorkUnit state = WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName)); Assert.assertEquals(WriterUtils.getWriterFilePath(state, 0, 0), new Path(state.getExtract().getOutputFilePath())); Assert.assertEquals(WriterUtils.getWriterFilePath(state, 2, 0), new Path(state.getExtract().getOutputFilePath(), ConfigurationKeys.DEFAULT_FORK_BRANCH_NAME + "0")); }
Extract previousExtract = pre.getWorkunit().getExtract(); if (previousExtract.getNamespace().equals(namespace) && previousExtract.getTable().equals(table)) { this.previousTableState.addAll(pre);
@Test public void testCopySource() throws Exception { SourceState state = new SourceState(); state.setProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, "file:///"); state.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, "file:///"); state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/target/dir"); state.setProp(DatasetUtils.DATASET_PROFILE_CLASS_KEY, TestCopyableDatasetFinder.class.getName()); CopySource source = new CopySource(); List<WorkUnit> workunits = source.getWorkunits(state); workunits = JobLauncherUtils.flattenWorkUnits(workunits); Assert.assertEquals(workunits.size(), TestCopyableDataset.FILE_COUNT); Extract extract = workunits.get(0).getExtract(); for (WorkUnit workUnit : workunits) { CopyableFile file = (CopyableFile) CopySource.deserializeCopyEntity(workUnit); Assert.assertTrue(file.getOrigin().getPath().toString().startsWith(TestCopyableDataset.ORIGIN_PREFIX)); Assert.assertEquals(file.getDestinationOwnerAndPermission(), TestCopyableDataset.OWNER_AND_PERMISSION); Assert.assertEquals(workUnit.getExtract(), extract); } }
extractBelow = workUnit.getExtract(); Assert.assertEquals(workUnit.getExtract(), extractBelow); } else { extractAbove = workUnit.getExtract(); Assert.assertEquals(workUnit.getExtract(), extractAbove);
/** * Get the {@link org.apache.gobblin.source.workunit.Extract} associated with the {@link WorkUnit}. * * @return {@link org.apache.gobblin.source.workunit.Extract} associated with the {@link WorkUnit} */ public Extract getExtract() { return new Extract(this.workUnit.getExtract()); }
/** * Copy constructor. * * @param other the other {@link WorkUnit} instance * * @deprecated Use {@link #copyOf(WorkUnit)} */ @Deprecated public WorkUnit(WorkUnit other) { super.addAll(other); this.extract = other.getExtract(); }
public ImmutableWorkUnit(WorkUnit workUnit) { super(workUnit.getExtract()); // Only copy the specProperties from the given workUnit. Properties specificPropertiesCopy = new Properties(); specificPropertiesCopy.putAll(workUnit.getSpecProperties()); super.setProps(workUnit.getCommonProperties(), specificPropertiesCopy); }
Extract previousExtract = pre.getWorkunit().getExtract(); if (previousExtract.getNamespace().equals(namespace) && previousExtract.getTable().equals(table)) { this.previousTableState.addAll(pre);