/** * Override {@link #workUnit}'s properties with new commonProps and specProps. */ public void setWuProperties(Properties commonProps, Properties specProps) { this.workUnit.setProps(commonProps, specProps); }
public ImmutableWorkUnit(WorkUnit workUnit) { super(workUnit.getExtract()); // Only copy the specProperties from the given workUnit. Properties specificPropertiesCopy = new Properties(); specificPropertiesCopy.putAll(workUnit.getSpecProperties()); super.setProps(workUnit.getCommonProperties(), specificPropertiesCopy); }
/** * Create a work unit for each configuration defined or a single work unit if no configurations are defined * @param state see {@link org.apache.gobblin.configuration.SourceState} * @return list of workunits */ @Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); Config config = ConfigUtils.propertiesToConfig(state.getProperties()); Config sourceConfig = ConfigUtils.getConfigOrEmpty(config, DATASET_CLEANER_SOURCE_PREFIX); List<String> configurationNames = ConfigUtils.getStringList(config, DATASET_CLEANER_CONFIGURATIONS); // use a dummy configuration name if none set if (configurationNames.isEmpty()) { configurationNames = ImmutableList.of("DummyConfig"); } for (String configurationName: configurationNames) { WorkUnit workUnit = WorkUnit.createEmpty(); // specific configuration prefixed by the configuration name has precedence over the source specific configuration // and the source specific configuration has precedence over the general configuration Config wuConfig = ConfigUtils.getConfigOrEmpty(sourceConfig, configurationName).withFallback(sourceConfig) .withFallback(config); workUnit.setProps(ConfigUtils.configToProperties(wuConfig), new Properties()); TaskUtils.setTaskFactoryClass(workUnit, DatasetCleanerTaskFactory.class); workUnits.add(workUnit); } return workUnits; }
initialProps.setProperty("spec1", String.valueOf(i)); initialProps.setProperty("spec2", "spec" + i); workUnit.setProps(new Properties(), initialProps); WorkUnitState workUnitState = new WorkUnitState(workUnit); workUnitState.setProp(ConfigurationKeys.JOB_ID_KEY, "TestJob-1");
/** * Override {@link #workUnit}'s properties with new commonProps and specProps. */ public void setWuProperties(Properties commonProps, Properties specProps) { this.workUnit.setProps(commonProps, specProps); }
public ImmutableWorkUnit(WorkUnit workUnit) { super(workUnit.getExtract()); // Only copy the specProperties from the given workUnit. Properties specificPropertiesCopy = new Properties(); specificPropertiesCopy.putAll(workUnit.getSpecProperties()); super.setProps(workUnit.getCommonProperties(), specificPropertiesCopy); }
/** * Create a work unit for each configuration defined or a single work unit if no configurations are defined * @param state see {@link org.apache.gobblin.configuration.SourceState} * @return list of workunits */ @Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); Config config = ConfigUtils.propertiesToConfig(state.getProperties()); Config sourceConfig = ConfigUtils.getConfigOrEmpty(config, DATASET_CLEANER_SOURCE_PREFIX); List<String> configurationNames = ConfigUtils.getStringList(config, DATASET_CLEANER_CONFIGURATIONS); // use a dummy configuration name if none set if (configurationNames.isEmpty()) { configurationNames = ImmutableList.of("DummyConfig"); } for (String configurationName: configurationNames) { WorkUnit workUnit = WorkUnit.createEmpty(); // specific configuration prefixed by the configuration name has precedence over the source specific configuration // and the source specific configuration has precedence over the general configuration Config wuConfig = ConfigUtils.getConfigOrEmpty(sourceConfig, configurationName).withFallback(sourceConfig) .withFallback(config); workUnit.setProps(ConfigUtils.configToProperties(wuConfig), new Properties()); TaskUtils.setTaskFactoryClass(workUnit, DatasetCleanerTaskFactory.class); workUnits.add(workUnit); } return workUnits; }