/** * Builds a State containing all properties added with {@link #addFinalConstructState} * to this {@link org.apache.gobblin.configuration.WorkUnitState}. All such properties will be stripped of * {@link #FINAL_CONSTRUCT_STATE_PREFIX} but not of any infixes. * * <p> * For example, if state={sample.property: sampleValue} * then * <pre> * {@code * this.addFinalConstructState("infix",state); * this.getFinalConstructState(); * } * </pre> * will return state={infix.sample.property: sampleValue} * </p> * * @return State containing all properties added with {@link #addFinalConstructState}. */ public State getFinalConstructStates() { State constructState = new State(); for (String property : getPropertyNames()) { if (property.startsWith(FINAL_CONSTRUCT_STATE_PREFIX)) { constructState.setProp(property.substring(FINAL_CONSTRUCT_STATE_PREFIX.length()), getProp(property)); } } return constructState; } }
@Override public Extractor<S, D> getExtractor(WorkUnitState workUnitState) throws IOException { if (!workUnitState.contains(HadoopFileInputSource.FILE_SPLIT_BYTES_STRING_KEY)) { throw new IOException("No serialized FileSplit found in WorkUnitState " + workUnitState.getId()); } JobConf jobConf = new JobConf(new Configuration()); for (String key : workUnitState.getPropertyNames()) { jobConf.set(key, workUnitState.getProp(key)); } String fileSplitBytesStr = workUnitState.getProp(HadoopFileInputSource.FILE_SPLIT_BYTES_STRING_KEY); FileSplit fileSplit = (FileSplit) HadoopUtils.deserializeFromString(FileSplit.class, fileSplitBytesStr); FileInputFormat<K, V> fileInputFormat = getFileInputFormat(workUnitState, jobConf); RecordReader<K, V> recordReader = fileInputFormat.getRecordReader(fileSplit, jobConf, Reporter.NULL); boolean readKeys = workUnitState.getPropAsBoolean( HadoopFileInputSource.FILE_INPUT_READ_KEYS_KEY, HadoopFileInputSource.DEFAULT_FILE_INPUT_READ_KEYS); return getExtractor(workUnitState, recordReader, fileSplit, readKeys); }
for (String key : workUnitState.getPropertyNames()) { conf.set(key, workUnitState.getProp(key));
@Test public void test() { ConstructState constructState = new ConstructState(); WorkUnitState workUnitState = new WorkUnitState(); String overrideKey = "overrideMe"; String nonOverrideKey = "overrideMe.not"; String workUnitToken = "workUnit"; String constructToken = "construct"; workUnitState.setProp(overrideKey, workUnitToken); workUnitState.setProp(nonOverrideKey, workUnitToken); constructState.addOverwriteProperties(ImmutableMap.<String, String>builder().put(overrideKey, constructToken).build()); constructState.setProp(nonOverrideKey, constructToken); constructState.mergeIntoWorkUnitState(workUnitState); Assert.assertEquals(workUnitState.getProp(overrideKey), constructToken); Assert.assertEquals(workUnitState.getProp(nonOverrideKey), workUnitToken); Assert.assertEquals(workUnitState.getPropertyNames().size(), 3); }
/** * Builds a State containing all properties added with {@link #addFinalConstructState} * to this {@link org.apache.gobblin.configuration.WorkUnitState}. All such properties will be stripped of * {@link #FINAL_CONSTRUCT_STATE_PREFIX} but not of any infixes. * * <p> * For example, if state={sample.property: sampleValue} * then * <pre> * {@code * this.addFinalConstructState("infix",state); * this.getFinalConstructState(); * } * </pre> * will return state={infix.sample.property: sampleValue} * </p> * * @return State containing all properties added with {@link #addFinalConstructState}. */ public State getFinalConstructStates() { State constructState = new State(); for (String property : getPropertyNames()) { if (property.startsWith(FINAL_CONSTRUCT_STATE_PREFIX)) { constructState.setProp(property.substring(FINAL_CONSTRUCT_STATE_PREFIX.length()), getProp(property)); } } return constructState; } }
@Override public Extractor<S, D> getExtractor(WorkUnitState workUnitState) throws IOException { if (!workUnitState.contains(HadoopFileInputSource.FILE_SPLIT_BYTES_STRING_KEY)) { throw new IOException("No serialized FileSplit found in WorkUnitState " + workUnitState.getId()); } JobConf jobConf = new JobConf(new Configuration()); for (String key : workUnitState.getPropertyNames()) { jobConf.set(key, workUnitState.getProp(key)); } String fileSplitBytesStr = workUnitState.getProp(HadoopFileInputSource.FILE_SPLIT_BYTES_STRING_KEY); FileSplit fileSplit = (FileSplit) HadoopUtils.deserializeFromString(FileSplit.class, fileSplitBytesStr); FileInputFormat<K, V> fileInputFormat = getFileInputFormat(workUnitState, jobConf); RecordReader<K, V> recordReader = fileInputFormat.getRecordReader(fileSplit, jobConf, Reporter.NULL); boolean readKeys = workUnitState.getPropAsBoolean( HadoopFileInputSource.FILE_INPUT_READ_KEYS_KEY, HadoopFileInputSource.DEFAULT_FILE_INPUT_READ_KEYS); return getExtractor(workUnitState, recordReader, fileSplit, readKeys); }
for (String key : workUnitState.getPropertyNames()) { conf.set(key, workUnitState.getProp(key));