@Override public void publish(Collection<? extends WorkUnitState> states) throws IOException { super.publish(states); if (!states.isEmpty()) { String jobName = Iterables.get(states, 0).getProp(ConfigurationKeys.JOB_NAME_KEY); String datasetUrn = Iterables.get(states, 0).getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN); this.commitSequenceBuilder.get().withJobName(jobName).withDatasetUrn(datasetUrn); } else { log.warn("No workunitstate to publish"); this.commitSequenceBuilder = Optional.<CommitSequence.Builder> absent(); } }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.delete(new Path(ROOT_DIR), true); Path storeRootDir = new Path(ROOT_DIR, "store"); Path dir1 = new Path(ROOT_DIR, "dir1"); Path dir2 = new Path(ROOT_DIR, "dir2"); this.fs.mkdirs(dir1); this.fs.mkdirs(dir2); Path src1 = new Path(dir1, "file1"); Path src2 = new Path(dir2, "file2"); Path dst1 = new Path(dir2, "file1"); Path dst2 = new Path(dir1, "file2"); this.fs.createNewFile(src1); this.fs.createNewFile(src2); DatasetState ds = new DatasetState("job-name", "job-id"); ds.setDatasetUrn("urn"); ds.setNoJobFailure(); State state = new State(); state.setProp(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeRootDir.toString()); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(src1).to(dst1).withProps(state).endStep() .beginStep(FsRenameCommitStep.Builder.class).from(src2).to(dst2).withProps(state).endStep() .beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn("urn").withDatasetState(ds).withProps(state) .endStep().build(); }
@BeforeClass public void setUp() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); this.store = new FsCommitSequenceStore(fs, new Path("commit-sequence-store-test")); State props = new State(); props.setId("propsId"); props.setProp("prop1", "valueOfProp1"); props.setProp("prop2", "valueOfProp2"); DatasetState datasetState = new DatasetState(); datasetState.setDatasetUrn(this.datasetUrn); datasetState.incrementJobFailures(); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(new Path("/ab/cd")).to(new Path("/ef/gh")).withProps(props) .endStep().beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn(this.datasetUrn) .withDatasetState(datasetState).withProps(props).endStep().build(); }
@Override public void publish(Collection<? extends WorkUnitState> states) throws IOException { super.publish(states); if (!states.isEmpty()) { String jobName = Iterables.get(states, 0).getProp(ConfigurationKeys.JOB_NAME_KEY); String datasetUrn = Iterables.get(states, 0).getProp(ConfigurationKeys.DATASET_URN_KEY, ConfigurationKeys.DEFAULT_DATASET_URN); this.commitSequenceBuilder.get().withJobName(jobName).withDatasetUrn(datasetUrn); } else { log.warn("No workunitstate to publish"); this.commitSequenceBuilder = Optional.<CommitSequence.Builder> absent(); } }