private synchronized void buildAndExecuteCommitSequence(CommitSequence.Builder builder, JobState.DatasetState datasetState, String datasetUrn) throws IOException { CommitSequence commitSequence = builder.addStep(buildDatasetStateCommitStep(datasetUrn, datasetState).get()).build(); this.jobContext.getCommitSequenceStore().get().put(commitSequence.getJobName(), datasetUrn, commitSequence); commitSequence.execute(); this.jobContext.getCommitSequenceStore().get().delete(commitSequence.getJobName(), datasetUrn); }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.delete(new Path(ROOT_DIR), true); Path storeRootDir = new Path(ROOT_DIR, "store"); Path dir1 = new Path(ROOT_DIR, "dir1"); Path dir2 = new Path(ROOT_DIR, "dir2"); this.fs.mkdirs(dir1); this.fs.mkdirs(dir2); Path src1 = new Path(dir1, "file1"); Path src2 = new Path(dir2, "file2"); Path dst1 = new Path(dir2, "file1"); Path dst2 = new Path(dir1, "file2"); this.fs.createNewFile(src1); this.fs.createNewFile(src2); DatasetState ds = new DatasetState("job-name", "job-id"); ds.setDatasetUrn("urn"); ds.setNoJobFailure(); State state = new State(); state.setProp(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeRootDir.toString()); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(src1).to(dst1).withProps(state).endStep() .beginStep(FsRenameCommitStep.Builder.class).from(src2).to(dst2).withProps(state).endStep() .beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn("urn").withDatasetState(ds).withProps(state) .endStep().build(); }
@BeforeClass public void setUp() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); this.store = new FsCommitSequenceStore(fs, new Path("commit-sequence-store-test")); State props = new State(); props.setId("propsId"); props.setProp("prop1", "valueOfProp1"); props.setProp("prop2", "valueOfProp2"); DatasetState datasetState = new DatasetState(); datasetState.setDatasetUrn(this.datasetUrn); datasetState.incrementJobFailures(); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(new Path("/ab/cd")).to(new Path("/ef/gh")).withProps(props) .endStep().beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn(this.datasetUrn) .withDatasetState(datasetState).withProps(props).endStep().build(); }
private synchronized void buildAndExecuteCommitSequence(CommitSequence.Builder builder, JobState.DatasetState datasetState, String datasetUrn) throws IOException { CommitSequence commitSequence = builder.addStep(buildDatasetStateCommitStep(datasetUrn, datasetState).get()).build(); this.jobContext.getCommitSequenceStore().get().put(commitSequence.getJobName(), datasetUrn, commitSequence); commitSequence.execute(); this.jobContext.getCommitSequenceStore().get().delete(commitSequence.getJobName(), datasetUrn); }