private void executeUnfinishedCommitSequences(String jobName) throws IOException { Preconditions.checkState(this.jobContext.getCommitSequenceStore().isPresent()); CommitSequenceStore commitSequenceStore = this.jobContext.getCommitSequenceStore().get(); for (String datasetUrn : commitSequenceStore.get(jobName)) { Optional<CommitSequence> commitSequence = commitSequenceStore.get(jobName, datasetUrn); if (commitSequence.isPresent()) { commitSequence.get().execute(); } commitSequenceStore.delete(jobName, datasetUrn); } }
/** * Staging data cannot be cleaned if exactly once semantics is used, and the job has unfinished * commit sequences. */ private boolean canCleanStagingData(JobState jobState) throws IOException { return this.jobContext.getSemantics() != DeliverySemantics.EXACTLY_ONCE || !this.jobContext.getCommitSequenceStore() .get().exists(jobState.getJobName()); }
private synchronized void buildAndExecuteCommitSequence(CommitSequence.Builder builder, JobState.DatasetState datasetState, String datasetUrn) throws IOException { CommitSequence commitSequence = builder.addStep(buildDatasetStateCommitStep(datasetUrn, datasetState).get()).build(); this.jobContext.getCommitSequenceStore().get().put(commitSequence.getJobName(), datasetUrn, commitSequence); commitSequence.execute(); this.jobContext.getCommitSequenceStore().get().delete(commitSequence.getJobName(), datasetUrn); }
private void executeUnfinishedCommitSequences(String jobName) throws IOException { Preconditions.checkState(this.jobContext.getCommitSequenceStore().isPresent()); CommitSequenceStore commitSequenceStore = this.jobContext.getCommitSequenceStore().get(); for (String datasetUrn : commitSequenceStore.get(jobName)) { Optional<CommitSequence> commitSequence = commitSequenceStore.get(jobName, datasetUrn); if (commitSequence.isPresent()) { commitSequence.get().execute(); } commitSequenceStore.delete(jobName, datasetUrn); } }
/** * Staging data cannot be cleaned if exactly once semantics is used, and the job has unfinished * commit sequences. */ private boolean canCleanStagingData(JobState jobState) throws IOException { return this.jobContext.getSemantics() != DeliverySemantics.EXACTLY_ONCE || !this.jobContext.getCommitSequenceStore() .get().exists(jobState.getJobName()); }
private synchronized void buildAndExecuteCommitSequence(CommitSequence.Builder builder, JobState.DatasetState datasetState, String datasetUrn) throws IOException { CommitSequence commitSequence = builder.addStep(buildDatasetStateCommitStep(datasetUrn, datasetState).get()).build(); this.jobContext.getCommitSequenceStore().get().put(commitSequence.getJobName(), datasetUrn, commitSequence); commitSequence.execute(); this.jobContext.getCommitSequenceStore().get().delete(commitSequence.getJobName(), datasetUrn); }