void checkForUnpublishedWUHandling(String datasetUrn, JobState.DatasetState datasetState, Class<? extends DataPublisher> dataPublisherClass, Closer closer) throws ReflectiveOperationException, IOException { if (UnpublishedHandling.class.isAssignableFrom(dataPublisherClass)) { // pass in jobstate to retrieve properties DataPublisher publisher = closer.register(DataPublisher.getInstance(dataPublisherClass, this.jobContext.getJobState())); log.info(String.format("Calling publisher to handle unpublished work units for dataset %s of job %s.", datasetUrn, this.jobContext.getJobId())); ((UnpublishedHandling) publisher).handleUnpublishedWorkUnits(datasetState.getTaskStatesAsWorkUnitStates()); } }
@Override protected WorkUnit workUnitForDataset(Dataset dataset) { WorkUnit workUnit = new WorkUnit(); if(isDatasetStateStoreEnabled) { JobState.DatasetState datasetState = null; try { datasetState = (JobState.DatasetState) this.fsDatasetStateStore.getLatestDatasetState(this.jobName, dataset.getUrn()); } catch (IOException e) { throw new RuntimeException(e); } LongWatermark previousWatermark; if(datasetState != null) { previousWatermark = datasetState.getTaskStatesAsWorkUnitStates().get(0).getActualHighWatermark(LongWatermark.class); } else { previousWatermark = new LongWatermark(0); } workUnit.setWatermarkInterval(new WatermarkInterval(previousWatermark, new LongWatermark(LAST_PROCESSED_TS))); } return workUnit; }
@Override protected WorkUnit workUnitForDatasetPartition(PartitionableDataset.DatasetPartition partition) { WorkUnit workUnit = new WorkUnit(); if(isDatasetStateStoreEnabled) { String datasetUrn = partition.getDataset().getUrn()+"@"+partition.getUrn(); JobState.DatasetState datasetState = null; try { datasetState = (JobState.DatasetState) this.fsDatasetStateStore.getLatestDatasetState(this.jobName, datasetUrn); } catch (IOException e) { throw new RuntimeException(e); } LongWatermark previousWatermark; if(datasetState != null) { previousWatermark = datasetState.getTaskStatesAsWorkUnitStates().get(0).getActualHighWatermark(LongWatermark.class); } else { previousWatermark = new LongWatermark(0); } workUnit.setWatermarkInterval(new WatermarkInterval(previousWatermark, new LongWatermark(LAST_PROCESSED_TS))); } return workUnit; }
void checkForUnpublishedWUHandling(String datasetUrn, JobState.DatasetState datasetState, Class<? extends DataPublisher> dataPublisherClass, Closer closer) throws ReflectiveOperationException, IOException { if (UnpublishedHandling.class.isAssignableFrom(dataPublisherClass)) { // pass in jobstate to retrieve properties DataPublisher publisher = closer.register(DataPublisher.getInstance(dataPublisherClass, this.jobContext.getJobState())); log.info(String.format("Calling publisher to handle unpublished work units for dataset %s of job %s.", datasetUrn, this.jobContext.getJobId())); ((UnpublishedHandling) publisher).handleUnpublishedWorkUnits(datasetState.getTaskStatesAsWorkUnitStates()); } }