/** * Get the expected high {@link Watermark}. A default {@link Gson} object will be used to deserialize the watermark. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @return the expected high watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getExpectedHighWatermark(Class<T> watermarkClass) { return getExpectedHighWatermark(watermarkClass, GSON); }
/** * Get the expected high {@link Watermark}. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @param gson a {@link Gson} object used to deserialize the watermark. * @return the expected high watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getExpectedHighWatermark(Class<T> watermarkClass, Gson gson) { JsonElement json = getExpectedHighWatermark(); if (json == null) { return null; } return gson.fromJson(json, watermarkClass); }
@Override public void setActualHighWatermark(WorkUnitState wus) { wus.setActualHighWatermark(wus.getWorkunit().getExpectedHighWatermark(LongWatermark.class)); }
/** * Sets the actual high watermark by reading the expected high watermark * {@inheritDoc} * @see org.apache.gobblin.data.management.conversion.hive.watermarker.HiveSourceWatermarker#setActualHighWatermark(org.apache.gobblin.configuration.WorkUnitState) */ @Override public void setActualHighWatermark(WorkUnitState wus) { if (Boolean.valueOf(wus.getPropAsBoolean(IS_WATERMARK_WORKUNIT_KEY))) { wus.setActualHighWatermark(wus.getWorkunit().getExpectedHighWatermark(MultiKeyValueLongWatermark.class)); } else { wus.setActualHighWatermark(wus.getWorkunit().getExpectedHighWatermark(LongWatermark.class)); } }
this.partitions = KafkaUtils.getPartitions(state); this.lowWatermark = state.getWorkunit().getLowWatermark(MultiLongWatermark.class); this.highWatermark = state.getWorkunit().getExpectedHighWatermark(MultiLongWatermark.class); this.nextWatermark = new MultiLongWatermark(this.lowWatermark); this.kafkaConsumerClientResolver = new ClassAliasResolver<>(GobblinKafkaConsumerClientFactory.class);
MultiLongWatermark previousExpectedHighWatermark = workUnit.getExpectedHighWatermark(MultiLongWatermark.class); Preconditions.checkArgument(partitions.size() == watermark.size(), String .format("Num of partitions doesn't match number of watermarks: partitions=%s, watermarks=%s", partitions,
public static Partition deserialize(WorkUnit workUnit) { long lowWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; long highWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; if (workUnit.getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY) != null) { lowWatermark = workUnit.getLowWatermark(LongWatermark.class).getValue(); highWatermark = workUnit.getExpectedHighWatermark(LongWatermark.class).getValue(); } return new Partition(lowWatermark, highWatermark, workUnit.getPropAsBoolean(Partition.IS_LAST_PARTIITON), workUnit.getPropAsBoolean(Partition.HAS_USER_SPECIFIED_HIGH_WATERMARK)); } }
wuState.getWorkunit().getExpectedHighWatermark(LongWatermark.class).getValue(), positionMap, dimensions, metrics, null, Arrays.asList(dataFetcher1, dataFetcher2));
Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset1"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), 0); watermarks1.add(workUnits.get(0).getExpectedHighWatermark(LongWatermark.class)); datasets1.add(dataset1); watermarks1.add(workUnits.get(1).getExpectedHighWatermark(LongWatermark.class)); datasets1.add(dataset2); watermarks1.add(workUnits.get(2).getExpectedHighWatermark(LongWatermark.class)); datasets1.add(dataset3); Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset4"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), 0); watermarks2.add(workUnits.get(0).getExpectedHighWatermark(LongWatermark.class)); datasets2.add(dataset4); watermarks2.add(workUnits.get(1).getExpectedHighWatermark(LongWatermark.class)); datasets2.add(dataset5); Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset1"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), watermarks1.get(0).getValue()); watermarks3.add(workUnits.get(0).getExpectedHighWatermark(LongWatermark.class)); datasets3.add(dataset1); watermarks3.add(workUnits.get(1).getExpectedHighWatermark(LongWatermark.class)); datasets3.add(dataset2); watermarks3.add(workUnits.get(2).getExpectedHighWatermark(LongWatermark.class)); datasets3.add(dataset3);
watermarkWu.getExpectedHighWatermark(MultiKeyValueLongWatermark.class).getWatermarks(); Assert.assertEquals(workunitWatermarks.size(), 3, "expectedHighWatermarks size");
.setTitle(Preconditions.checkNotNull(wuState.getProp(SOURCE_ENTITY), SOURCE_ENTITY + " is required.")) .setStartDate(convertFormat(wuState.getWorkunit().getLowWatermark(LongWatermark.class).getValue())) .setEndDate(convertFormat(wuState.getWorkunit().getExpectedHighWatermark(LongWatermark.class).getValue())) .setMetrics(Preconditions.checkNotNull(wuState.getProp(METRICS), METRICS + " is required."))
Assert.assertEquals(workunits.get(1).getProp(ConfigurationKeys.DATASET_URN_KEY), table2.getCompleteName()); Assert.assertEquals(workunits.get(0).getExpectedHighWatermark(MultiKeyValueLongWatermark.class).getWatermarks(), ImmutableMap.of(PartitionLevelWatermarker.partitionKey(part1), now + 2015l)); Assert.assertEquals(workunits.get(1).getExpectedHighWatermark(MultiKeyValueLongWatermark.class).getWatermarks(), ImmutableMap.of(PartitionLevelWatermarker.partitionKey(part2), now + 16l));
Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset1"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), 0); watermarks1.add(workUnits.get(0).getExpectedHighWatermark(LongWatermark.class)); datasets1.add(dataset1); watermarks1.add(workUnits.get(1).getExpectedHighWatermark(LongWatermark.class)); datasets1.add(new SimpleDatasetForTesting("dataset2@p1")); watermarks1.add(workUnits.get(2).getExpectedHighWatermark(LongWatermark.class)); datasets1.add(new SimpleDatasetForTesting("dataset2@p2")); Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset2@p3"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), 0); watermarks2.add(workUnits.get(0).getExpectedHighWatermark(LongWatermark.class)); datasets2.add(new SimpleDatasetForTesting("dataset2@p3")); watermarks2.add(workUnits.get(1).getExpectedHighWatermark(LongWatermark.class)); datasets2.add(new SimpleDatasetForTesting("dataset3@p1")); watermarks2.add(workUnits.get(2).getExpectedHighWatermark(LongWatermark.class)); datasets2.add(new SimpleDatasetForTesting("dataset3@p2")); Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset3@p3"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), 0); watermarks3.add(workUnits.get(0).getExpectedHighWatermark(LongWatermark.class)); datasets3.add(new SimpleDatasetForTesting("dataset3@p3")); Assert.assertEquals(workUnits.get(0).getProp(ConfigurationKeys.DATASET_URN_KEY), "dataset1"); Assert.assertEquals(workUnits.get(0).getLowWatermark(LongWatermark.class).getValue(), watermarks1.get(0).getValue());
watermarkWu.getExpectedHighWatermark(MultiKeyValueLongWatermark.class).getWatermarks(); Assert.assertEquals(workunitWatermarks.size(), 3, "expectedHighWatermarks size");
@Test public void testWatermarkWorkUnitSerialization() { long lowWatermarkValue = 0; long expectedHighWatermarkValue = 100; TestWatermark lowWatermark = new TestWatermark(); lowWatermark.setLongWatermark(lowWatermarkValue); TestWatermark expectedHighWatermark = new TestWatermark(); expectedHighWatermark.setLongWatermark(expectedHighWatermarkValue); WatermarkInterval watermarkInterval = new WatermarkInterval(lowWatermark, expectedHighWatermark); WorkUnit workUnit = new WorkUnit(null, null, watermarkInterval); TestWatermark deserializedLowWatermark = WatermarkSerializerHelper.convertJsonToWatermark(workUnit.getLowWatermark(), TestWatermark.class); TestWatermark deserializedExpectedHighWatermark = WatermarkSerializerHelper.convertJsonToWatermark(workUnit.getExpectedHighWatermark(), TestWatermark.class); Assert.assertEquals(deserializedLowWatermark.getLongWatermark(), lowWatermarkValue); Assert.assertEquals(deserializedExpectedHighWatermark.getLongWatermark(), expectedHighWatermarkValue); }
/** * Get the expected high {@link Watermark}. A default {@link Gson} object will be used to deserialize the watermark. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @return the expected high watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getExpectedHighWatermark(Class<T> watermarkClass) { return getExpectedHighWatermark(watermarkClass, GSON); }
/** * Get the expected high {@link Watermark}. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @param gson a {@link Gson} object used to deserialize the watermark. * @return the expected high watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getExpectedHighWatermark(Class<T> watermarkClass, Gson gson) { JsonElement json = getExpectedHighWatermark(); if (json == null) { return null; } return gson.fromJson(json, watermarkClass); }
@Override public void setActualHighWatermark(WorkUnitState wus) { wus.setActualHighWatermark(wus.getWorkunit().getExpectedHighWatermark(LongWatermark.class)); }
/** * Sets the actual high watermark by reading the expected high watermark * {@inheritDoc} * @see org.apache.gobblin.data.management.conversion.hive.watermarker.HiveSourceWatermarker#setActualHighWatermark(org.apache.gobblin.configuration.WorkUnitState) */ @Override public void setActualHighWatermark(WorkUnitState wus) { if (Boolean.valueOf(wus.getPropAsBoolean(IS_WATERMARK_WORKUNIT_KEY))) { wus.setActualHighWatermark(wus.getWorkunit().getExpectedHighWatermark(MultiKeyValueLongWatermark.class)); } else { wus.setActualHighWatermark(wus.getWorkunit().getExpectedHighWatermark(LongWatermark.class)); } }
public static Partition deserialize(WorkUnit workUnit) { long lowWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; long highWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; if (workUnit.getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY) != null) { lowWatermark = workUnit.getLowWatermark(LongWatermark.class).getValue(); highWatermark = workUnit.getExpectedHighWatermark(LongWatermark.class).getValue(); } return new Partition(lowWatermark, highWatermark, workUnit.getPropAsBoolean(Partition.IS_LAST_PARTIITON), workUnit.getPropAsBoolean(Partition.HAS_USER_SPECIFIED_HIGH_WATERMARK)); } }