private static boolean isAnyDataProcessed(WorkUnitState wus) { return wus.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, 0) > 0; }
/** * Get the high watermark as set in {@link org.apache.gobblin.source.extractor.Extractor}. * * @return high watermark * @deprecated use {@link #getActualHighWatermark}. */ @Deprecated public long getHighWaterMark() { return getPropAsLong(ConfigurationKeys.WORK_UNIT_STATE_RUNTIME_HIGH_WATER_MARK, ConfigurationKeys.DEFAULT_WATERMARK_VALUE); }
@Override public Extractor<String, Long> getExtractor(WorkUnitState state) throws IOException { return new FastSequentialExtractor(state.getPropAsLong(MAX_RECORDS_PER_WORK_UNIT), state.getPropAsLong(MAX_SECONDS_PER_WORK_UNIT)); }
public KafkaSimpleStreamingExtractor(WorkUnitState state) { super(state); _consumer = KafkaSimpleStreamingSource.getKafkaConsumer(ConfigUtils.propertiesToConfig(state.getProperties())); closer.register(_consumer); _partition = new TopicPartition(KafkaSimpleStreamingSource.getTopicNameFromState(state), KafkaSimpleStreamingSource.getPartitionIdFromState(state)); _consumer.assign(Collections.singletonList(_partition)); this._schemaRegistry = state.contains(KafkaSchemaRegistry.KAFKA_SCHEMA_REGISTRY_CLASS) ? Optional .of(KafkaSchemaRegistry.<String, S>get(state.getProperties())) : Optional.<KafkaSchemaRegistry<String, S>>absent(); this.fetchTimeOut = state.getPropAsLong(AbstractBaseKafkaConsumerClient.CONFIG_KAFKA_FETCH_TIMEOUT_VALUE, AbstractBaseKafkaConsumerClient.CONFIG_KAFKA_FETCH_TIMEOUT_VALUE_DEFAULT); }
Assert.assertEquals(state.getPropAsList("list", "item1,item2").get(0), "item1"); Assert.assertEquals(state.getPropAsList("list", "item1,item2").get(1), "item2"); Assert.assertEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE); Assert.assertEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE); Assert.assertEquals(state.getPropAsDouble("double", Double.MAX_VALUE), Double.MAX_VALUE); Assert.assertEquals(state.getPropAsList("list").get(0), "item1"); Assert.assertEquals(state.getPropAsList("list").get(1), "item2"); Assert.assertEquals(state.getPropAsLong("long"), Long.MAX_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MAX_VALUE); Assert.assertEquals(state.getPropAsDouble("double"), Double.MAX_VALUE); Assert.assertNotEquals(state.getPropAsList("list", "item1,item2").get(0), "item1"); Assert.assertNotEquals(state.getPropAsList("list", "item1,item2").get(1), "item2"); Assert.assertNotEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE); Assert.assertNotEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE); Assert.assertNotEquals(state.getPropAsDouble("double", Double.MAX_VALUE), Double.MAX_VALUE); Assert.assertNotEquals(state.getPropAsList("list").get(0), "item1"); Assert.assertNotEquals(state.getPropAsList("list").get(1), "item2"); Assert.assertNotEquals(state.getPropAsLong("long"), Long.MAX_VALUE); Assert.assertNotEquals(state.getPropAsInt("int"), Integer.MAX_VALUE); Assert.assertNotEquals(state.getPropAsDouble("double"), Double.MAX_VALUE); Assert.assertEquals(state.getPropAsList("list").get(0), "item3"); Assert.assertEquals(state.getPropAsList("list").get(1), "item4"); Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE); Assert.assertEquals(state.getPropAsDouble("double"), Double.MIN_VALUE);
private static boolean isAnyDataProcessed(WorkUnitState wus) { return wus.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, 0) > 0; }
/** * Get the high watermark as set in {@link org.apache.gobblin.source.extractor.Extractor}. * * @return high watermark * @deprecated use {@link #getActualHighWatermark}. */ @Deprecated public long getHighWaterMark() { return getPropAsLong(ConfigurationKeys.WORK_UNIT_STATE_RUNTIME_HIGH_WATER_MARK, ConfigurationKeys.DEFAULT_WATERMARK_VALUE); }
@Override public Extractor<String, Long> getExtractor(WorkUnitState state) throws IOException { return new FastSequentialExtractor(state.getPropAsLong(MAX_RECORDS_PER_WORK_UNIT), state.getPropAsLong(MAX_SECONDS_PER_WORK_UNIT)); }