Codota Logo
SourceFunction$SourceContext.collectWithTimestamp
Code IndexAdd Codota to your IDE (free)

How to use
collectWithTimestamp
method
in
org.apache.flink.streaming.api.functions.source.SourceFunction$SourceContext

Best Java code snippets using org.apache.flink.streaming.api.functions.source.SourceFunction$SourceContext.collectWithTimestamp (Showing top 20 results out of 315)

  • Add the Codota plugin to your IDE and get smart completions
private void myMethod () {
StringBuilder s =
  • Codota Iconnew StringBuilder()
  • Codota Iconnew StringBuilder(32)
  • Codota IconString str;new StringBuilder(str)
  • Smart code suggestions by Codota
}
origin: apache/flink

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
  for (int i = 0; i < numWatermarks; i++) {
    ctx.collectWithTimestamp(i, initialTime + i);
    ctx.emitWatermark(new Watermark(initialTime + i));
  }
  while (running) {
    Thread.sleep(20);
  }
}
origin: apache/flink

  break;
case COLLECT_WITH_TIMESTAMP:
  context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
  expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
  expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
  break;
case COLLECT_WITH_TIMESTAMP:
  context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
  expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
  assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
origin: apache/flink

  break;
case COLLECT_WITH_TIMESTAMP:
  context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
  break;
case EMIT_WATERMARK:
  break;
case COLLECT_WITH_TIMESTAMP:
  context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
  break;
case EMIT_WATERMARK:
origin: apache/flink

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
  for (int i = 0; i < numWatermarks; i++) {
    ctx.collectWithTimestamp(i, initialTime + i);
    ctx.emitWatermark(new Watermark(initialTime + i));
  }
}
origin: apache/flink

@Override
public void run(SourceContext<Tuple2<String, Integer>> ctx) {
  ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L);
  ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L);
  ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L);
  ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L);
  ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L);
  ctx.emitWatermark(new Watermark(5));
  ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L);
  ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L);
  ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L);
  ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L);
}
origin: apache/flink

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
  int index = 1;
  while (index <= numElements) {
    ctx.collectWithTimestamp(index, index);
    ctx.collectWithTimestamp(index - 1, index - 1);
    index++;
    ctx.emitWatermark(new Watermark(index - 2));
  }
  // emit the final Long.MAX_VALUE watermark, do it twice and verify that
  // we only see one in the result
  ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
  ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
}
origin: apache/flink

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
  int index = 1;
  while (index <= numElements) {
    ctx.collectWithTimestamp(index, index);
    ctx.collectWithTimestamp(index - 1, index - 1);
    index++;
    ctx.emitWatermark(new Watermark(index - 2));
  }
  // emit the final Long.MAX_VALUE watermark, do it twice and verify that
  // we only see one in the result
  ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
  ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
}
origin: apache/flink

@Override
public void run(SourceContext<SessionEvent<Integer, TestEventPayload>> ctx) {
  ParallelSessionsEventGenerator<Integer, SessionEvent<Integer, TestEventPayload>> generator = createGenerator();
  this.isRunning = true;
  //main data source driver loop
  while (isRunning) {
    synchronized (ctx.getCheckpointLock()) {
      SessionEvent<Integer, TestEventPayload> evt = generator.nextEvent();
      if (evt != null) {
        ctx.collectWithTimestamp(evt, evt.getEventTimestamp());
        ctx.emitWatermark(new Watermark(generator.getWatermark()));
      } else {
        break;
      }
    }
  }
}
origin: apache/flink

@Override
public void run(SourceContext<Tuple2<String, Integer>> ctx) {
  ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L);
  ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L);
  ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L);
  ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L);
  ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L);
  ctx.emitWatermark(new Watermark(5));
  ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L);
  ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L);
  ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L);
  ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L);
}
origin: apache/flink

/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a punctuated watermark generator.
 */
private void emitRecordWithTimestampAndPunctuatedWatermark(
    T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
  @SuppressWarnings("unchecked")
  final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
      (KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState;
  // only one thread ever works on accessing timestamps and watermarks
  // from the punctuated extractor
  final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
  final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp);
  // emit the record with timestamp, using the usual checkpoint lock to guarantee
  // atomicity of record emission and offset state update
  synchronized (checkpointLock) {
    sourceContext.collectWithTimestamp(record, timestamp);
    partitionState.setOffset(offset);
  }
  // if we also have a new per-partition watermark, check if that is also a
  // new cross-partition watermark
  if (newWatermark != null) {
    updateMinPunctuatedWatermark(newWatermark);
  }
}
origin: apache/flink

sourceContext.collectWithTimestamp(record, timestamp);
partitionState.setOffset(offset);
origin: apache/bahir-flink

@Override
public void run(SourceContext<Tuple4<Integer, String, Double, Long>> ctx) throws Exception {
  while (isRunning) {
    long timestamp = initialTimestamp + 1000 * number.get();
    ctx.collectWithTimestamp(Tuple4.of(number.get(), "test_tuple", random.nextDouble(), timestamp), timestamp);
    if (number.incrementAndGet() >= this.count) {
      cancel();
    }
  }
}
origin: apache/bahir-flink

@Override
public void run(SourceContext<Event> ctx) throws Exception {
  while (isRunning) {
    long timestamp = initialTimestamp + 1000 * number.get();
    ctx.collectWithTimestamp(Event.of(number.get(), "test_event", random.nextDouble(), timestamp), timestamp);
    if (number.incrementAndGet() >= this.count) {
      cancel();
    }
  }
}
origin: apache/flink

  @Override
  public void emitEvent(SourceFunction.SourceContext<Tuple2<Long, IntType>> ctx, int eventSequenceNo) {
    final IntType intTypeNext = new IntType(eventSequenceNo);
    for (long i = 0; i < keyUniverseSize; i++) {
      final Tuple2<Long, IntType> generatedEvent = new Tuple2<>(i, intTypeNext);
      ctx.collectWithTimestamp(generatedEvent, eventSequenceNo);
    }
    ctx.emitWatermark(new Watermark(eventSequenceNo - watermarkTrailing));
  }
}
origin: dataArtisans/oscon

@Override
public void run(SourceContext<DataPoint<Long>> ctx) throws Exception {
 while (running) {
  synchronized (ctx.getCheckpointLock()) {
   ctx.collectWithTimestamp(new DataPoint<>(currentTimeMs, 0L), currentTimeMs);
   ctx.emitWatermark(new Watermark(currentTimeMs));
   currentTimeMs += periodMs;
  }
  timeSync();
 }
}
origin: apache/flink

@Override
public void run(SourceContext<Integer> ctx) throws Exception {
  ctx.collectWithTimestamp(1, 0);
  ctx.emitWatermark(new Watermark(0));
  ctx.collectWithTimestamp(2, 1);
  ctx.collectWithTimestamp(5, 2);
  ctx.emitWatermark(new Watermark(2));
  ctx.collectWithTimestamp(3, 3);
  ctx.collectWithTimestamp(4, 4);
}
origin: com.alibaba.blink/flink-examples-streaming

@Override
public void run(SourceContext<Tuple3<String, Long, Integer>> ctx) throws Exception {
  for (Tuple3<String, Long, Integer> value : input) {
    ctx.collectWithTimestamp(value, value.f1);
    ctx.emitWatermark(new Watermark(value.f1 - 1));
  }
  ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
}
origin: apache/flink

@Override
public void run(SourceContext<Tuple3<String, Long, Integer>> ctx) throws Exception {
  for (Tuple3<String, Long, Integer> value : input) {
    ctx.collectWithTimestamp(value, value.f1);
    ctx.emitWatermark(new Watermark(value.f1 - 1));
  }
  ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
}
origin: apache/bahir-flink

@Override
public void run(SourceContext<String> ctx) throws Exception {
  while (isRunning) {
    long timestamp = initialTimestamp + 1000 * number.get();
    ctx.collectWithTimestamp(WORDS[random.nextInt(WORDS.length)], timestamp);
    if (number.incrementAndGet() >= this.count) {
      cancel();
    }
  }
}
origin: apache/flink

/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a periodic watermark generator.
 */
private void emitRecordWithTimestampAndPeriodicWatermark(
    T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
  @SuppressWarnings("unchecked")
  final KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH> withWatermarksState =
      (KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH>) partitionState;
  // extract timestamp - this accesses/modifies the per-partition state inside the
  // watermark generator instance, so we need to lock the access on the
  // partition state. concurrent access can happen from the periodic emitter
  final long timestamp;
  //noinspection SynchronizationOnLocalVariableOrMethodParameter
  synchronized (withWatermarksState) {
    timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
  }
  // emit the record with timestamp, using the usual checkpoint lock to guarantee
  // atomicity of record emission and offset state update
  synchronized (checkpointLock) {
    sourceContext.collectWithTimestamp(record, timestamp);
    partitionState.setOffset(offset);
  }
}
org.apache.flink.streaming.api.functions.sourceSourceFunction$SourceContextcollectWithTimestamp

Javadoc

Emits one element from the source, and attaches the given timestamp. This method is relevant for programs using TimeCharacteristic#EventTime, where the sources assign timestamps themselves, rather than relying on a TimestampAssigneron the stream.

On certain time characteristics, this timestamp may be ignored or overwritten. This allows programs to switch between the different time characteristics and behaviors without changing the code of the source functions.

  • On TimeCharacteristic#ProcessingTime, the timestamp will be ignored, because processing time never works with element timestamps.
  • On TimeCharacteristic#IngestionTime, the timestamp is overwritten with the system's current time, to realize proper ingestion time semantics.
  • On TimeCharacteristic#EventTime, the timestamp will be used.

Popular methods of SourceFunction$SourceContext

  • collect
    Emits one element from the source, without attaching a timestamp. In most cases, this is the default
  • getCheckpointLock
    Returns the checkpoint lock. Please refer to the class-level comment in SourceFunction for details a
  • emitWatermark
    Emits the given Watermark. A Watermark of value t declares that no elements with a timestamp t' late
  • close
    This method is called by the system to shut down the context.
  • markAsTemporarilyIdle
    Marks the source to be temporarily idle. This tells the system that this source will temporarily sto

Popular in Java

  • Making http requests using okhttp
  • orElseThrow (Optional)
    Return the contained value, if present, otherwise throw an exception to be created by the provided s
  • setRequestProperty (URLConnection)
    Sets the general request property. If a property with the key already exists, overwrite its value wi
  • getContentResolver (Context)
  • Container (java.awt)
    A generic Abstract Window Toolkit(AWT) container object is a component that can contain other AWT co
  • Permission (java.security)
    Abstract class for representing access to a system resource. All permissions have a name (whose inte
  • Date (java.util)
    A specific moment in time, with millisecond precision. Values typically come from System#currentTime
  • LinkedHashMap (java.util)
    Hash table and linked list implementation of the Map interface, with predictable iteration order. Th
  • ThreadPoolExecutor (java.util.concurrent)
    An ExecutorService that executes each submitted task using one of possibly several pooled threads, n
  • JLabel (javax.swing)
Codota Logo
  • Products

    Search for Java codeSearch for JavaScript codeEnterprise
  • IDE Plugins

    IntelliJ IDEAWebStormAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogCodota Academy Plugin user guide Terms of usePrivacy policyJava Code IndexJavascript Code Index
Get Codota for your IDE now