/** * Creates a new data stream that contains a sequence of numbers. This is a parallel source, * if you manually set the parallelism to {@code 1} * (using {@link org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#setParallelism(int)}) * the generated sequence of elements is in order. * * @param from * The number to start at (inclusive) * @param to * The number to stop at (inclusive) * @return A data stream, containing all number in the [from, to] interval */ public DataStreamSource<Long> generateSequence(long from, long to) { if (from > to) { throw new IllegalArgumentException("Start of sequence must not be greater than the end"); } return addSource(new StatefulSequenceSource(from, to), "Sequence Source"); }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "stateful-sequence-source-state", LongSerializer.INSTANCE ) ); this.valuesToEmit = new ArrayDeque<>(); if (context.isRestored()) { // upon restoring for (Long v : this.checkpointedState.get()) { this.valuesToEmit.add(v); } } else { // the first time the job is executed final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks(); final int taskIdx = getRuntimeContext().getIndexOfThisSubtask(); final long congruence = start + taskIdx; long totalNoOfElements = Math.abs(end - start + 1); final int baseSize = safeDivide(totalNoOfElements, stepSize); final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize; for (long collected = 0; collected < toCollect; collected++) { this.valuesToEmit.add(collected * stepSize + congruence); } } }
@Override public void run() { try { source3.run(new BlockingSourceContext("3", latchToTrigger3, latchToWait3, outputCollector, 3)); } catch (Throwable t) { t.printStackTrace(); error[2] = t; } } };
final OneShotLatch latchToWait2 = new OneShotLatch(); final StatefulSequenceSource source1 = new StatefulSequenceSource(initElement, maxElement); StreamSource<Long, StatefulSequenceSource> src1 = new StreamSource<>(source1); testHarness1.open(); final StatefulSequenceSource source2 = new StatefulSequenceSource(initElement, maxElement); StreamSource<Long, StatefulSequenceSource> src2 = new StreamSource<>(source2); ); final StatefulSequenceSource source3 = new StatefulSequenceSource(initElement, maxElement); StreamSource<Long, StatefulSequenceSource> src3 = new StreamSource<>(source3);
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "stateful-sequence-source-state", LongSerializer.INSTANCE ) ); this.valuesToEmit = new ArrayDeque<>(); if (context.isRestored()) { // upon restoring for (Long v : this.checkpointedState.get()) { this.valuesToEmit.add(v); } } else { // the first time the job is executed final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks(); final int taskIdx = getRuntimeContext().getIndexOfThisSubtask(); final long congruence = start + taskIdx; long totalNoOfElements = Math.abs(end - start + 1); final int baseSize = safeDivide(totalNoOfElements, stepSize); final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize; for (long collected = 0; collected < toCollect; collected++) { this.valuesToEmit.add(collected * stepSize + congruence); } } }
@Override public void run() { try { source1.run(new BlockingSourceContext("1", latchToTrigger1, latchToWait1, outputCollector, 21)); } catch (Throwable t) { t.printStackTrace(); error[0] = t; } } };
/** * Creates a new data stream that contains a sequence of numbers. This is a parallel source, * if you manually set the parallelism to {@code 1} * (using {@link org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#setParallelism(int)}) * the generated sequence of elements is in order. * * @param from * The number to start at (inclusive) * @param to * The number to stop at (inclusive) * @return A data stream, containing all number in the [from, to] interval */ public DataStreamSource<Long> generateSequence(long from, long to) { if (from > to) { throw new IllegalArgumentException("Start of sequence must not be greater than the end"); } return addSource(new StatefulSequenceSource(from, to), "Sequence Source"); }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "stateful-sequence-source-state", LongSerializer.INSTANCE ) ); this.valuesToEmit = new ArrayDeque<>(); if (context.isRestored()) { // upon restoring for (Long v : this.checkpointedState.get()) { this.valuesToEmit.add(v); } } else { // the first time the job is executed final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks(); final int taskIdx = getRuntimeContext().getIndexOfThisSubtask(); final long congruence = start + taskIdx; long totalNoOfElements = Math.abs(end - start + 1); final int baseSize = safeDivide(totalNoOfElements, stepSize); final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize; for (long collected = 0; collected < toCollect; collected++) { this.valuesToEmit.add(collected * stepSize + congruence); } } }
@Override public void run() { try { source2.run(new BlockingSourceContext("2", latchToTrigger2, latchToWait2, outputCollector, 32)); } catch (Throwable t) { t.printStackTrace(); error[1] = t; } } };
/** * Creates a new data stream that contains a sequence of numbers. This is a parallel source, * if you manually set the parallelism to {@code 1} * (using {@link org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#setParallelism(int)}) * the generated sequence of elements is in order. * * @param from * The number to start at (inclusive) * @param to * The number to stop at (inclusive) * @return A data stream, containing all number in the [from, to] interval */ public DataStreamSource<Long> generateSequence(long from, long to) { if (from > to) { throw new IllegalArgumentException("Start of sequence must not be greater than the end"); } return addSource(new StatefulSequenceSource(from, to), "Sequence Source"); }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "stateful-sequence-source-state", LongSerializer.INSTANCE ) ); this.valuesToEmit = new ArrayDeque<>(); if (context.isRestored()) { // upon restoring for (Long v : this.checkpointedState.get()) { this.valuesToEmit.add(v); } } else { // the first time the job is executed final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks(); final int taskIdx = getRuntimeContext().getIndexOfThisSubtask(); final long congruence = start + taskIdx; long totalNoOfElements = Math.abs(end - start + 1); final int baseSize = safeDivide(totalNoOfElements, stepSize); final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize; for (long collected = 0; collected < toCollect; collected++) { this.valuesToEmit.add(collected * stepSize + congruence); } } }
/** * Creates a new data stream that contains a sequence of numbers. This is a parallel source, * if you manually set the parallelism to {@code 1} * (using {@link org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#setParallelism(int)}) * the generated sequence of elements is in order. * * @param from * The number to start at (inclusive) * @param to * The number to stop at (inclusive) * @return A data stream, containing all number in the [from, to] interval */ public DataStreamSource<Long> generateSequence(long from, long to) { if (from > to) { throw new IllegalArgumentException("Start of sequence must not be greater than the end"); } return addSource(new StatefulSequenceSource(from, to), "Sequence Source"); }
/** * Creates a new data stream that contains a sequence of numbers. This is a parallel source, * if you manually set the parallelism to {@code 1} * (using {@link org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#setParallelism(int)}) * the generated sequence of elements is in order. * * @param from * The number to start at (inclusive) * @param to * The number to stop at (inclusive) * @return A data stream, containing all number in the [from, to] interval */ public DataStreamSource<Long> generateSequence(long from, long to) { if (from > to) { throw new IllegalArgumentException("Start of sequence must not be greater than the end"); } return addSource(new StatefulSequenceSource(from, to), "Sequence Source"); }