@Override protected Source.Reader<OutputT> createReader(PipelineOptions options) throws IOException { return source.createReader(options, null); } }
@Override public void prepare() { try { reader = unboundedSource.createReader(null, null); } catch (final Exception e) { throw new RuntimeException(e); } }
localSplitSources.add(restored.getKey()); localReaders.add( restored.getKey().createReader(serializedOptions.get(), restored.getValue())); UnboundedSource<OutputT, CheckpointMarkT> source = splitSources.get(i); UnboundedSource.UnboundedReader<OutputT> reader = source.createReader(serializedOptions.get(), null); localSplitSources.add(source); localReaders.add(reader);
@Override public void setup(OperatorContext context) { this.traceTuples = ApexStreamTuple.Logging.isDebugEnabled( pipelineOptions.get().as(ApexPipelineOptions.class), this); try { reader = source.createReader(this.pipelineOptions.get(), null); available = reader.start(); } catch (IOException e) { throw new RuntimeException(e); } }
stateForCheckpoint.get()) { localSplitSources.add(restored.getKey()); localReaders.add(restored.getKey().createReader( serializedOptions.get(), restored.getValue())); splitSources.get(i); UnboundedSource.UnboundedReader<OutputT> reader = source.createReader(serializedOptions.get(), null); localSplitSources.add(source); localReaders.add(reader);
localSplitSources.add(restored.getKey()); localReaders.add( restored.getKey().createReader(serializedOptions.get(), restored.getValue())); UnboundedSource<OutputT, CheckpointMarkT> source = splitSources.get(i); UnboundedSource.UnboundedReader<OutputT> reader = source.createReader(serializedOptions.get(), null); localSplitSources.add(source); localReaders.add(reader);
private UnboundedReader<OutputT> getReader(UnboundedSourceShard<OutputT, CheckpointMarkT> shard) throws IOException { UnboundedReader<OutputT> existing = shard.getExistingReader(); if (existing == null) { CheckpointMarkT checkpoint = shard.getCheckpoint(); if (checkpoint != null) { checkpoint = CoderUtils.clone(shard.getSource().getCheckpointMarkCoder(), checkpoint); } return shard.getSource().createReader(options, checkpoint); } else { return existing; } }
@Test public void generatesInitialSplits() throws Exception { when(context.createRootBundle()).thenAnswer(invocation -> bundleFactory.createRootBundle()); int numSplits = 5; Collection<CommittedBundle<?>> initialInputs = new UnboundedReadEvaluatorFactory.InputProvider(context, options) .getInitialInputs(graph.getProducer(longs), numSplits); // CountingSource.unbounded has very good splitting behavior assertThat(initialInputs, hasSize(numSplits)); int readPerSplit = 100; int totalSize = numSplits * readPerSplit; Set<Long> expectedOutputs = ContiguousSet.create(Range.closedOpen(0L, (long) totalSize), DiscreteDomain.longs()); Collection<Long> readItems = new ArrayList<>(totalSize); for (CommittedBundle<?> initialInput : initialInputs) { CommittedBundle<UnboundedSourceShard<Long, ?>> shardBundle = (CommittedBundle<UnboundedSourceShard<Long, ?>>) initialInput; WindowedValue<UnboundedSourceShard<Long, ?>> shard = Iterables.getOnlyElement(shardBundle.getElements()); assertThat(shard.getTimestamp(), equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE)); assertThat(shard.getWindows(), Matchers.contains(GlobalWindow.INSTANCE)); UnboundedSource<Long, ?> shardSource = shard.getValue().getSource(); readItems.addAll( SourceTestUtils.readNItemsFromUnstartedReader( shardSource.createReader( PipelineOptionsFactory.create(), null /* No starting checkpoint */), readPerSplit)); } assertThat(readItems, containsInAnyOrder(expectedOutputs.toArray(new Long[0]))); }
@ProcessElement public void process( @Element Shard<T> shard, OutputReceiver<ValueWithRecordId<T>> out, PipelineOptions options) throws Exception { Instant endTime = shard.getMaxReadTime() == null ? null : Instant.now().plus(shard.getMaxReadTime()); if (shard.getMaxNumRecords() <= 0 || (shard.getMaxReadTime() != null && shard.getMaxReadTime().getMillis() == 0)) { return; } try (UnboundedSource.UnboundedReader<T> reader = SerializableUtils.clone(shard.getSource()).createReader(options, null)) { for (long i = 0L; i < shard.getMaxNumRecords(); ++i) { boolean available = (i == 0) ? reader.start() : reader.advance(); if (!available && !advanceWithBackoff(reader, endTime)) { break; } out.outputWithTimestamp( new ValueWithRecordId<T>(reader.getCurrent(), reader.getCurrentRecordId()), reader.getCurrentTimestamp()); } reader.getCheckpointMark().finalizeCheckpoint(); } }
.get(0); UnboundedReader<KafkaRecord<Integer, Long>> reader = source.createReader(null, null); .get(0); reader = source.createReader(null, mark);
@Test public void testUnboundedSourceCheckpointMark() throws Exception { UnboundedSource<Long, CounterMark> source = CountingSource.unboundedWithTimestampFn(new ValueAsTimestampFn()); UnboundedReader<Long> reader = source.createReader(null, null); final long numToSkip = 3; assertTrue(reader.start()); // Advance the source numToSkip elements and manually save state. for (long l = 0; l < numToSkip; ++l) { reader.advance(); } // Confirm that we get the expected element in sequence before checkpointing. assertEquals(numToSkip, (long) reader.getCurrent()); assertEquals(numToSkip, reader.getCurrentTimestamp().getMillis()); // Checkpoint and restart, and confirm that the source continues correctly. CounterMark mark = CoderUtils.clone(source.getCheckpointMarkCoder(), (CounterMark) reader.getCheckpointMark()); reader = source.createReader(null, mark); assertTrue(reader.start()); // Confirm that we get the next element in sequence. assertEquals(numToSkip + 1, (long) reader.getCurrent()); assertEquals(numToSkip + 1, reader.getCurrentTimestamp().getMillis()); } }
.get(0); UnboundedReader<KafkaRecord<Integer, Long>> reader = source.createReader(null, null); final int numToSkip = 20; // one from each partition. CoderUtils.clone( source.getCheckpointMarkCoder(), (KafkaCheckpointMark) reader.getCheckpointMark()); reader = source.createReader(null, mark);