unbounded.withMaxReadTime(getMaxReadTime()).withMaxNumRecords(getMaxNumRecords());
unbounded.withMaxReadTime(getMaxReadTime()).withMaxNumRecords(getMaxNumRecords());
readUnbounded.withMaxReadTime(getMaxReadTime()); if (isRangeUnbounded) { return input.apply(withMaxReadTime);
@Override public PCollection<T> expand(PBegin input) { checkArgument(getConnectionFactory() != null, "withConnectionFactory() is required"); checkArgument( getQueue() != null || getTopic() != null, "Either withQueue() or withTopic() is required"); checkArgument( getQueue() == null || getTopic() == null, "withQueue() and withTopic() are exclusive"); checkArgument(getMessageMapper() != null, "withMessageMapper() is required"); checkArgument(getCoder() != null, "withCoder() is required"); // handles unbounded source to bounded conversion if maxNumRecords is set. Unbounded<T> unbounded = org.apache.beam.sdk.io.Read.from(createSource()); PTransform<PBegin, PCollection<T>> transform = unbounded; if (getMaxNumRecords() < Long.MAX_VALUE || getMaxReadTime() != null) { transform = unbounded.withMaxReadTime(getMaxReadTime()).withMaxNumRecords(getMaxNumRecords()); } return input.getPipeline().apply(transform); }
@Override public PCollection<KinesisRecord> expand(PBegin input) { Unbounded<KinesisRecord> unbounded = org.apache.beam.sdk.io.Read.from( new KinesisSource( getAWSClientsProvider(), getStreamName(), getInitialPosition(), getUpToDateThreshold(), getRequestRecordsLimit())); PTransform<PBegin, PCollection<KinesisRecord>> transform = unbounded; if (getMaxNumRecords() < Long.MAX_VALUE || getMaxReadTime() != null) { transform = unbounded.withMaxReadTime(getMaxReadTime()).withMaxNumRecords(getMaxNumRecords()); } return input.apply(transform); } }
@Override public PCollection<Message> expand(PBegin input) { org.apache.beam.sdk.io.Read.Unbounded<Message> unbounded = org.apache.beam.sdk.io.Read.from( new SqsUnboundedSource( this, new SqsConfiguration(input.getPipeline().getOptions().as(AwsOptions.class)))); PTransform<PBegin, PCollection<Message>> transform = unbounded; if (maxNumRecords() < Long.MAX_VALUE || maxReadTime() != null) { transform = unbounded.withMaxReadTime(maxReadTime()).withMaxNumRecords(maxNumRecords()); } return input.getPipeline().apply(transform); } }
private void test(boolean dedup, boolean timeBound) throws Exception { TestCountingSource source = new TestCountingSource(Integer.MAX_VALUE).withoutSplitting(); if (dedup) { source = source.withDedup(); } PCollection<KV<Integer, Integer>> output = timeBound ? p.apply(Read.from(source).withMaxReadTime(Duration.millis(200))) : p.apply(Read.from(source).withMaxNumRecords(NUM_RECORDS)); // Because some of the NUM_RECORDS elements read are dupes, the final output // will only have output from 0 to n where n < NUM_RECORDS. PAssert.that(output).satisfies(new Checker(dedup, timeBound)); p.run(); }
@Override public PCollection<Message> expand(PBegin input) { checkArgument(addresses() != null, "withAddresses() is required"); org.apache.beam.sdk.io.Read.Unbounded<Message> unbounded = org.apache.beam.sdk.io.Read.from(new UnboundedAmqpSource(this)); PTransform<PBegin, PCollection<Message>> transform = unbounded; if (maxNumRecords() < Long.MAX_VALUE || maxReadTime() != null) { transform = unbounded.withMaxReadTime(maxReadTime()).withMaxNumRecords(maxNumRecords()); } return input.getPipeline().apply(transform); } }