@Override public final PCollection<T> expand(PBegin input) { source.validate(); return Pipeline.applyTransform(input, new UnboundedReadFromBoundedSource<>(source)) .setIsBoundedInternal(IsBounded.BOUNDED); } }
private static PCollection<Integer> makeUnboundedCollection(Pipeline pipeline) { return pipeline .apply("unbounded", Create.of(1, 2, 3)) .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED); }
private static PCollection<Integer> makeBoundedCollection(Pipeline pipeline) { return pipeline .apply("bounded", Create.of(1, 2, 3)) .setIsBoundedInternal(PCollection.IsBounded.BOUNDED); }
users = users.setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
@Test @Category(NeedsRunner.class) public void testUnboundedNeedsWindowed() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Must use windowed writes when applying WriteFiles to an unbounded PCollection"); SimpleSink<Void> sink = makeSimpleSink(); p.apply(Create.of("foo")).setIsBoundedInternal(IsBounded.UNBOUNDED).apply(WriteFiles.to(sink)); p.run(); }
@Test @Category(NeedsRunner.class) public void testUnboundedWritesNeedSharding() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "When applying WriteFiles to an unbounded PCollection, " + "must specify number of output shards explicitly"); SimpleSink<Void> sink = makeSimpleSink(); p.apply(Create.of("foo")) .setIsBoundedInternal(IsBounded.UNBOUNDED) .apply(WriteFiles.to(sink).withWindowedWrites()); p.run(); }
input = input.setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
WriteFilesResult<Integer> res; if (!bounded) { input.setIsBoundedInternal(IsBounded.UNBOUNDED); input = input.apply(Window.into(FixedWindows.of(Duration.standardDays(1)))); res = input.apply(writeFiles.withWindowedWrites());
generateStreamWithBoundedDisorder("main", base, duration, numMainElements, lateness); if (mainWindowFn == null) { input.setIsBoundedInternal(PCollection.IsBounded.BOUNDED); } else { input = generateStreamWithBoundedDisorder("signal", base, duration, numSignalElements, lateness); if (signalWindowFn == null) { signal.setIsBoundedInternal(PCollection.IsBounded.BOUNDED); } else { signal =
new TableRow().set("name", "d").set("number", 4)) .withCoder(TableRowJsonCoder.of())) .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED) .apply( BigQueryIO.writeTableRows()