@ProcessElement public void processElement(@Element Attribution attribution, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo pane, OutputReceiver<String> output) { StringBuilder builder = new StringBuilder(String.format( "%s: %s %s %-7s", Utils.formatWindow(window), attribution, Utils.formatTime(timestamp), pane.getTiming())); if (pane.getTiming() != PaneInfo.Timing.UNKNOWN) { builder.append(String.format(" index=%d", pane.getIndex())); if (pane.getNonSpeculativeIndex() > -1) builder.append(" onTimeIndex=" + pane.getNonSpeculativeIndex()); if (pane.isFirst()) builder.append(" isFirst"); if (pane.isLast()) builder.append(" isLast"); } output.output(builder.toString()); } }
public static PaneInfo createPane(boolean isFirst, boolean isLast, Timing timing) { checkArgument(isFirst, "Indices must be provided for non-first pane info."); return createPane(isFirst, isLast, timing, 0, timing == Timing.EARLY ? -1 : 0); }
private String paneInfoToString(PaneInfo paneInfo) { String paneString = String.format("pane-%d", paneInfo.getIndex()); if (paneInfo.getTiming() == Timing.LATE) { paneString = String.format("%s-late", paneString); } if (paneInfo.isLast()) { paneString = String.format("%s-last", paneString); } return paneString; }
@Override public Iterable<T> apply(Iterable<ValueInSingleWindow<T>> input) { List<T> outputs = new ArrayList<>(); for (ValueInSingleWindow<T> value : input) { if (!value.getPane().isFirst() || !value.getPane().isLast()) { throw site.wrap( String.format( "Expected elements to be produced by a trigger that fires at most once, but got " + "a value %s in a pane that is %s.", value, value.getPane().isFirst() ? "not the last pane" : "not the first pane")); } outputs.add(value.getValue()); } return outputs; } }
private <W> PaneInfo describePane( Object key, Instant windowMaxTimestamp, PaneInfo previousPane, boolean isFinal) { boolean isFirst = previousPane == null; Timing previousTiming = isFirst ? null : previousPane.getTiming(); long index = isFirst ? 0 : previousPane.getIndex() + 1; long nonSpeculativeIndex = isFirst ? 0 : previousPane.getNonSpeculativeIndex() + 1; Instant outputWM = timerInternals.currentOutputWatermarkTime(); Instant inputWM = timerInternals.currentInputWatermarkTime(); break; checkState(!previousPane.isLast(), "Last pane was not last after all."); return PaneInfo.createPane(isFirst, isFinal, timing, index, nonSpeculativeIndex);
/** * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to * end-of-time. */ @Test public void fireNonEmptyOnDrainInGlobalWindow() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(new GlobalWindows()) .withTrigger(Repeatedly.forever(AfterPane.elementCountAtLeast(3))) .withMode(AccumulationMode.DISCARDING_FIRED_PANES)); tester.advanceInputWatermark(new Instant(0)); final int n = 20; for (int i = 0; i < n; i++) { tester.injectElements(TimestampedValue.of(i, new Instant(i))); } List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertEquals(n / 3, output.size()); for (int i = 0; i < output.size(); i++) { assertEquals(Timing.EARLY, output.get(i).getPane().getTiming()); assertEquals(i, output.get(i).getPane().getIndex()); assertEquals(3, Iterables.size(output.get(i).getValue())); } tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); output = tester.extractOutput(); assertEquals(1, output.size()); assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming()); assertEquals(n / 3, output.get(0).getPane().getIndex()); assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue())); }
@ProcessElement public void processElement(ProcessContext c, BoundedWindow window) throws Exception { String[] values = c.element().getValue().split(",", -1); TableRow row = new TableRow() .set("trigger_type", triggerType) .set("freeway", c.element().getKey()) .set("total_flow", Integer.parseInt(values[0])) .set("number_of_records", Long.parseLong(values[1])) .set("window", window.toString()) .set("isFirst", c.pane().isFirst()) .set("isLast", c.pane().isLast()) .set("timing", c.pane().getTiming().toString()) .set("event_time", c.timestamp().toString()) .set("processing_time", Instant.now().toString()); c.output(row); } }
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { String filenamePrefix = outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), ""); IntervalWindow interval = (IntervalWindow) window; String windowStr = String.format("%s-%s", interval.start().toString(), interval.end().toString()); String filename = String.format( "%s-%s-%s-of-%s-pane-%s%s%s.avro", filenamePrefix, windowStr, shardNumber, numShards, paneInfo.getIndex(), paneInfo.isLast() ? "-last" : "", outputFileHints.getSuggestedFilenameSuffix()); return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE); }
@Override public Iterable<T> apply(Iterable<ValueInSingleWindow<T>> input) { List<T> outputs = new ArrayList<>(); for (ValueInSingleWindow<T> value : input) { if (value.getPane().getTiming() == PaneInfo.Timing.EARLY) { outputs.add(value.getValue()); } } return outputs; } }
/** * Set the value of the {@link PCollectionView} in the {@link BoundedWindow} to be based on the * specified values, if the values are part of a later pane than currently exist within the {@link * PCollectionViewWindow}. */ private void updatePCollectionViewWindowValues( PCollectionView<?> view, BoundedWindow window, Collection<WindowedValue<?>> windowValues) { PCollectionViewWindow<?> windowedView = PCollectionViewWindow.of(view, window); AtomicReference<Iterable<? extends WindowedValue<?>>> contents = viewByWindows.getUnchecked(windowedView); if (contents.compareAndSet(null, windowValues)) { // the value had never been set, so we set it and are done. return; } PaneInfo newPane = windowValues.iterator().next().getPane(); Iterable<? extends WindowedValue<?>> existingValues; long existingPane; do { existingValues = contents.get(); existingPane = Iterables.isEmpty(existingValues) ? -1L : existingValues.iterator().next().getPane().getIndex(); } while (newPane.getIndex() > existingPane && !contents.compareAndSet(existingValues, windowValues)); }
@Override public Iterable<T> apply(Iterable<ValueInSingleWindow<T>> input) { List<T> outputs = new ArrayList<>(); for (ValueInSingleWindow<T> value : input) { if (value.getPane().isLast()) { outputs.add(value.getValue()); } } return outputs; } }
"PaneInfo encoding should remain the same.", 0x0, PaneInfo.createPane(false, false, Timing.EARLY, 1, -1).getEncodedByte()); assertEquals( "PaneInfo encoding should remain the same.", 0x1, PaneInfo.createPane(true, false, Timing.EARLY).getEncodedByte()); assertEquals( "PaneInfo encoding should remain the same.", 0x3, PaneInfo.createPane(true, true, Timing.EARLY).getEncodedByte()); assertEquals( "PaneInfo encoding should remain the same.", 0x7, PaneInfo.createPane(true, true, Timing.ON_TIME).getEncodedByte()); assertEquals( "PaneInfo encoding should remain the same.", 0xB, PaneInfo.createPane(true, true, Timing.LATE).getEncodedByte()); assertEquals( "PaneInfo encoding should remain the same.", 0xF, PaneInfo.createPane(true, true, Timing.UNKNOWN).getEncodedByte());
@ProcessElement public void processElement( @Element KV<T, Void> element, PaneInfo pane, OutputReceiver<T> receiver) { if (pane.isFirst()) { // Only output the key if it's the first time it's been seen. receiver.output(element.getKey()); } } }));
assertEquals((n + 3) / 4, output.size()); for (int i = 0; i < output.size(); i++) { assertEquals(Timing.EARLY, output.get(i).getPane().getTiming()); assertEquals(i, output.get(i).getPane().getIndex()); assertEquals(4, Iterables.size(output.get(i).getValue())); assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming()); assertEquals((n + 3) / 4, output.get(0).getPane().getIndex()); assertEquals(0, Iterables.size(output.get(0).getValue()));
@Override public Iterable<T> apply(Iterable<ValueInSingleWindow<T>> input) { List<T> outputs = new ArrayList<>(); for (ValueInSingleWindow<T> value : input) { if (value.getPane().getTiming() != PaneInfo.Timing.LATE) { outputs.add(value.getValue()); } } return outputs; } }
private PendingJobData startWriteRename( TableDestination finalTableDestination, Iterable<String> tempTableNames, ProcessContext c) throws Exception { WriteDisposition writeDisposition = (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND; CreateDisposition createDisposition = (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER; List<TableReference> tempTables = StreamSupport.stream(tempTableNames.spliterator(), false) .map(table -> BigQueryHelpers.fromJsonString(table, TableReference.class)) .collect(Collectors.toList()); ; // Make sure each destination table gets a unique job id. String jobIdPrefix = BigQueryHelpers.createJobId( c.sideInput(jobIdToken), finalTableDestination, -1, c.pane().getIndex()); BigQueryHelpers.PendingJob retryJob = startCopy( bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)), bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)), jobIdPrefix, finalTableDestination.getTableReference(), tempTables, writeDisposition, createDisposition); return new PendingJobData(retryJob, finalTableDestination, tempTables); }
@ProcessElement public void processElement(ProcessContext c) { if (c.pane().isLast()) { c.output(c.element()); } } }
@Test public void testInterned() throws Exception { assertSame( PaneInfo.createPane(true, true, Timing.EARLY), PaneInfo.createPane(true, true, Timing.EARLY)); }
@ProcessElement public void processElement(@Element KV<String, Integer> kv, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo pane, OutputReceiver<String> output) { StringBuilder builder = new StringBuilder(String.format( "%s: %s:%-2d %s %-7s index=%d", Utils.formatWindow(window), kv.getKey(), kv.getValue(), Utils.formatTime(timestamp), pane.getTiming(), pane.getIndex())); if (pane.getNonSpeculativeIndex() > -1) builder.append(" onTimeIndex=" + pane.getNonSpeculativeIndex()); if (pane.isFirst()) builder.append(" isFirst"); if (pane.isLast()) builder.append(" isLast"); output.output(builder.toString()); } }