@Override public IntervalWindow assignWindow(Instant timestamp) { DateTime datetime = new DateTime(timestamp, timeZone); int dayOffset = Days.daysBetween(startDate, datetime).getDays() / number * number; DateTime begin = startDate.plusDays(dayOffset); DateTime end = begin.plusDays(number); return new IntervalWindow(begin.toInstant(), end.toInstant()); }
@Override public Coder<IntervalWindow> windowCoder() { return IntervalWindow.getCoder(); }
private String windowToString(BoundedWindow window) { if (window instanceof GlobalWindow) { return "GlobalWindow"; } if (window instanceof IntervalWindow) { IntervalWindow iw = (IntervalWindow) window; return String.format("%s-%s", iw.start().toString(), iw.end().toString()); } return window.toString(); }
@Override public void mergeWindows(MergeContext c) throws Exception { IntervalWindow last = null; for (IntervalWindow w : sortedWindows(c)) { if (last != null && w.start().isBefore(last.end())) { c.merge(Arrays.asList(last), new IntervalWindow(last.start(), w.start())); } last = w; } }
@ProcessElement public void processElement(ProcessContext c) throws Exception { assertThat( c.timestamp(), equalTo( new IntervalWindow( new Instant(0), new Instant(0).plus(Duration.standardMinutes(10))) .maxTimestamp())); } }));
@Test public void testCompareByNamespace() { Instant timestamp = new Instant(100); IntervalWindow firstWindow = new IntervalWindow(new Instant(0), timestamp); IntervalWindow secondWindow = new IntervalWindow(timestamp, new Instant(200)); Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder(); StateNamespace firstWindowNs = StateNamespaces.window(windowCoder, firstWindow); StateNamespace secondWindowNs = StateNamespaces.window(windowCoder, secondWindow); TimerData secondEventTime = TimerData.of(firstWindowNs, timestamp, TimeDomain.EVENT_TIME); TimerData thirdEventTime = TimerData.of(secondWindowNs, timestamp, TimeDomain.EVENT_TIME); assertThat(secondEventTime, lessThan(thirdEventTime)); }
IntervalWindow window = new IntervalWindow(base, base.plus(Duration.standardSeconds(5))); if (!window.start().isAfter(tv.getTimestamp()) && !tv.getTimestamp().isAfter(window.maxTimestamp())) { expected.add(tv);
/** * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs * correctly per window and per key. */ @Test public void testScoreUpdatesPerWindow() { TestStream<KV<String, GameActionInfo>> createEvents = TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class))) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.RED_ONE, 50, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 50, Duration.standardMinutes(2)), event(TestUser.RED_ONE, 50, Duration.standardMinutes(3)), event(TestUser.RED_ONE, 60, Duration.standardMinutes(6)), event(TestUser.RED_TWO, 60, Duration.standardMinutes(7))) .advanceWatermarkToInfinity(); Duration teamWindowDuration = Duration.standardMinutes(5); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents) .apply(Window.<KV<String, GameActionInfo>>into(FixedWindows.of(teamWindowDuration))) .apply(ParDo.of(new UpdateTeamScoreFn(100))); String redTeam = TestUser.RED_ONE.getTeam(); String blueTeam = TestUser.BLUE_ONE.getTeam(); IntervalWindow window1 = new IntervalWindow(baseTime, teamWindowDuration); IntervalWindow window2 = new IntervalWindow(window1.end(), teamWindowDuration); PAssert.that(teamScores).inWindow(window1).containsInAnyOrder(KV.of(redTeam, 100)); PAssert.that(teamScores).inWindow(window2).containsInAnyOrder(KV.of(redTeam, 120)); p.run().waitUntilFinish(); }
/** * Create a map of information that describes how to write pipeline output to BigQuery. This map * is used to write information about mean user session time. */ protected static Map<String, WriteWindowedToBigQuery.FieldInfo<Double>> configureSessionWindowWrite() { Map<String, WriteWindowedToBigQuery.FieldInfo<Double>> tableConfigure = new HashMap<>(); tableConfigure.put( "window_start", new WriteWindowedToBigQuery.FieldInfo<>( "STRING", (c, w) -> { IntervalWindow window = (IntervalWindow) w; return GameConstants.DATE_TIME_FORMATTER.print(window.start()); })); tableConfigure.put( "mean_duration", new WriteWindowedToBigQuery.FieldInfo<>("FLOAT", (c, w) -> c.element())); return tableConfigure; }
IntervalWindow nextWindow = (IntervalWindow) Iterables.getOnlyElement(nextValue.getWindows()); if (currentWindow.intersects(nextWindow)) { currentWindow = currentWindow.span(nextWindow); } else {
outputTimestamp, greaterThan((ReadableInstant) earlierEndingWindow.maxTimestamp()));
/** * Resolves any date variables which exist in the output directory path. This allows for the * dynamically changing of the output location based on the window end time. * * @return The new output directory with all variables resolved. */ private ResourceId resolveWithDateTemplates( ValueProvider<String> outputDirectoryStr, BoundedWindow window) { ResourceId outputDirectory = FileSystems.matchNewResource(outputDirectoryStr.get(), true); if (window instanceof IntervalWindow) { IntervalWindow intervalWindow = (IntervalWindow) window; DateTime time = intervalWindow.end().toDateTime(); String outputPath = outputDirectory.toString(); outputPath = outputPath.replace("YYYY", YEAR.print(time)); outputPath = outputPath.replace("MM", MONTH.print(time)); outputPath = outputPath.replace("DD", DAY.print(time)); outputPath = outputPath.replace("HH", HOUR.print(time)); outputDirectory = FileSystems.matchNewResource(outputPath, true); } return outputDirectory; } }
public boolean intersects(IntervalWindow window) { return union == null || union.intersects(window); }
protected static IntervalWindow merge(IntervalWindow union, IntervalWindow window) { return union == null ? window : union.span(window); }
@Test public void testTimerForEndOfWindow() throws Exception { tester = TriggerStateMachineTester.forTrigger( AfterWatermarkStateMachine.pastEndOfWindow(), FixedWindows.of(Duration.millis(100))); assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), nullValue()); injectElements(1); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100)); assertThat(tester.getNextTimer(TimeDomain.EVENT_TIME), equalTo(window.maxTimestamp())); }
@Test public void testDecodeEncodeEqual() throws Exception { Instant now = Instant.now(); ValueInSingleWindow<String> value = ValueInSingleWindow.of( "foo", now, new IntervalWindow(now, now.plus(Duration.standardSeconds(10))), PaneInfo.NO_FIRING); CoderProperties.coderDecodeEncodeEqual( ValueInSingleWindow.Coder.of(StringUtf8Coder.of(), IntervalWindow.getCoder()), value); }
for (int window = 0; window < 2; window++) { Instant windowStart = new Instant(0).plus(Duration.standardMinutes(window)); IntervalWindow iw = new IntervalWindow(windowStart, Duration.standardMinutes(1)); String baseAndWindow = baseFilename + "-" + iw.start() + "-" + iw.end(); switch (method) { case AVROIO_WRITE:
/** * Create a map of information that describes how to write pipeline output to text. This map is * passed to the {@link WriteToText} constructor to write team score sums and includes information * about window start time. */ protected static Map<String, WriteToText.FieldFn<KV<String, Integer>>> configureOutput() { Map<String, WriteToText.FieldFn<KV<String, Integer>>> config = new HashMap<>(); config.put("team", (c, w) -> c.element().getKey()); config.put("total_score", (c, w) -> c.element().getValue()); config.put( "window_start", (c, w) -> { IntervalWindow window = (IntervalWindow) w; return GameConstants.DATE_TIME_FORMATTER.print(window.start()); }); return config; }
WindowedValue<KV<K, InputT>> nextValue = elements.get(i); IntervalWindow nextWindow = (IntervalWindow) Iterables.getOnlyElement(nextValue.getWindows()); if (currentWindow.intersects(nextWindow)) { currentWindow = currentWindow.span(nextWindow); } else {
/** * Ensures that later sliding windows have an output time that is past the end of earlier windows. * * <p>If this is the earliest sliding window containing {@code inputTimestamp}, that's fine. * Otherwise, we pick the earliest time that doesn't overlap with earlier windows. */ @Experimental(Kind.OUTPUT_TIME) @Override public Instant getOutputTime(Instant inputTimestamp, IntervalWindow window) { Instant startOfLastSegment = window.maxTimestamp().minus(period); return startOfLastSegment.isBefore(inputTimestamp) ? inputTimestamp : startOfLastSegment.plus(1); }