public String filenamePrefixForWindow(IntervalWindow window) { String filePrefix = prefix.isDirectory() ? "" : prefix.getFilename(); return String.format( "%s-%s-%s", filePrefix, formatter.print(window.start()), formatter.print(window.end())); }
private String windowToString(BoundedWindow window) { if (window instanceof GlobalWindow) { return "GlobalWindow"; } if (window instanceof IntervalWindow) { IntervalWindow iw = (IntervalWindow) window; return String.format("%s-%s", iw.start().toString(), iw.end().toString()); } return window.toString(); }
public String filenamePrefixForWindow(IntervalWindow window) { String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); return String.format( "%s-%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end())); }
/** * @param window interval window * @return file name prefix. */ String filenamePrefixForWindow(final IntervalWindow window) { final String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); return String.format( "%s-%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end())); } @Override
/** * Resolves any date variables which exist in the output directory path. This allows for the * dynamically changing of the output location based on the window end time. * * @return The new output directory with all variables resolved. */ private ResourceId resolveWithDateTemplates( ValueProvider<String> outputDirectoryStr, BoundedWindow window) { ResourceId outputDirectory = FileSystems.matchNewResource(outputDirectoryStr.get(), true); if (window instanceof IntervalWindow) { IntervalWindow intervalWindow = (IntervalWindow) window; DateTime time = intervalWindow.end().toDateTime(); String outputPath = outputDirectory.toString(); outputPath = outputPath.replace("YYYY", YEAR.print(time)); outputPath = outputPath.replace("MM", MONTH.print(time)); outputPath = outputPath.replace("DD", DAY.print(time)); outputPath = outputPath.replace("HH", HOUR.print(time)); outputDirectory = FileSystems.matchNewResource(outputPath, true); } return outputDirectory; } }
window.getClass().getSimpleName()); IntervalWindow iw = (IntervalWindow) window; res.append(iw.start().toString()).append("-").append(iw.end().toString());
@ProcessElement public void processElement(ProcessContext c, BoundedWindow window) { IntervalWindow w = (IntervalWindow) window; int duration = new Duration(w.start(), w.end()).toPeriod().toStandardMinutes().getMinutes(); c.output(duration); } }
public static String formatWindow(BoundedWindow window) { if (window instanceof GlobalWindow) { return "[global window]"; } else if (window instanceof IntervalWindow) { IntervalWindow interval = (IntervalWindow) window; return "[" + formatTime(interval.start()) + ", " + formatTime(interval.end()) + ")"; } else { return "..., " + formatTime(window.maxTimestamp()) + "]"; } }
public String filenamePrefixForWindow(IntervalWindow window) { String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); return String.format( "%s%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end())); }
@Override public void mergeWindows(MergeContext c) throws Exception { IntervalWindow last = null; for (IntervalWindow w : sortedWindows(c)) { if (last != null && w.start().isBefore(last.end())) { c.merge(Arrays.asList(last), new IntervalWindow(last.start(), w.start())); } last = w; } }
@Override public CustomWindow decode(InputStream inStream) throws IOException { IntervalWindow superWindow = INTERVAL_WINDOW_CODER.decode(inStream); boolean isBig = VAR_INT_CODER.decode(inStream) != 0; return new CustomWindow(superWindow.start(), superWindow.end(), isBig); }
IntervalWindow sideInputWindow = (IntervalWindow) invocation.getArguments()[1]; long startMs = sideInputWindow.start().getMillis(); long endMs = sideInputWindow.end().getMillis();
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { String filenamePrefix = outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), ""); IntervalWindow interval = (IntervalWindow) window; String windowStr = String.format("%s-%s", interval.start().toString(), interval.end().toString()); String filename = String.format( "%s-%s-%s-of-%s-pane-%s%s%s.avro", filenamePrefix, windowStr, shardNumber, numShards, paneInfo.getIndex(), paneInfo.isLast() ? "-last" : "", outputFileHints.getSuggestedFilenameSuffix()); return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE); }
Instant windowStart = new Instant(0).plus(Duration.standardMinutes(window)); IntervalWindow iw = new IntervalWindow(windowStart, Duration.standardMinutes(1)); String baseAndWindow = baseFilename + "-" + iw.start() + "-" + iw.end(); switch (method) { case AVROIO_WRITE:
/** * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs * correctly per window and per key. */ @Test public void testScoreUpdatesPerWindow() { TestStream<KV<String, GameActionInfo>> createEvents = TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class))) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.RED_ONE, 50, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 50, Duration.standardMinutes(2)), event(TestUser.RED_ONE, 50, Duration.standardMinutes(3)), event(TestUser.RED_ONE, 60, Duration.standardMinutes(6)), event(TestUser.RED_TWO, 60, Duration.standardMinutes(7))) .advanceWatermarkToInfinity(); Duration teamWindowDuration = Duration.standardMinutes(5); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents) .apply(Window.<KV<String, GameActionInfo>>into(FixedWindows.of(teamWindowDuration))) .apply(ParDo.of(new UpdateTeamScoreFn(100))); String redTeam = TestUser.RED_ONE.getTeam(); String blueTeam = TestUser.BLUE_ONE.getTeam(); IntervalWindow window1 = new IntervalWindow(baseTime, teamWindowDuration); IntervalWindow window2 = new IntervalWindow(window1.end(), teamWindowDuration); PAssert.that(teamScores).inWindow(window1).containsInAnyOrder(KV.of(redTeam, 100)); PAssert.that(teamScores).inWindow(window2).containsInAnyOrder(KV.of(redTeam, 120)); p.run().waitUntilFinish(); }