SlidingWindows.of(durationParameter(parameters, 2)) .every(durationParameter(parameters, 1)); slidingWindows = slidingWindows.withOffset(durationParameter(parameters, 3));
@Test public void testVerifyCompatibility() throws IncompatibleWindowException { SlidingWindows.of(new Duration(10)).verifyCompatibility(SlidingWindows.of(new Duration(10))); thrown.expect(IncompatibleWindowException.class); SlidingWindows.of(new Duration(10)).verifyCompatibility(SlidingWindows.of(new Duration(20))); }
@Test public void testEquality() { assertTrue( SlidingWindows.of(new Duration(10)).isCompatible(SlidingWindows.of(new Duration(10)))); assertTrue( SlidingWindows.of(new Duration(10)).isCompatible(SlidingWindows.of(new Duration(10)))); assertFalse( SlidingWindows.of(new Duration(10)).isCompatible(SlidingWindows.of(new Duration(20)))); assertFalse( SlidingWindows.of(new Duration(10)).isCompatible(SlidingWindows.of(new Duration(20)))); }
@Test public void testEqualSize() throws Exception { Map<IntervalWindow, Set<String>> expected = new HashMap<>(); expected.put(new IntervalWindow(new Instant(0), new Instant(3)), set(1, 2)); expected.put(new IntervalWindow(new Instant(3), new Instant(6)), set(3, 4, 5)); expected.put(new IntervalWindow(new Instant(6), new Instant(9)), set(6, 7)); SlidingWindows windowFn = SlidingWindows.of(new Duration(3)).every(new Duration(3)); assertEquals(expected, runWindowFn(windowFn, Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L))); assertThat(windowFn.assignsToOneWindow(), is(true)); }
@Test public void testDefaultWindowMappingFn() { // [40, 1040), [340, 1340), [640, 1640) ... SlidingWindows slidingWindows = SlidingWindows.of(new Duration(1000)).every(new Duration(300)).withOffset(new Duration(40)); WindowMappingFn<?> mapping = slidingWindows.getDefaultWindowMappingFn(); assertThat(mapping.maximumLookback(), equalTo(Duration.ZERO)); // Prior assertEquals( new IntervalWindow(new Instant(340), new Instant(1340)), mapping.getSideInputWindow(new IntervalWindow(new Instant(0), new Instant(1041)))); assertEquals( new IntervalWindow(new Instant(340), new Instant(1340)), mapping.getSideInputWindow(new IntervalWindow(new Instant(0), new Instant(1339)))); // Align assertEquals( new IntervalWindow(new Instant(340), new Instant(1340)), mapping.getSideInputWindow(new IntervalWindow(new Instant(0), new Instant(1340)))); // After assertEquals( new IntervalWindow(new Instant(640), new Instant(1640)), mapping.getSideInputWindow(new IntervalWindow(new Instant(0), new Instant(1341)))); }
@Test public void testWindowedAutoComplete() { List<TimestampedValue<String>> words = Arrays.asList( TimestampedValue.of("xA", new Instant(1)), TimestampedValue.of("xA", new Instant(1)), TimestampedValue.of("xB", new Instant(1)), TimestampedValue.of("xB", new Instant(2)), TimestampedValue.of("xB", new Instant(2))); PCollection<String> input = p.apply(Create.timestamped(words)); PCollection<KV<String, List<CompletionCandidate>>> output = input .apply(Window.into(SlidingWindows.of(new Duration(2)))) .apply(new ComputeTopCompletions(2, recursive)); PAssert.that(output) .containsInAnyOrder( // Window [0, 2) KV.of("x", parseList("xA:2", "xB:1")), KV.of("xA", parseList("xA:2")), KV.of("xB", parseList("xB:1")), // Window [1, 3) KV.of("x", parseList("xB:3", "xA:2")), KV.of("xA", parseList("xA:2")), KV.of("xB", parseList("xB:3")), // Window [2, 3) KV.of("x", parseList("xB:2")), KV.of("xB", parseList("xB:2"))); p.run().waitUntilFinish(); }
@Test public void testSlightlyOverlapping() throws Exception { Map<IntervalWindow, Set<String>> expected = new HashMap<>(); expected.put(new IntervalWindow(new Instant(-5), new Instant(2)), set(1)); expected.put(new IntervalWindow(new Instant(0), new Instant(7)), set(1, 2, 5)); expected.put(new IntervalWindow(new Instant(5), new Instant(12)), set(5, 9, 10, 11)); expected.put(new IntervalWindow(new Instant(10), new Instant(17)), set(10, 11)); SlidingWindows windowFn = SlidingWindows.of(new Duration(7)).every(new Duration(5)); assertEquals(expected, runWindowFn(windowFn, Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L))); assertThat(windowFn.assignsToOneWindow(), is(false)); }
TimestampedValue.of(KV.of("b", 13), new Instant(10L))) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))) .apply(Window.into(SlidingWindows.of(Duration.millis(2))));
@Test public void testDisplayData() { Duration windowSize = Duration.standardSeconds(1234); Duration offset = Duration.standardSeconds(2345); Duration period = Duration.standardSeconds(3456); SlidingWindows slidingWindowFn = SlidingWindows.of(windowSize).every(period).withOffset(offset); DisplayData displayData = DisplayData.from(slidingWindowFn); assertThat(displayData, hasDisplayItem("size", windowSize)); assertThat(displayData, hasDisplayItem("period", period)); assertThat(displayData, hasDisplayItem("offset", offset)); } }
@Test public void testElidings() throws Exception { Map<IntervalWindow, Set<String>> expected = new HashMap<>(); expected.put(new IntervalWindow(new Instant(0), new Instant(3)), set(1, 2)); expected.put(new IntervalWindow(new Instant(10), new Instant(13)), set(10, 11)); expected.put(new IntervalWindow(new Instant(100), new Instant(103)), set(100)); SlidingWindows windowFn = SlidingWindows.of(new Duration(3)).every(new Duration(10)); assertEquals( expected, runWindowFn( // Only look at the first 3 millisecs of every 10-millisec interval. windowFn, Arrays.asList(1L, 2L, 3L, 5L, 9L, 10L, 11L, 100L))); assertThat(windowFn.assignsToOneWindow(), is(true)); }
@Override public PCollection<IndexedRecord> expand(PCollection<IndexedRecord> indexedRecordPCollection) { PCollection<IndexedRecord> windowed_items; if (properties.windowLength.getValue() < 1) { TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).setAndThrow(properties.windowLength.getName(), String.valueOf(properties.windowLength.getValue())); } // Session Window if (properties.windowSession.getValue()) { windowed_items = indexedRecordPCollection.apply(Window.<IndexedRecord> into( Sessions.withGapDuration(Duration.millis(properties.windowLength.getValue().intValue())))); return windowed_items; } if (properties.windowSlideLength.getValue() < 1) { // Fixed Window windowed_items = indexedRecordPCollection.apply( Window.<IndexedRecord> into(FixedWindows.of(new Duration(properties.windowLength.getValue().intValue())))); } else { // Sliding Window windowed_items = indexedRecordPCollection.apply( Window.<IndexedRecord> into(SlidingWindows.of(new Duration(properties.windowLength.getValue().intValue())) .every(new Duration(properties.windowSlideLength.getValue().intValue())))); } return windowed_items; }
@Test public void testOffset() throws Exception { Map<IntervalWindow, Set<String>> expected = new HashMap<>(); expected.put(new IntervalWindow(new Instant(-8), new Instant(2)), set(1)); expected.put(new IntervalWindow(new Instant(-3), new Instant(7)), set(1, 2, 5)); expected.put(new IntervalWindow(new Instant(2), new Instant(12)), set(2, 5, 9, 10, 11)); expected.put(new IntervalWindow(new Instant(7), new Instant(17)), set(9, 10, 11)); assertEquals( expected, runWindowFn( SlidingWindows.of(new Duration(10)).every(new Duration(5)).withOffset(new Duration(2)), Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L))); }
@Test public void testSimple() throws Exception { Map<IntervalWindow, Set<String>> expected = new HashMap<>(); expected.put(new IntervalWindow(new Instant(-5), new Instant(5)), set(1, 2)); expected.put(new IntervalWindow(new Instant(0), new Instant(10)), set(1, 2, 5, 9)); expected.put(new IntervalWindow(new Instant(5), new Instant(15)), set(5, 9, 10, 11)); expected.put(new IntervalWindow(new Instant(10), new Instant(20)), set(10, 11)); SlidingWindows windowFn = SlidingWindows.of(new Duration(10)).every(new Duration(5)); assertEquals(expected, runWindowFn(windowFn, Arrays.asList(1L, 2L, 5L, 9L, 10L, 11L))); assertThat(windowFn.assignsToOneWindow(), is(false)); }
SlidingWindowsPayload slidingWindowsPayload = SlidingWindowsPayload.newBuilder() .setSize(Durations.fromMillis(((SlidingWindows) windowFn).getSize().getMillis())) .setOffset(Timestamps.fromMillis(((SlidingWindows) windowFn).getOffset().getMillis())) .setPeriod(Durations.fromMillis(((SlidingWindows) windowFn).getPeriod().getMillis())) .build(); return SdkFunctionSpec.newBuilder()
@Override public PCollection<IndexedRecord> expand(PCollection<IndexedRecord> indexedRecordPCollection) { PCollection<IndexedRecord> windowed_items; if (properties.windowLength.getValue() < 1) { TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).setAndThrow(properties.windowLength.getName(), String.valueOf(properties.windowLength.getValue())); } // Session Window if (properties.windowSession.getValue()) { windowed_items = indexedRecordPCollection.apply(Window.<IndexedRecord> into( Sessions.withGapDuration(Duration.millis(properties.windowLength.getValue().intValue())))); return windowed_items; } if (properties.windowSlideLength.getValue() < 1) { // Fixed Window windowed_items = indexedRecordPCollection.apply( Window.<IndexedRecord> into(FixedWindows.of(new Duration(properties.windowLength.getValue().intValue())))); } else { // Sliding Window windowed_items = indexedRecordPCollection.apply( Window.<IndexedRecord> into(SlidingWindows.of(new Duration(properties.windowLength.getValue().intValue())) .every(new Duration(properties.windowSlideLength.getValue().intValue())))); } return windowed_items; }
SlidingWindowsPayload slidingParams = SlidingWindowsPayload.parseFrom(windowFnSpec.getSpec().getPayload()); return SlidingWindows.of(Duration.millis(Durations.toMillis(slidingParams.getSize()))) .every(Duration.millis(Durations.toMillis(slidingParams.getPeriod()))) .withOffset(Duration.millis(Timestamps.toMillis(slidingParams.getOffset()))); } else if (s.equals(getUrn(SessionsPayload.Enum.PROPERTIES))) { SessionsPayload sessionParams =