@Override public Datum get() { if (!this.reader.hasNext()) { try { this.reader.close(); } catch (IOException e) { throw new RuntimeException( "Failed to close reader for partition: " + this.partition, e); } return Datum.endOfStream(); } Object next = this.reader.next(); // we assign it to batch return Datum.of( GlobalWindowing.Window.get(), next, // ingestion time System.currentTimeMillis()); } }
@Test public void testTimeSlidingLabelAssignment() { TimeSliding<Long> windowing = TimeSliding .of(Duration.ofHours(1), Duration.ofMinutes(20)); UnaryFunction<Long, Long> eventTimeAssigner = e -> e * 1000L; long[] data = { 3590, 3600, 3610, 3800, 7190, 7200, 7210 }; for (long event : data) { Iterable<TimeInterval> labels = windowing .assignWindowsToElement(new Elem<>( GlobalWindowing.Window.get(), event, eventTimeAssigner.apply(event))); // verify window count assertEquals(3, Iterables.size(labels)); // verify that each window contains the original event for (TimeInterval l : labels) { long stamp = event * 1000L; assertTrue(stamp >= l.getStartMillis()); assertTrue(stamp <= l.getEndMillis()); } } }
@Override public JavaRDD<?> translate(FlowUnfolder.InputOperator operator, SparkExecutorContext context) { // get original datasource from operator DataSource<?> ds = operator.output().getSource(); try { final long desiredSplitSize = context.getSettings() .getLong(DESIRED_SPLIT_SIZE, DEFAULT_DESIRED_SPLIT_SIZE); final Configuration conf = DataSourceInputFormat.configure( new Configuration(), ds, desiredSplitSize); @SuppressWarnings("unchecked") JavaPairRDD<Object, Object> pairs = context .getExecutionEnvironment() .newAPIHadoopRDD(conf, DataSourceInputFormat.class, Object.class, Object.class) .setName(operator.getName() + "::read"); // map values to WindowedElement return pairs .values() .map(v -> new SparkElement<>(GlobalWindowing.Window.get(), 0L, v)) .setName(operator.getName() + "::windowed-element"); } catch (IOException e) { throw new RuntimeException(e); } } }
@Override public Datum get() { if (!this.reader.hasNext()) { try { this.reader.close(); } catch (IOException e) { throw new RuntimeException( "Failed to close reader for partition: " + this.partition, e); } return Datum.endOfStream(); } Object next = this.reader.next(); // we commit right away this.reader.commitOffset(this.reader.getCurrentOffset()); // we assign it to global windowing return Datum.of( GlobalWindowing.Window.get(), next, // ingestion time System.currentTimeMillis()); } }
@Override public Datum get() { if (!this.reader.hasNext()) { try { this.reader.close(); } catch (IOException e) { throw new RuntimeException( "Failed to close reader for partition: " + this.partition, e); } return Datum.endOfStream(); } Object next = this.reader.next(); // we commit right away this.reader.commitOffset(this.reader.getCurrentOffset()); // we assign it to global windowing return Datum.of( GlobalWindowing.Window.get(), next, // ingestion time System.currentTimeMillis()); } }
@Override public Datum get() { if (!this.reader.hasNext()) { try { this.reader.close(); } catch (IOException e) { throw new RuntimeException( "Failed to close reader for partition: " + this.partition, e); } return Datum.endOfStream(); } Object next = this.reader.next(); // we assign it to batch return Datum.of( GlobalWindowing.Window.get(), next, // ingestion time System.currentTimeMillis()); } }
@Override public Iterable<GlobalWindowing.Window> assignWindowsToElement(WindowedElement<?, T> el) { return singleton(GlobalWindowing.Window.get()); }
private StreamingElement<GlobalWindowing.Window, T> toStreamingElement(T elem) { // assign ingestion timestamp to elements return new StreamingElement<>(GlobalWindowing.Window.get(), elem); }
@Override public Iterable<GlobalWindowing.Window> assignWindowsToElement(WindowedElement<?, T> el) { return singleton(GlobalWindowing.Window.get()); }
@Override public BatchElement<GlobalWindowing.Window, T> nextRecord( BatchElement<GlobalWindowing.Window, T> reuse) throws IOException { return new BatchElement<>(GlobalWindowing.Window.get(), 0L, reader.next()); }