@Override public int compare(PartitionedEvent[] o1, PartitionedEvent[] o2) { if (o1.length > 0 && o2.length > 0) { return (int) (o1[0].getTimestamp() - o2[0].getTimestamp()); } else { return 0; } } }
@Override public boolean equals(PartitionedEvent[] a1, PartitionedEvent[] a2) { return a1[0].getTimestamp() == a2[0].getTimestamp(); }
@Override public int compare(PartitionedEvent o1, PartitionedEvent o2) { if (Objects.equals(o1, o2)) { return 0; } else { if (o1 == null && o2 == null) { return 0; } else if (o1 != null && o2 == null) { return 1; } else if (o1 == null) { return -1; } // Unstable Sorting Algorithm if (o1.getTimestamp() <= o2.getTimestamp()) { return -1; } else { return 1; } } } }
/** * @param event input event. * @return whether sorted. */ private boolean dispatchToSortHandler(PartitionedEvent event) { if (event.getTimestamp() <= 0) { return false; } StreamSortHandler sortHandler = streamSortHandlers.get(event.getPartition()); if (sortHandler == null) { if (event.isSortRequired()) { LOG.warn("Stream sort handler required has not been loaded so emmit directly: {}", event); this.context.counter().incr("miss_sort_count"); } return false; } else { sortHandler.nextEvent(event); return true; } }
/** * TODO: Potential improvement: if StreamSortHandler is expensive, we can use DISRUPTOR to buffer. * * @param event StreamEvent */ public void nextEvent(PartitionedEvent event) { this.context.counter().incr("receive_count"); if (!dispatchToSortHandler(event)) { this.context.counter().incr("direct_count"); // Pass through directly if no need to sort outputCollector.emit(event); } this.context.counter().incr("sort_count"); // Update stream clock time if moving forward and trigger all tick listeners streamTimeClockManager.onTimeUpdate(event.getStreamId(), event.getTimestamp()); }
@Test public void testSortedInPatient() throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl(); sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT1h", 5000), mockCollector); List<PartitionedEvent> sortedList = new LinkedList<>(); int i = 0; while (i < 1000000) { PartitionedEvent event = MockSampleMetadataFactory.createRandomPartitionedEvent("sampleStream_1", System.currentTimeMillis() + i); sortHandler.nextEvent(event); sortedList.add(event); if (event.getTimestamp() > timeClock.getTime()) { timeClock.moveForward(event.getTimestamp()); } sortHandler.onTick(timeClock, System.currentTimeMillis()); i++; } sortHandler.close(); Assert.assertTrue(timeOrdering.isOrdered(sortedList)); Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get())); Assert.assertEquals(1000000, mockCollector.get().size()); }
sortHandler.nextEvent(event); sortedList.add(event); if (event.getTimestamp() > timeClock.getTime()) { timeClock.moveForward(event.getTimestamp());
@Test public void testJavaSerialization() { PartitionedEvent partitionedEvent = new PartitionedEvent(); partitionedEvent.setPartitionKey(partitionedEvent.hashCode()); partitionedEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream", Arrays.asList("name", "host"))); StreamEvent event = new StreamEvent(); event.setStreamId("sampleStream"); event.setTimestamp(System.currentTimeMillis()); event.setData(new Object[] {"CPU", "LOCALHOST", true, Long.MAX_VALUE, 60.0}); partitionedEvent.setEvent(event); int javaSerializationLength = SerializationUtils.serialize(partitionedEvent).length; LOG.info("Java serialization length: {}, event: {}", javaSerializationLength, partitionedEvent); int compactLength = 0; compactLength += "sampleStream".getBytes().length; compactLength += ByteUtils.intToBytes(partitionedEvent.getPartition().hashCode()).length; compactLength += ByteUtils.longToBytes(partitionedEvent.getTimestamp()).length; compactLength += "CPU".getBytes().length; compactLength += "LOCALHOST".getBytes().length; compactLength += 1; compactLength += ByteUtils.longToBytes(Long.MAX_VALUE).length; compactLength += ByteUtils.doubleToBytes(60.0).length; LOG.info("Compact serialization length: {}, event: {}", compactLength, partitionedEvent); Assert.assertTrue(compactLength * 20 < javaSerializationLength); }
sortHandler.nextEvent(event); unsortedList.add(event); if (event.getTimestamp() > timeClock.getTime()) { timeClock.moveForward(event.getTimestamp());
/** * Used to debug window bucket lifecycle * <p> * Window period: PT1h, margin: 5s * * @throws InterruptedException */ @Test public void testWithSortedEventsAndExpireBySystemTime() throws InterruptedException { MockPartitionedCollector mockCollector = new MockPartitionedCollector(); StreamTimeClockInLocalMemory timeClock = new StreamTimeClockInLocalMemory("sampleStream_1"); Ordering<PartitionedEvent> timeOrdering = Ordering.from(PartitionedEventTimeOrderingComparator.INSTANCE); StreamSortWindowHandlerImpl sortHandler = new StreamSortWindowHandlerImpl(); sortHandler.prepare("sampleStream_1", MockSampleMetadataFactory.createSampleStreamSortSpec("sampleStream_1", "PT10s", 1000), mockCollector); List<PartitionedEvent> sortedList = new LinkedList<>(); PartitionedEvent event = MockSampleMetadataFactory.createRandomSortedEventGroupedByName("sampleStream_1"); sortHandler.nextEvent(event); sortedList.add(event); timeClock.moveForward(event.getTimestamp()); sortHandler.onTick(timeClock, System.currentTimeMillis()); // Triggered to become expired by System time sortHandler.onTick(timeClock, System.currentTimeMillis() + 10 * 1000 + 1000L + 1); Assert.assertTrue(timeOrdering.isOrdered(sortedList)); Assert.assertTrue(timeOrdering.isOrdered(mockCollector.get())); Assert.assertEquals(1, mockCollector.get().size()); sortHandler.close(); }
Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000"), list.get(0).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:00,000"), list.get(1).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:01,000"), list.get(2).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:30,000"), list.get(3).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:35,000"), list.get(4).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:40,000"), list.get(5).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:50,000"), list.get(6).getTimestamp()); Assert.assertEquals(DateTimeUtil.humanDateToMillisecondsWithoutException("2016-05-04 00:00:56,000"), list.get(7).getTimestamp());