public PartitionedEvent copy() { PartitionedEvent copied = new PartitionedEvent(); copied.setEvent(this.getEvent()); copied.setPartition(this.partition); copied.setPartitionKey(this.partitionKey); return copied; }
@Override public boolean equals(Object obj) { if (obj == this) { return true; } if (obj == null) { return false; } if (obj instanceof PartitionedEvent) { PartitionedEvent another = (PartitionedEvent) obj; return !(this.partitionKey != another.getPartitionKey() || !Objects.equals(this.event, another.getEvent()) || !Objects.equals(this.partition, another.getPartition()) || !Objects.equals(this.anchor, another.anchor)); } else { return false; } }
/** * @param event input event. * @return whether sorted. */ private boolean dispatchToSortHandler(PartitionedEvent event) { if (event.getTimestamp() <= 0) { return false; } StreamSortHandler sortHandler = streamSortHandlers.get(event.getPartition()); if (sortHandler == null) { if (event.isSortRequired()) { LOG.warn("Stream sort handler required has not been loaded so emmit directly: {}", event); this.context.counter().incr("miss_sort_count"); } return false; } else { sortHandler.nextEvent(event); return true; } }
public boolean isSortRequired() { return isPartitionRequired() && this.getPartition().getSortSpec() != null; }
@Test public void testJavaSerialization() { PartitionedEvent partitionedEvent = new PartitionedEvent(); partitionedEvent.setPartitionKey(partitionedEvent.hashCode()); partitionedEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream", Arrays.asList("name", "host"))); StreamEvent event = new StreamEvent(); event.setStreamId("sampleStream"); event.setTimestamp(System.currentTimeMillis()); event.setData(new Object[] {"CPU", "LOCALHOST", true, Long.MAX_VALUE, 60.0}); partitionedEvent.setEvent(event); int javaSerializationLength = SerializationUtils.serialize(partitionedEvent).length; LOG.info("Java serialization length: {}, event: {}", javaSerializationLength, partitionedEvent); int compactLength = 0; compactLength += "sampleStream".getBytes().length; compactLength += ByteUtils.intToBytes(partitionedEvent.getPartition().hashCode()).length; compactLength += ByteUtils.longToBytes(partitionedEvent.getTimestamp()).length; compactLength += "CPU".getBytes().length; compactLength += "LOCALHOST".getBytes().length; compactLength += 1; compactLength += ByteUtils.longToBytes(Long.MAX_VALUE).length; compactLength += ByteUtils.doubleToBytes(60.0).length; LOG.info("Compact serialization length: {}, event: {}", compactLength, partitionedEvent); Assert.assertTrue(compactLength * 20 < javaSerializationLength); }
@SuppressWarnings("serial") public static PartitionedEvent createSimpleStreamEvent() { StreamEvent event = StreamEvent.builder() .schema(createSampleStreamDefinition("sampleStream_1")) .streamId("sampleStream_1") .timestamep(System.currentTimeMillis()) .attributes(new HashMap<String, Object>() {{ put("name", "cpu"); put("host", "localhost"); put("flag", true); put("value", 60.0); put("data", Long.MAX_VALUE); put("unknown", "unknown column value"); }}).build(); PartitionedEvent pEvent = new PartitionedEvent(); pEvent.setEvent(event); pEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream_1", Arrays.asList("name", "host"))); return pEvent; } }
public static PartitionedEvent createRandomOutOfTimeOrderEventGroupedByName(String streamId) { StreamEvent event = createRandomStreamEvent(streamId); event.setTimestamp(System.currentTimeMillis() + TIME_DELTA_OPTIONS[RANDOM.nextInt(TIME_DELTA_OPTIONS.length)]); return new PartitionedEvent(event, createSampleStreamGroupbyPartition(streamId, Arrays.asList("name")), event.getData()[0].hashCode()); }
public void emit(PartitionedEvent event) { try { this.streamContext.counter().incr("send_count"); StreamPartition partition = event.getPartition(); List<StreamRouterSpec> routerSpecs = routeSpecMap.get(partition); if (routerSpecs == null || routerSpecs.size() <= 0) { if (LOG.isDebugEnabled()) { LOG.debug("Drop event {} as StreamPartition {} is not pointed to any router metadata {}", event, event.getPartition(), routeSpecMap); StreamEvent newEvent = event.getEvent().copy(); String targetStreamId = StreamIdConversion.generateStreamIdBetween(sourceId, streamRoute.getTargetComponentId()); try { PartitionedEvent emittedEvent = new PartitionedEvent(newEvent, partition, streamRoute.getPartitionKey());
@Test public void testPartitionedEvent() { PartitionedEvent partitionedEvent = new PartitionedEvent(); Assert.assertEquals("PartitionedEvent[partition=null,event=null,key=0", partitionedEvent.toString()); Object[] data = new Object[]{"namevalue", "hostvalue", "1", 10, 0.1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}", 1}; StreamEvent streamEvent = new StreamEvent("streamId", 1478667686971l, data); StreamSortSpec streamSortSpec = new StreamSortSpec(); streamSortSpec.setWindowPeriod("PT10S"); StreamPartition streamPartition = new StreamPartition(); List<String> columns = new ArrayList<>(); columns.add("jobId"); streamPartition.setColumns(columns); streamPartition.setSortSpec(streamSortSpec); streamPartition.setStreamId("test"); streamPartition.setType(StreamPartition.Type.GROUPBY); partitionedEvent = new PartitionedEvent(streamEvent, streamPartition, 1); Assert.assertEquals("PartitionedEvent[partition=StreamPartition[streamId=test,type=GROUPBY,columns=[jobId],sortSpec=[StreamSortSpec[windowPeriod=PT10S,windowMargin=30000]]],event=StreamEvent[stream=STREAMID,timestamp=2016-11-09 05:01:26,971,data=[namevalue,hostvalue,1,10,0.1,-0.2,{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"},1],metaVersion=null],key=1", partitionedEvent.toString()); PartitionedEvent partitionedEventCopy = partitionedEvent.copy(); Assert.assertFalse(partitionedEventCopy == partitionedEvent); Assert.assertTrue(partitionedEventCopy.equals(partitionedEvent)); Assert.assertTrue(partitionedEventCopy.hashCode() == partitionedEvent.hashCode()); } }
public static PartitionedEvent createSimpleStreamEvent() { StreamEvent event = null; try { event = StreamEvent.builder() .schema(MockSampleMetadataFactory.createSingletonMetadataServiceWithSample().getStreamDefinition("sampleStream_1")) .streamId("sampleStream_1") .timestamep(System.currentTimeMillis()) .attributes(new HashMap<String, Object>() {{ put("name", "cpu"); put("value", 60.0); put("unknown", "unknown column value"); }}).build(); } catch (StreamNotDefinedException e) { e.printStackTrace(); } PartitionedEvent pEvent = new PartitionedEvent(); pEvent.setEvent(event); return pEvent; }
@Override public List<Integer> chooseTasks(int taskId, List<Object> values) { Object routingKeyObj = values.get(0); if (routingKeyObj != null) { PartitionedEvent partitionedEvent = (PartitionedEvent) routingKeyObj; if (partitionedEvent.getPartition().getType() == StreamPartition.Type.GLOBAL) { return globalGroupingDelegate.chooseTasks(taskId, values); } else if (partitionedEvent.getPartition().getType() == StreamPartition.Type.GROUPBY) { return Collections.singletonList(outdegreeTasks.get((int) (partitionedEvent.getPartitionKey() % this.outdegreeTasks.size()))); } // Shuffle by defaults return shuffleGroupingDelegate.chooseTasks(taskId, values); } LOG.warn("Illegal null StreamRoute, throw event"); return Collections.emptyList(); } }
private static boolean isAcceptedByPolicy(PartitionedEvent event, PolicyDefinition policy) { return policy.getPartitionSpec().contains(event.getPartition()) && (policy.getInputStreams().contains(event.getEvent().getStreamId()) || policy.getDefinition().getInputStreams().contains(event.getEvent().getStreamId())); }
@Override public boolean add(PartitionedEvent partitionedEvent) { synchronized (treeMultisetCache) { if (accept(partitionedEvent.getEvent().getTimestamp())) { treeMultisetCache.add(partitionedEvent); return true; } else { if (LOG.isDebugEnabled()) { LOG.debug("{} is not acceptable, ignored", partitionedEvent); } return false; } } }
@Override public int compare(PartitionedEvent[] o1, PartitionedEvent[] o2) { if (o1.length > 0 && o2.length > 0) { return (int) (o1[0].getTimestamp() - o2[0].getTimestamp()); } else { return 0; } } }
PartitionedEvent pe = deserialize(input.getValueByField(AlertConstants.FIELD_0)); if (logEventEnabled) { LOG.info("Alert bolt {} received event: {}", boltId, pe.getEvent()); String streamEventVersion = pe.getEvent().getMetaVersion(); pe.getEvent().setMetaVersion(specVersion); } else if (streamEventVersion != null && !streamEventVersion.equals(specVersion)) { if (specVersion != null && streamEventVersion != null specVersionOutofdate = timestampOfSpecVersion < timestampOfStreamEventVersion; if (!specVersionOutofdate) { pe.getEvent().setMetaVersion(specVersion); policyGroupEvaluator.nextEvent(pe.withAnchor(input)); synchronized (outputLock) { this.collector.ack(input);
public boolean isPartitionRequired() { return this.getPartition() != null; }
/** * This unit test is not to mock the end2end logic of correlation spout, * but simply generate some sample data for following bolts testing */ @Override public void nextTuple() { PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1"); LOG.info("Receive {}", event); collector.emit(outputStreamIds.get( // group by the first field in event i.e. name (int) (event.getPartitionKey() % outputStreamIds.size())), Collections.singletonList(event)); Utils.sleep(500); }
/** * TODO: Potential improvement: if StreamSortHandler is expensive, we can use DISRUPTOR to buffer. * * @param event StreamEvent */ public void nextEvent(PartitionedEvent event) { this.context.counter().incr("receive_count"); if (!dispatchToSortHandler(event)) { this.context.counter().incr("direct_count"); // Pass through directly if no need to sort outputCollector.emit(event); } this.context.counter().incr("sort_count"); // Update stream clock time if moving forward and trigger all tick listeners streamTimeClockManager.onTimeUpdate(event.getStreamId(), event.getTimestamp()); }
event1.setData(data); event1.setStreamId(streamId); PartitionedEvent pEvent1 = new PartitionedEvent(); pEvent1.setEvent(event1); pEvent1.setPartition(partition); event2.setData(data2); event2.setStreamId(streamId); PartitionedEvent pEvent2 = new PartitionedEvent(); pEvent2.setEvent(event2); pEvent2.setPartition(partition);