public ReflectStandardEvent(StandardEvent event) { setEvent_initiator(event.getEventInitiator()); setEvent_name(event.getEventName()); setIp(event.getIp()); setSession_id(event.getSessionId()); setTimestamp(event.getTimestamp()); setUser_id(event.getUserId()); }
/** * Gets the Builder instance for the 'event' field and creates one if it doesn't exist yet. * The events the other events are correlated with */ public org.kitesdk.data.event.StandardEvent.Builder getEventBuilder() { if (eventBuilder == null) { if (hasEvent()) { setEventBuilder(org.kitesdk.data.event.StandardEvent.newBuilder(event)); } else { setEventBuilder(org.kitesdk.data.event.StandardEvent.newBuilder()); } } return eventBuilder; }
@Override public Tuple2<List<StandardEvent>, List<StandardEvent>> call(Tuple2<CorrelationKey, Iterable<StandardEvent>> t) throws Exception { Iterable<StandardEvent> allEvents = t._2(); List<StandardEvent> alerts = new ArrayList<StandardEvent>(); List<StandardEvent> clicks = new ArrayList<StandardEvent>(); for (StandardEvent event : allEvents) { if (event.getEventDetails() != null && event.getEventDetails().containsKey(new Utf8("type")) && "alert".equals(event.getEventDetails().get(new Utf8("type")).toString())) { alerts.add(event); } else if (event.getEventDetails() != null && event.getEventDetails().containsKey(new Utf8("type")) && "click".equals(event.getEventDetails().get(new Utf8("type")).toString())) { clicks.add(event); } } if (alerts.isEmpty()) { return new Tuple2<List<StandardEvent>, List<StandardEvent>>(alerts, alerts); } else { return new Tuple2<List<StandardEvent>, List<StandardEvent>>(alerts, clicks); } } });
@Override public String map(StandardEvent event) { // Create a key from the session id and user id return event.getSessionId() + event.getUserId(); } }
@Override public Iterable<Tuple2<CorrelationKey, StandardEvent>> call(Tuple2<StandardEvent, Void> t) throws Exception { List<Tuple2<CorrelationKey, StandardEvent>> result = new ArrayList<Tuple2<CorrelationKey, StandardEvent>>(2); StandardEvent event = t._1(); long loTimestamp = createLoTimestamp(event.getTimestamp()); long hiTimestamp = createHiTimestamp(event.getTimestamp()); String ip = event.getIp().toString(); result.add(new Tuple2<CorrelationKey, StandardEvent>( new CorrelationKey(loTimestamp, ip), event)); result.add(new Tuple2<CorrelationKey, StandardEvent>( new CorrelationKey(hiTimestamp, ip), event)); return result; } });
long startTime = firstEvent.getTimestamp(); long endTime = firstEvent.getTimestamp(); int numEvents = 1; startTime = Math.min(startTime, event.getTimestamp()); endTime = Math.max(endTime, event.getTimestamp()); numEvents += 1; .setUserId(firstEvent.getUserId()) // the user id (grouped by) .setSessionId(firstEvent.getSessionId()) // session id (grouped by) .setIp(firstEvent.getIp()) // the source IP address .setStartTimestamp(startTime) .setDuration(endTime - startTime)
@Override public Iterable<Tuple2<CorrelatedEvents, Void>> call(Tuple2<List<StandardEvent>, List<StandardEvent>> t) throws Exception { List<Tuple2<CorrelatedEvents, Void>> results = new ArrayList<Tuple2<CorrelatedEvents, Void>>(); List<StandardEvent> alerts = t._1(); List<StandardEvent> clicks = t._2(); for (StandardEvent alert : alerts) { List<StandardEvent> correlated = new ArrayList<StandardEvent>(); for (StandardEvent click : clicks) { if (Math.abs(alert.getTimestamp() - click.getTimestamp()) <= FIVE_MIN_MILLIS) { correlated.add(click); } } if (!correlated.isEmpty()) { results.add(new Tuple2(CorrelatedEvents.newBuilder() .setEvent(alert) .setCorrelated(correlated) .build(), null)); } } return results; } });
@BeforeClass public static void setup() throws IOException { fs = LocalFileSystem.getInstance(); testDirectory = new Path(Files.createTempDir().getAbsolutePath()); FileSystemDatasetRepository repo = new FileSystemDatasetRepository(fs.getConf(), testDirectory); Dataset<StandardEvent> writerDataset = repo.create("ns", "test", new DatasetDescriptor.Builder() .schema(StandardEvent.class) .build(), StandardEvent.class); DatasetWriter<StandardEvent> writer = writerDataset.newWriter(); for (long i = 0; i < totalRecords; i++) { String text = String.valueOf(i); writer.write(new StandardEvent(text, text, i, text, text, i)); } writer.close(); readerDataset = repo.load("ns", "test", GenericData.Record.class); }
@Test public void testGetDatumReaderForSpecificType() { Class<StandardEvent> type = StandardEvent.class; Schema writerSchema = StandardEvent.getClassSchema(); DatumReader result = DataModelUtil.getDatumReaderForType(type, writerSchema); assertEquals(SpecificDatumReader.class, result.getClass()); }
assertTrue(v1.includes(standardEvent(sepEvent.getTimestamp()))); assertFalse(v1.includes(standardEvent(octEvent.getTimestamp()))); assertFalse(v1.includes(standardEvent(novEvent.getTimestamp()))); assertFalse(v2.includes(standardEvent(sepEvent.getTimestamp()))); assertTrue(v2.includes(standardEvent(octEvent.getTimestamp()))); assertFalse(v2.includes(standardEvent(novEvent.getTimestamp()))); assertFalse(v3.includes(standardEvent(sepEvent.getTimestamp()))); assertFalse(v3.includes(standardEvent(octEvent.getTimestamp()))); assertTrue(v3.includes(standardEvent(novEvent.getTimestamp())));
public ReflectSmallEvent(StandardEvent event) { setSession_id(event.getSessionId()); setUser_id(event.getUserId()); }
@BeforeClass public static void setup() throws IOException { fs = LocalFileSystem.getInstance(); testDirectory = new Path(Files.createTempDir().getAbsolutePath()); FileSystemDatasetRepository repo = new FileSystemDatasetRepository(fs.getConf(), testDirectory); Dataset<StandardEvent> writerDataset = repo.create("ns", "test", new DatasetDescriptor.Builder() .schema(StandardEvent.class) .format(Formats.PARQUET) .build(), StandardEvent.class); DatasetWriter<StandardEvent> writer = writerDataset.newWriter(); for (long i = 0; i < totalRecords; i++) { String text = String.valueOf(i); writer.write(new StandardEvent(text, text, i, text, text, i)); } writer.close(); readerDataset = repo.load("ns", "test", GenericRecord.class); }
@Test public void testGetDatumReaderForGenericType() { Class<GenericData.Record> type = GenericData.Record.class; Schema writerSchema = StandardEvent.getClassSchema(); DatumReader result = DataModelUtil.getDatumReaderForType(type, writerSchema); assertEquals(GenericDatumReader.class, result.getClass()); }
@Test public void testEmptyCheck() throws IOException { Assert.assertTrue("New dataset should be empty", unbounded.isEmpty()); // NOTE: this is an un-restricted write so all should succeed DatasetWriter<StandardEvent> writer = null; try { writer = unbounded.newWriter(); writer.write(sepEvent); } finally { Closeables.close(writer, false); } Assert.assertFalse("Should not be empty after write", unbounded.isEmpty()); Assert.assertFalse("Should find event in September", unbounded.with("timestamp", sepEvent.getTimestamp()).isEmpty()); Assert.assertTrue("Should not find event in October", unbounded.with("timestamp", octEvent.getTimestamp()).isEmpty()); }
private static SmallEvent toSmallEvent(StandardEvent event) { return SmallEvent.newBuilder() .setUserId(event.getUserId()) .setSessionId(event.getSessionId()) .build(); }
@Override public StandardEvent build() { try { StandardEvent record = new StandardEvent(); record.event_initiator = fieldSetFlags()[0] ? this.event_initiator : (java.lang.String) defaultValue(fields()[0]); record.event_name = fieldSetFlags()[1] ? this.event_name : (java.lang.String) defaultValue(fields()[1]); record.user_id = fieldSetFlags()[2] ? this.user_id : (java.lang.Long) defaultValue(fields()[2]); record.session_id = fieldSetFlags()[3] ? this.session_id : (java.lang.String) defaultValue(fields()[3]); record.ip = fieldSetFlags()[4] ? this.ip : (java.lang.String) defaultValue(fields()[4]); record.timestamp = fieldSetFlags()[5] ? this.timestamp : (java.lang.Long) defaultValue(fields()[5]); return record; } catch (Exception e) { throw new org.apache.avro.AvroRuntimeException(e); } } }
private StandardEvent standardEvent(long timestamp) { return StandardEvent.newBuilder(event).setTimestamp(timestamp).build(); }
@Test public void testResolveTypeObjectToSpecifc() { Class<Object> type = Object.class; Schema schema = StandardEvent.getClassSchema(); Class expResult = StandardEvent.class; Class result = DataModelUtil.resolveType(type, schema); assertEquals(expResult, result); }
unbounded); long octInstant = octEvent.getTimestamp();
@Override public StandardEvent build() { try { StandardEvent record = new StandardEvent(); record.event_initiator = fieldSetFlags()[0] ? this.event_initiator : (org.apache.avro.util.Utf8) defaultValue(fields()[0]); record.event_name = fieldSetFlags()[1] ? this.event_name : (org.apache.avro.util.Utf8) defaultValue(fields()[1]); record.user_id = fieldSetFlags()[2] ? this.user_id : (java.lang.Long) defaultValue(fields()[2]); record.session_id = fieldSetFlags()[3] ? this.session_id : (org.apache.avro.util.Utf8) defaultValue(fields()[3]); record.ip = fieldSetFlags()[4] ? this.ip : (org.apache.avro.util.Utf8) defaultValue(fields()[4]); record.timestamp = fieldSetFlags()[5] ? this.timestamp : (java.lang.Long) defaultValue(fields()[5]); record.event_details = fieldSetFlags()[6] ? this.event_details : (java.util.Map<org.apache.avro.util.Utf8,java.lang.Object>) defaultValue(fields()[6]); return record; } catch (Exception e) { throw new org.apache.avro.AvroRuntimeException(e); } } }