return new EventList(Event.EventContext.apiKey(apiKey), project, list);
public void store(String jsonRaw) throws IOException { if (jsonEventDeserializer == null) { throw new RakamException("Event store is not supported.", BAD_REQUEST); } JsonParser jp = JsonHelper.getMapper().getFactory().createParser(jsonRaw); JsonToken t = jp.nextToken(); if (t != JsonToken.START_ARRAY) { throw new RakamException("The script didn't return an array", BAD_REQUEST); } t = jp.nextToken(); List<Event> list = new ArrayList<>(); for (; t == START_OBJECT; t = jp.nextToken()) { list.add(jsonEventDeserializer.deserializeWithProject(jp, project, Event.EventContext.empty(), true)); } EventCollectionHttpService.mapEvent(eventMapperSet, eventMapper -> eventMapper.mapAsync(new EventList(Event.EventContext.empty(), list), EMPTY_PARAMS, localhost, HttpHeaders.EMPTY_HEADERS)); int[] ints = eventStore.storeBatch(list); if (ints.length > 0) { logger.error(format("Failed to save events: %s", Arrays.stream(ints).boxed().map(i -> i + "").collect(Collectors.joining(", ")))); } } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof Event)) { return false; } Event event = (Event) o; if (!project.equals(event.project)) { return false; } if (!collection.equals(event.collection)) { return false; } if (schema != null ? !schema.equals(event.schema) : event.schema != null) { return false; } if (api != null ? !api.equals(event.api) : event.api != null) { return false; } return properties.equals(event.properties); }
return new EventList(EventContext.apiKey(apiKey), project, events); } else if ("application/avro".equals(contentType)) { String apiKey = getParam(request.params(), MASTER_KEY.getKey());
record2.put("price", 1500.0); EventList eventList = new EventList(Event.EventContext.apiKey("apiKey"), "project", ImmutableList.of( new Event("project", "collection", null, ImmutableList.copyOf(collection), record1), new Event("project", "collection", null, ImmutableList.copyOf(collection), record2)));
@Test public void testPrimitiveTypes() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of( "test", 1L, "test1", false, "test2", Instant.now(), "test3", "test", "test4", LocalDate.now()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); assertEquals(ImmutableSet.copyOf(metastore.getCollection("test", "test")), ImmutableSet.of( new SchemaField("test", FieldType.DOUBLE), new SchemaField("_user", FieldType.STRING), new SchemaField("test1", FieldType.BOOLEAN), new SchemaField("test2", FieldType.TIMESTAMP), new SchemaField("test3", FieldType.STRING), new SchemaField("test4", FieldType.DATE))); }
@Test public void testBatchWithoutProject() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test0", "test", "test1", ImmutableList.of("test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "events", ImmutableList.of( ImmutableMap.of("collection", "test", "properties", props), ImmutableMap.of("collection", "test", "properties", props)))); EventList events = mapper.readValue(bytes, EventList.class); assertEquals("test", events.project); assertEquals(api, events.api); for (Event event : events.events) { assertEquals("test", event.collection()); assertEquals(eventBuilder.createEvent("test", props).properties(), event.properties()); } }
@Test public void testBatch() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test0", "test", "test1", ImmutableList.of("test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "events", ImmutableList.of( ImmutableMap.of("collection", "test", "properties", props), ImmutableMap.of("collection", "test", "properties", props)))); EventList events = mapper.readValue(bytes, EventList.class); assertEquals("test", events.project); assertEquals(api, events.api); for (Event event : events.events) { assertEquals("test", event.collection()); assertEquals(eventBuilder.createEvent("test", props).properties(), event.properties()); } }
@Test(expectedExceptions = RakamException.class) public void testInvalidField() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false), "test", "test" )); Event event = mapper.readValue(bytes, Event.class); ; assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testEmptyMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", new HashMap<String, String>() { { put("a", null); } }), "test20", ImmutableMap.of(), "test3", true)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
public Event createEvent(String collection, Map<String, Object> properties) { List<SchemaField> cache = fieldCache.get(collection); List<SchemaField> fields; List<SchemaField> generatedSchema = generateSchema(properties); if (cache == null || !generatedSchema.stream().allMatch(f -> cache.contains(f))) { fields = metastore.getOrCreateCollectionFields(project, collection, ImmutableSet.copyOf(generatedSchema)); fieldCache.put(collection, fields); } else { fields = cache; } try { GenericData.Record record = new GenericData.Record(AvroUtil.convertAvroSchema(fields)); properties.forEach((key, value) -> record.put(key, cast(value, record.getSchema().getField(key).schema().getTypes().get(1).getType()))); return new Event(project, collection, Event.EventContext.empty(), fields, record); } catch (Exception e) { throw Throwables.propagate(e); } }
@Test public void testMapType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableMap.of("a", 4.0, "b", 5.0, "c", 6.0, "d", 7.0), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test public void testArrayType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test public void testEmptyArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", Arrays.asList(null, null), "test20", Arrays.asList(), "test3", true))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Test public void testInvalidMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableMap.of("test", 1, "test2", "test")))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test1", ImmutableMap.of("test", 1.0, "test2", 0.0))).properties(), event.properties()); }
public EventList deserialize(String project, String collection, SliceInput slice) throws IOException { String json = slice.readSlice(slice.readInt()).toStringUtf8(); Schema schema = new Schema.Parser().parse(json); int records = slice.readInt(); BinaryDecoder binaryDecoder = DecoderFactory.get().directBinaryDecoder(slice, null); List<SchemaField> fields = metastore.getCollection(project, collection); Schema avroSchema = AvroUtil.convertAvroSchema(fields); GenericDatumReader<GenericRecord> reader = new GenericDatumReader(schema, avroSchema); List<Event> list = new ArrayList<>(records); for (int i = 0; i < records; i++) { GenericRecord record = reader.read(null, binaryDecoder); list.add(new Event(project, collection, null, fields, record)); } return new EventList(Event.EventContext.empty(), project, list); } }
@Test public void testSimple() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testScalarSentToObjectValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.ARRAY_BOOLEAN))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", "test"); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event event = mapper.readValue(bytes, Event.class); assertEquals(event.getAttribute("test"), ImmutableList.of(false)); }
@Test() public void testObjectSentToInvalidScalarValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.INTEGER))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", ImmutableList.of("test")); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event event = mapper.readValue(bytes, Event.class); assertNull(event.getAttribute("test")); }
@Test public void testObjectSentToScalarValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.STRING))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", ImmutableList.of("test")); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event events = mapper.readValue(bytes, Event.class); assertEquals(events.properties().get("test"), "[\"test\"]"); }