@Override public CompletableFuture<List<Cookie>> mapAsync(Event event, RequestParams requestParams, InetAddress sourceAddress, HttpHeaders responseHeaders) { return mapInternal(event.project(), new EventsProxy() { @Override public Event.EventContext api() { return event.api(); } @Override public String project() { return event.project(); } @Override public Iterator<EventProxy> events() { return Iterators.singletonIterator(new ListEventProxy(event)); } }, requestParams, sourceAddress, responseHeaders); }
private String getPartitionKey(Event event) { Object user = event.getAttribute("_user"); return event.project() + "|" + (user == null ? event.collection() : user.toString()); }
@Override public List<Cookie> map(Event event, RequestParams requestParams, InetAddress sourceAddress, HttpHeaders responseHeaders) { List<CollectionStreamHolder> streamHolder = lists.get(event.project());
@Override public void sync() { Event message = streamHolder.messageQueue.poll(); StringBuilder builder = new StringBuilder("["); boolean isFirst = true; while (message != null) { if (!isFirst) { builder.append(","); } builder.append("{\"project\":") .append(JsonHelper.encode(message.project())) .append(", \"collection\":") .append(JsonHelper.encode(message.collection())) .append(", \"properties\": ") .append(message.properties().toString()).append("}"); isFirst = false; message = streamHolder.messageQueue.poll(); } builder.append("]"); response.send("data", builder.toString()); }
@Override public User get() { if (user == null) { String userAttr = event.getAttribute("_user"); if (userAttr != null) { user = userStorage.getUser(new RequestContext(event.project(), null), userAttr).join(); } } return user; } };
@Override public void storeBulk(List<Event> events) { if (events.isEmpty()) { return; } String project = events.get(0).project(); try { bulkClient.upload(project, events, 3); } catch (OutOfMemoryError e) { LOGGER.error(e, "OOM error while uploading bulk"); throw new RakamException("Too much data", HttpResponseStatus.BAD_REQUEST); } catch (Throwable e) { LOGGER.error(e); throw new RakamException("An error occurred while storing events", INTERNAL_SERVER_ERROR); } }
public void store(Event event, boolean partitionCheckDone) { GenericRecord record = event.properties(); try (Connection connection = connectionPool.getConnection()) { Schema schema = event.properties().getSchema(); PreparedStatement ps = connection.prepareStatement(getQuery(event.project(), event.collection(), schema)); bindParam(connection, ps, event.schema(), record); ps.executeUpdate(); } catch (SQLException e) { // check_violation -> https://www.postgresql.org/docs/8.2/static/errcodes-appendix.html if (version.getVersion() == PG10 && !partitionCheckDone && "23514".equals(e.getSQLState())) { generateMissingPartitions(event.project(), event.collection(), ImmutableList.of(event), 0); store(event, true); } else { throw new RuntimeException(e); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void store(Event event) { GenericDatumWriter writer = new SourceFilteredRecordWriter(event.properties().getSchema(), GenericData.get(), sourceFields); ByteBuf buffer = Unpooled.buffer(100); BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder( new ByteBufOutputStream(buffer), null); try { writer.write(event.properties(), encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } try { producer.send(new KeyedMessage<>(event.project() + "_" + event.collection(), buffer.array())); } catch (FailedToSendMessageException e) { throw new RuntimeException("Couldn't send event to Kafka", e); } }
@Test public void testInvalidArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableList.of(true, 10)))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder.createEvent("test", ImmutableMap.of("test1", ImmutableList.of(true, true))).properties(), event.properties()); }
@Test public void testInvalidMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableMap.of("test", 1, "test2", "test")))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test1", ImmutableMap.of("test", 1.0, "test2", 0.0))).properties(), event.properties()); }
@Test public void testEmptyArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", Arrays.asList(null, null), "test20", Arrays.asList(), "test3", true))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Test public void testSimple() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testSimpleWithoutProject() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testMapType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableMap.of("a", 4.0, "b", 5.0, "c", 6.0, "d", 7.0), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test public void testArrayType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test(expectedExceptions = RakamException.class) public void testInvalidField() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false), "test", "test" )); Event event = mapper.readValue(bytes, Event.class); ; assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testEmptyMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", new HashMap<String, String>() { { put("a", null); } }), "test20", ImmutableMap.of(), "test3", true)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Override public int[] storeBatch(List<Event> events) { for (Map.Entry<String, List<Event>> collection : events.stream().collect(Collectors.groupingBy(e -> e.collection())).entrySet()) { QueryResult join = queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s (_shard_time, %s) (%s)", config.getColdStorageConnector(), checkProject(events.get(0).project(), '"'), checkCollection(collection.getKey()), collection.getValue().get(0).schema().stream().map(e -> ValidationUtil.checkCollection(e.getName())) .collect(Collectors.joining(", ")), collection.getValue().stream() .map(e -> buildValues(e.properties(), e.schema())) .collect(Collectors.joining(" union all ")))) .getResult().join(); if (join.isFailed()) { try { Thread.sleep(300000); } catch (InterruptedException e) { e.printStackTrace(); } throw new IllegalStateException(join.getError().message); } } return EventStore.SUCCESSFUL_BATCH; }
@Test public void testPrimitiveTypes() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of( "test", 1L, "test1", false, "test2", Instant.now(), "test3", "test", "test4", LocalDate.now()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); assertEquals(ImmutableSet.copyOf(metastore.getCollection("test", "test")), ImmutableSet.of( new SchemaField("test", FieldType.DOUBLE), new SchemaField("_user", FieldType.STRING), new SchemaField("test1", FieldType.BOOLEAN), new SchemaField("test2", FieldType.TIMESTAMP), new SchemaField("test3", FieldType.STRING), new SchemaField("test4", FieldType.DATE))); }
@Override public void store(Event event) { queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s VALUES %s", config.getColdStorageConnector(), event.project(), event.collection(), buildValues(event.properties(), event.schema()))); }