@Override public String collection() { return event.collection(); }
private String getPartitionKey(Event event) { Object user = event.getAttribute("_user"); return event.project() + "|" + (user == null ? event.collection() : user.toString()); }
@Override public void sync() { Event message = streamHolder.messageQueue.poll(); StringBuilder builder = new StringBuilder("["); boolean isFirst = true; while (message != null) { if (!isFirst) { builder.append(","); } builder.append("{\"project\":") .append(JsonHelper.encode(message.project())) .append(", \"collection\":") .append(JsonHelper.encode(message.collection())) .append(", \"properties\": ") .append(message.properties().toString()).append("}"); isFirst = false; message = streamHolder.messageQueue.poll(); } builder.append("]"); response.send("data", builder.toString()); }
if (item.collection != null && !item.collection.equals(event.collection())) { continue;
public void store(Event event, boolean partitionCheckDone) { GenericRecord record = event.properties(); try (Connection connection = connectionPool.getConnection()) { Schema schema = event.properties().getSchema(); PreparedStatement ps = connection.prepareStatement(getQuery(event.project(), event.collection(), schema)); bindParam(connection, ps, event.schema(), record); ps.executeUpdate(); } catch (SQLException e) { // check_violation -> https://www.postgresql.org/docs/8.2/static/errcodes-appendix.html if (version.getVersion() == PG10 && !partitionCheckDone && "23514".equals(e.getSQLState())) { generateMissingPartitions(event.project(), event.collection(), ImmutableList.of(event), 0); store(event, true); } else { throw new RuntimeException(e); } } catch (Exception e) { throw new RuntimeException(e); } }
private ByteBuf getBuffer(Event event) { DatumWriter writer = new FilteredRecordWriter(event.properties().getSchema(), GenericData.get()); ByteBuf buffer = DEFAULT.buffer(100); buffer.writeByte(2); BinaryEncoder encoder = EncoderFactory.get() .directBinaryEncoder(new ByteBufOutputStream(buffer), null); try { encoder.writeString(event.collection()); writer.write(event.properties(), encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } return buffer; } }
@Override public void store(Event event) { GenericDatumWriter writer = new SourceFilteredRecordWriter(event.properties().getSchema(), GenericData.get(), sourceFields); ByteBuf buffer = Unpooled.buffer(100); BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder( new ByteBufOutputStream(buffer), null); try { writer.write(event.properties(), encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } try { producer.send(new KeyedMessage<>(event.project() + "_" + event.collection(), buffer.array())); } catch (FailedToSendMessageException e) { throw new RuntimeException("Couldn't send event to Kafka", e); } }
@Test public void testInvalidArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableList.of(true, 10)))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder.createEvent("test", ImmutableMap.of("test1", ImmutableList.of(true, true))).properties(), event.properties()); }
@Test public void testBatch() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test0", "test", "test1", ImmutableList.of("test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "events", ImmutableList.of( ImmutableMap.of("collection", "test", "properties", props), ImmutableMap.of("collection", "test", "properties", props)))); EventList events = mapper.readValue(bytes, EventList.class); assertEquals("test", events.project); assertEquals(api, events.api); for (Event event : events.events) { assertEquals("test", event.collection()); assertEquals(eventBuilder.createEvent("test", props).properties(), event.properties()); } }
@Test public void testBatchWithoutProject() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test0", "test", "test1", ImmutableList.of("test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "events", ImmutableList.of( ImmutableMap.of("collection", "test", "properties", props), ImmutableMap.of("collection", "test", "properties", props)))); EventList events = mapper.readValue(bytes, EventList.class); assertEquals("test", events.project); assertEquals(api, events.api); for (Event event : events.events) { assertEquals("test", event.collection()); assertEquals(eventBuilder.createEvent("test", props).properties(), event.properties()); } }
@Test public void testInvalidMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableMap.of("test", 1, "test2", "test")))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test1", ImmutableMap.of("test", 1.0, "test2", 0.0))).properties(), event.properties()); }
@Test public void testSimpleWithoutProject() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testEmptyArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", Arrays.asList(null, null), "test20", Arrays.asList(), "test3", true))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Test public void testSimple() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testArrayType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test public void testMapType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableMap.of("a", 4.0, "b", 5.0, "c", 6.0, "d", 7.0), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test(expectedExceptions = RakamException.class) public void testInvalidField() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false), "test", "test" )); Event event = mapper.readValue(bytes, Event.class); ; assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testEmptyMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", new HashMap<String, String>() { { put("a", null); } }), "test20", ImmutableMap.of(), "test3", true)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Override public int[] storeBatch(List<Event> events) { for (Map.Entry<String, List<Event>> collection : events.stream().collect(Collectors.groupingBy(e -> e.collection())).entrySet()) { QueryResult join = queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s (_shard_time, %s) (%s)", config.getColdStorageConnector(), checkProject(events.get(0).project(), '"'), checkCollection(collection.getKey()), collection.getValue().get(0).schema().stream().map(e -> ValidationUtil.checkCollection(e.getName())) .collect(Collectors.joining(", ")), collection.getValue().stream() .map(e -> buildValues(e.properties(), e.schema())) .collect(Collectors.joining(" union all ")))) .getResult().join(); if (join.isFailed()) { try { Thread.sleep(300000); } catch (InterruptedException e) { e.printStackTrace(); } throw new IllegalStateException(join.getError().message); } } return EventStore.SUCCESSFUL_BATCH; }
@Override public void store(Event event) { queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s VALUES %s", config.getColdStorageConnector(), event.project(), event.collection(), buildValues(event.properties(), event.schema()))); }