public <T> T getAttribute(String attr) { return (T) properties().get(attr); }
@Override public List<Cookie> map(Event event, RequestParams extraProperties, InetAddress sourceAddress, HttpHeaders responseHeaders) { Object referrer = event.properties().get("_referrer"); Object host = event.properties().get("_host"); mapInternal(extraProperties, referrer, host, event.properties()); return null; }
@Override public List<Cookie> map(Event event, RequestParams requestParams, InetAddress sourceAddress, HttpHeaders responseHeaders) { if (event.properties().get("_user") == null) { throw new RakamException("_user cannot be null", BAD_REQUEST); } return null; }
@Override public List<Cookie> map(Event event, RequestParams extraProperties, InetAddress sourceAddress, HttpHeaders responseHeaders) { GenericRecord properties = event.properties(); mapInternal(extraProperties, properties, properties.get("_user_agent")); return null; }
@Override public List<Cookie> map(Event event, RequestParams requestParams, InetAddress sourceAddress, HttpHeaders responseHeaders) { GenericRecord properties = event.properties(); if (properties.get("_user") == null) { Schema.Field user = event.properties().getSchema().getField("_user"); if (user == null) { return null; } Schema.Type type = user.schema().getTypes().get(1).getType(); Object anonymousUser = requestParams.cookies().stream() .filter(e -> e.name().equals("_anonymous_user")).findAny() .map(e -> cast(type, e.value())).orElse(generate(type)); properties.put("_user", anonymousUser); DefaultCookie cookie = new DefaultCookie("_anonymous_user", String.valueOf(anonymousUser)); cookie.setPath("/"); return ImmutableList.of(cookie); } return null; }
private ByteBuf getBuffer(Event event) { DatumWriter writer = new FilteredRecordWriter(event.properties().getSchema(), GenericData.get()); ByteBuf buffer = DEFAULT.buffer(100); buffer.writeByte(2); BinaryEncoder encoder = EncoderFactory.get() .directBinaryEncoder(new ByteBufOutputStream(buffer), null); try { encoder.writeString(event.collection()); writer.write(event.properties(), encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } return buffer; } }
@Override public List<Cookie> map(Event event, RequestParams extraProperties, InetAddress sourceAddress, HttpHeaders responseHeaders) { GenericRecord properties = event.properties(); Object time = properties.get(projectConfig.getTimeColumn()); if (time == null) { long serverTime = Instant.now().getEpochSecond(); properties.put(projectConfig.getTimeColumn(), serverTime * 1000); } else if (time instanceof Number && event.api() != null && event.api().uploadTime != null) { // match server time and client time and get an estimate long fixedTime = ((Number) time).longValue() + ((Instant.now().getEpochSecond() - (event.api().uploadTime / 1000)) * 1000); properties.put(projectConfig.getTimeColumn(), fixedTime); } return null; }
public void store(Event event, boolean partitionCheckDone) { GenericRecord record = event.properties(); try (Connection connection = connectionPool.getConnection()) { Schema schema = event.properties().getSchema(); PreparedStatement ps = connection.prepareStatement(getQuery(event.project(), event.collection(), schema)); bindParam(connection, ps, event.schema(), record); ps.executeUpdate(); } catch (SQLException e) { // check_violation -> https://www.postgresql.org/docs/8.2/static/errcodes-appendix.html if (version.getVersion() == PG10 && !partitionCheckDone && "23514".equals(e.getSQLState())) { generateMissingPartitions(event.project(), event.collection(), ImmutableList.of(event), 0); store(event, true); } else { throw new RuntimeException(e); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void store(Event event) { GenericDatumWriter writer = new SourceFilteredRecordWriter(event.properties().getSchema(), GenericData.get(), sourceFields); ByteBuf buffer = Unpooled.buffer(100); BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder( new ByteBufOutputStream(buffer), null); try { writer.write(event.properties(), encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } try { producer.send(new KeyedMessage<>(event.project() + "_" + event.collection(), buffer.array())); } catch (FailedToSendMessageException e) { throw new RuntimeException("Couldn't send event to Kafka", e); } }
@Test public void testInvalidArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableList.of(true, 10)))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder.createEvent("test", ImmutableMap.of("test1", ImmutableList.of(true, true))).properties(), event.properties()); }
@Test public void testObjectSentToScalarValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.STRING))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> props = ImmutableMap.of( "test", ImmutableList.of("test")); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event events = mapper.readValue(bytes, Event.class); assertEquals(events.properties().get("test"), "[\"test\"]"); }
@Test public void testSimple() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testSimpleWithoutProject() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of())); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of()).properties(), event.properties()); }
@Test public void testInvalidMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test1", ImmutableMap.of("test", 1, "test2", "test")))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test1", ImmutableMap.of("test", 1.0, "test2", 0.0))).properties(), event.properties()); }
@Test public void testNullSentToObjectValue() throws Exception { metastore.getOrCreateCollectionFields("test", "test", ImmutableSet.of(new SchemaField("test", FieldType.ARRAY_BOOLEAN))); Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); HashMap<String, Object> props = new HashMap<>(); props.put("test", null); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "api", api, "collection", "test", "properties", props)); Event event = mapper.readValue(bytes, Event.class); assertNull(event.properties().get("test")); } }
@Test public void testEmptyArray() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", Arrays.asList(null, null), "test20", Arrays.asList(), "test3", true))); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Test public void testMapType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableMap.of("a", 4.0, "b", 5.0, "c", 6.0, "d", 7.0), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test public void testArrayType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); ImmutableMap<String, Object> properties = ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", "test"), "test2", false); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", properties)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", properties).properties(), event.properties()); }
@Test public void testEmptyMap() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test", 1, "test2", new HashMap<String, String>() { { put("a", null); } }), "test20", ImmutableMap.of(), "test3", true)); Event event = mapper.readValue(bytes, Event.class); assertEquals("test", event.project()); assertEquals("test", event.collection()); assertEquals(api, event.api()); assertEquals(eventBuilder .createEvent("test", ImmutableMap.of("test", 1.0, "test3", true)).properties(), event.properties()); }
@Override public void store(Event event) { queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s VALUES %s", config.getColdStorageConnector(), event.project(), event.collection(), buildValues(event.properties(), event.schema()))); }