/** * Updates cursors in case the currently stored offset is no longer available. Streaming will start at the oldest available offset (BEGIN) to minimize the amount of events skipped. */ public LowLevelStreamBuilder skipUnavailableOffsets(List<Partition> partitions) throws IOException { final Map<String, Cursor> cursorsByPartition = cursorManager.getCursors(eventName).stream().collect(toMap(Cursor::getPartition, identity())); final List<Cursor> cursors = partitions.stream().filter(p -> isNoLongerAvailable(cursorsByPartition, p)).map(p -> new Cursor(p.getPartition(), "BEGIN")).collect(toList()); if (!cursors.isEmpty()) { cursorManager.onSuccess(eventName, cursors); } return this; }
/** * Initializes offsets to start streaming at the oldest available offset (BEGIN). */ public LowLevelStreamBuilder readFromBegin(List<Partition> partitions) throws IOException { final List<Cursor> cursors = partitions.stream().map(p -> new Cursor(p.getPartition(), "BEGIN")).collect(toList()); cursorManager.onSuccess(eventName, cursors); return this; }
private Collection<Cursor> getLockedCursors() throws IOException { final Collection<Cursor> cursors = cursorManager.getCursors(eventNames.iterator().next()); if (lock.isPresent()) { final Map<String, String> offsets = cursors.stream().collect(toMap(Cursor::getPartition, Cursor::getOffset)); final List<Partition> partitions = lock.get().getPartitions(); return partitions.stream().map(partition -> new Cursor(partition.getPartition(), offsets.getOrDefault(partition.getPartition(), "BEGIN"))).collect(toList()); } else { return cursors; } }
private JsonInput openJsonInput() throws IOException { final String cursorsHeader = getCursorsHeader(); final Request request = requestFactory.createRequest(uri, "GET"); if (cursorsHeader != null) { request.getHeaders().put("X-Nakadi-Cursors", cursorsHeader); } final Response response = request.execute(); try { final Optional<String> streamId = getStreamId(response); if (subscription.isPresent() && streamId.isPresent()) { cursorManager.addStreamId(subscription.get(), streamId.get()); } return new JsonInput(jsonFactory, response); } catch (Throwable throwable) { try { response.close(); } catch (Throwable suppressed) { throwable.addSuppressed(suppressed); } throw throwable; } }
Subscription subscribe(String applicationName, Set<String> eventNames, String consumerGroup, SubscriptionRequest.Position readFrom, @Nullable List<Cursor> initialCursors) throws IOException { checkArgument(readFrom != SubscriptionRequest.Position.CURSORS || (initialCursors != null && !initialCursors.isEmpty()), "Initial cursors are required for position: cursors"); final SubscriptionRequest subscription = new SubscriptionRequest(applicationName, eventNames, consumerGroup, readFrom, initialCursors); final URI uri = baseUri.resolve("/subscriptions"); final Request request = clientHttpRequestFactory.createRequest(uri, "POST"); request.getHeaders().setContentType(ContentType.APPLICATION_JSON); try (final OutputStream os = request.getBody()) { internalObjectMapper.writeValue(os, subscription); } try (final Response response = request.execute()) { try (final InputStream is = response.getBody()) { final Subscription subscriptionResponse = internalObjectMapper.readValue(is, Subscription.class); LOG.info("Created subscription for event {} with id [{}]", subscription.getEventTypes(), subscriptionResponse.getId()); cursorManager.addSubscription(subscriptionResponse); return subscriptionResponse; } } }
/** * Updates cursors to the newest available offset. * This is similar to the default behaviour, but allows to specify the partitions to stream from instead of getting events from all partitions. */ public LowLevelStreamBuilder readFromNewestAvailableOffset(List<Partition> partitions) throws IOException { final List<Cursor> cursors = partitions.stream().map(p -> new Cursor(p.getPartition(), p.getNewestAvailableOffset())).collect(toList()); cursorManager.onSuccess(eventName, cursors); return this; }
@Test public void shouldBeEmptyByDefault() throws IOException { final Collection<Cursor> cursors = cursorManager().getCursors("test"); assertNotNull(cursors); assertTrue(cursors.isEmpty()); }
@Test public void shouldCreateCursorOnSuccess() throws IOException { cursorManager().onSuccess("test", new Cursor("0", "123")); final Collection<Cursor> cursors = cursorManager().getCursors("test"); assertNotNull(cursors); assertEquals(1, cursors.size()); final Cursor cursor = cursors.iterator().next(); assertEquals("0", cursor.getPartition()); assertEquals("123", cursor.getOffset()); }
@Override public void run() throws IOException { try { listener.accept(batch.getEvents()); cursorManager.onSuccess(eventName, cursor); } catch (EventAlreadyProcessedException e) { LOG.info("Events for [{}] partition [{}] at offset [{}] were already processed", eventName, cursor.getPartition(), cursor.getOffset()); } catch (Throwable throwable) { LOG.warn("Exception while processing events for [{}] on partition [{}] at offset [{}]", eventName, cursor.getPartition(), cursor.getOffset(), throwable); throw throwable; } } });
@Test public void shouldUpdateCursorOnSuccess() throws IOException { cursorManager().onSuccess("test", new Cursor("0", "123")); { final Collection<Cursor> cursors = cursorManager().getCursors("test"); assertNotNull(cursors); assertEquals(1, cursors.size()); final Cursor cursor = cursors.iterator().next(); assertEquals("0", cursor.getPartition()); assertEquals("123", cursor.getOffset()); } cursorManager().onSuccess("test", new Cursor("0", "124")); { final Collection<Cursor> cursors = cursorManager().getCursors("test"); assertNotNull(cursors); assertEquals(1, cursors.size()); final Cursor cursor = cursors.iterator().next(); assertEquals("0", cursor.getPartition()); assertEquals("124", cursor.getOffset()); } }
@Test public void shouldCreateCursorsForMultiplePartitions() throws IOException { cursorManager().onSuccess("test", new Cursor("0", "12")); cursorManager().onSuccess("test", new Cursor("1", "13")); final List<Cursor> cursors = new ArrayList<>(cursorManager().getCursors("test")); Collections.sort(cursors, Comparator.comparing(Cursor::getPartition)); assertNotNull(cursors); assertEquals(2, cursors.size()); { final Cursor cursor = cursors.get(0); assertEquals("0", cursor.getPartition()); assertEquals("12", cursor.getOffset()); } { final Cursor cursor = cursors.get(1); assertEquals("1", cursor.getPartition()); assertEquals("13", cursor.getOffset()); } }