private void handleEndOfSegment(EventSegmentReader oldSegment, boolean fetchSuccessors) throws ReaderNotInReaderGroupException { log.info("{} encountered end of segment {} ", this, oldSegment.getSegmentId()); readers.remove(oldSegment); oldSegment.close(); groupState.handleEndOfSegment(oldSegment.getSegmentId(), fetchSuccessors); }
private PositionInternal getPosition() { Map<Segment, Long> positions = readers.stream() .collect(Collectors.toMap(e -> e.getSegmentId(), e -> e.getOffset())); return new PositionImpl(positions); }
@GuardedBy("readers") private void releaseSegmentsIfNeeded() throws ReaderNotInReaderGroupException { Segment segment = groupState.findSegmentToReleaseIfRequired(); if (segment != null) { log.info("{} releasing segment {}", this, segment); EventSegmentReader reader = readers.stream().filter(r -> r.getSegmentId().equals(segment)).findAny().orElse(null); if (reader != null) { if (groupState.releaseSegment(segment, reader.getOffset(), getLag())) { readers.remove(reader); reader.close(); } } } }
/** * Given a list of segments this reader owns, (which contain their positions) returns the one * that should be read from next. This is done in way to minimize blocking and ensure fairness. * * This is done by calling {@link EventSegmentReader#isSegmentReady()} on each segment. * This method will prefer to return streams where that method is true. This method should * reflect that the next call to {@link EventSegmentReader#read()} will not block (either * because it has data or will throw {@link EndOfSegmentException} * * @param <T> The type of the SegmentInputStream that is being selected from. * @param segments The logs to get the next reader for. * @return A segment that this reader should read from next. */ @VisibleForTesting public <T extends EventSegmentReader> T nextSegment(List<T> segments) { if (segments.isEmpty()) { return null; } for (int i = 0; i < segments.size(); i++) { T inputStream = segments.get(MathHelpers.abs(counter.incrementAndGet()) % segments.size()); if (inputStream.isSegmentReady()) { log.trace("Selecting segment: " + inputStream.getSegmentId()); return inputStream; } else { inputStream.fillBuffer(); } } return segments.get(MathHelpers.abs(counter.incrementAndGet()) % segments.size()); } }
private void handleSegmentTruncated(EventSegmentReader segmentReader) throws ReaderNotInReaderGroupException, TruncatedDataException { Segment segmentId = segmentReader.getSegmentId(); log.info("{} encountered truncation for segment {} ", this, segmentId); String delegationToken = groupState.getOrRefreshDelegationTokenFor(segmentId); @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segmentId, delegationToken); try { long startingOffset = metadataClient.getSegmentInfo().getStartingOffset(); segmentReader.setOffset(startingOffset); } catch (NoSuchSegmentException e) { handleEndOfSegment(segmentReader, true); } throw new TruncatedDataException(); }
EventSegmentReader segmentInputStream = Mockito.mock(EventSegmentReader.class); Mockito.when(segmentMetadataClientFactory.createSegmentMetadataClient(any(Segment.class), any())).thenReturn(metadataClient); Mockito.when(segmentInputStream.getSegmentId()).thenReturn(segment); Mockito.when(segInputStreamFactory.createEventReaderForSegment(any(Segment.class), anyLong())).thenReturn(segmentInputStream);
EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class); Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OFFSET_REACHED)); Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment); ByteBuffer buffer = writeInt(stream, 1); Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer); Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0")); Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
EventSegmentReader segmentInputStream1 = Mockito.mock(EventSegmentReader.class); Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new SegmentTruncatedException()); Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment); ByteBuffer buffer = writeInt(stream, 1); Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer); Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0")); Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
buffer = null; } else { segment = segmentReader.getSegmentId(); offset = segmentReader.getOffset(); try {
List<EventSegmentReader> readers = reader.getReaders(); assertEquals(1, readers.size()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); Assert.assertEquals(segment2, readers.get(1).getSegmentId()); reader.close();
List<EventSegmentReader> readers = reader.getReaders(); assertEquals(2, readers.size()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); Assert.assertEquals(segment2, readers.get(1).getSegmentId()); readers = reader.getReaders(); assertEquals(1, readers.size()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); reader.close();