/** * Reads the bytes of a single event from the segment. Buffering is performed internally to try to prevent * blocking. If there is no event after timeout, then it returns null. EndOfSegmentException indicates the * segment has ended and there are no more events to read. * * @return A ByteBuffer containing the serialized data that was written via * {@link EventStreamWriter#writeEvent(String, Object)} * @throws EndOfSegmentException If no event could be read because the end of the segment was reached. * @throws SegmentTruncatedException If the segment has been truncated beyond the current offset and the data cannot be read. */ public default ByteBuffer read() throws EndOfSegmentException, SegmentTruncatedException { return read(Long.MAX_VALUE); }
@GuardedBy("readers") private void releaseSegmentsIfNeeded() throws ReaderNotInReaderGroupException { Segment segment = groupState.findSegmentToReleaseIfRequired(); if (segment != null) { log.info("{} releasing segment {}", this, segment); EventSegmentReader reader = readers.stream().filter(r -> r.getSegmentId().equals(segment)).findAny().orElse(null); if (reader != null) { if (groupState.releaseSegment(segment, reader.getOffset(), getLag())) { readers.remove(reader); reader.close(); } } } }
public SegmentIteratorImpl(SegmentInputStreamFactory factory, Segment segment, Serializer<T> deserializer, long startingOffset, long endingOffset) { this.segment = segment; this.deserializer = deserializer; this.startingOffset = startingOffset; this.endingOffset = endingOffset; input = factory.createEventReaderForSegment(segment); input.setOffset(startingOffset); }
private void handleEndOfSegment(EventSegmentReader oldSegment, boolean fetchSuccessors) throws ReaderNotInReaderGroupException { log.info("{} encountered end of segment {} ", this, oldSegment.getSegmentId()); readers.remove(oldSegment); oldSegment.close(); groupState.handleEndOfSegment(oldSegment.getSegmentId(), fetchSuccessors); }
private PositionInternal getPosition() { Map<Segment, Long> positions = readers.stream() .collect(Collectors.toMap(e -> e.getSegmentId(), e -> e.getOffset())); return new PositionImpl(positions); }
buffer = null; } else { segment = segmentReader.getSegmentId(); offset = segmentReader.getOffset(); try { buffer = segmentReader.read(waitTime); } catch (EndOfSegmentException e) { boolean fetchSuccessors = e.getErrorType().equals(ErrorType.END_OF_SEGMENT_REACHED);
Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new EndOfSegmentException(EndOfSegmentException.ErrorType.END_OFFSET_REACHED)); Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment); Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer); Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0")); Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L); Mockito.verify(segmentInputStream1, Mockito.times(1)).close();
EventSegmentReader segmentInputStream = Mockito.mock(EventSegmentReader.class); Mockito.when(segmentMetadataClientFactory.createSegmentMetadataClient(any(Segment.class), any())).thenReturn(metadataClient); Mockito.when(segmentInputStream.getSegmentId()).thenReturn(segment); Mockito.when(segInputStreamFactory.createEventReaderForSegment(any(Segment.class), anyLong())).thenReturn(segmentInputStream); Mockito.when(segmentInputStream.read(anyLong())).thenThrow(SegmentTruncatedException.class); Mockito.verify(segmentInputStream, Mockito.times(1)).close();
/** * Given a list of segments this reader owns, (which contain their positions) returns the one * that should be read from next. This is done in way to minimize blocking and ensure fairness. * * This is done by calling {@link EventSegmentReader#isSegmentReady()} on each segment. * This method will prefer to return streams where that method is true. This method should * reflect that the next call to {@link EventSegmentReader#read()} will not block (either * because it has data or will throw {@link EndOfSegmentException} * * @param <T> The type of the SegmentInputStream that is being selected from. * @param segments The logs to get the next reader for. * @return A segment that this reader should read from next. */ @VisibleForTesting public <T extends EventSegmentReader> T nextSegment(List<T> segments) { if (segments.isEmpty()) { return null; } for (int i = 0; i < segments.size(); i++) { T inputStream = segments.get(MathHelpers.abs(counter.incrementAndGet()) % segments.size()); if (inputStream.isSegmentReady()) { log.trace("Selecting segment: " + inputStream.getSegmentId()); return inputStream; } else { inputStream.fillBuffer(); } } return segments.get(MathHelpers.abs(counter.incrementAndGet()) % segments.size()); } }
@Override public Type fetchEvent(EventPointer pointer) throws NoSuchEventException { Preconditions.checkNotNull(pointer); // Create SegmentInputStream @Cleanup EventSegmentReader inputStream = inputStreamFactory.createEventReaderForSegment(pointer.asImpl().getSegment(), pointer.asImpl().getEventLength()); inputStream.setOffset(pointer.asImpl().getEventStartOffset()); // Read event try { ByteBuffer buffer = inputStream.read(); Type result = deserializer.deserialize(buffer); return result; } catch (EndOfSegmentException e) { throw new NoSuchEventException(e.getMessage()); } catch (NoSuchSegmentException | SegmentTruncatedException e) { throw new NoSuchEventException("Event no longer exists."); } }
private void handleSegmentTruncated(EventSegmentReader segmentReader) throws ReaderNotInReaderGroupException, TruncatedDataException { Segment segmentId = segmentReader.getSegmentId(); log.info("{} encountered truncation for segment {} ", this, segmentId); String delegationToken = groupState.getOrRefreshDelegationTokenFor(segmentId); @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segmentId, delegationToken); try { long startingOffset = metadataClient.getSegmentInfo().getStartingOffset(); segmentReader.setOffset(startingOffset); } catch (NoSuchSegmentException e) { handleEndOfSegment(segmentReader, true); } throw new TruncatedDataException(); }
@Override public void close() { input.close(); }
@Override public boolean hasNext() { return input.getOffset() < endingOffset; }
List<EventSegmentReader> readers = reader.getReaders(); assertEquals(1, readers.size()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); Assert.assertEquals(segment2, readers.get(1).getSegmentId()); reader.close();
Mockito.when(segmentInputStream1.read(anyLong())).thenThrow(new SegmentTruncatedException()); Mockito.when(segmentInputStream1.getSegmentId()).thenReturn(segment); Mockito.when(segmentInputStream2.read(anyLong())).thenReturn(buffer); Mockito.when(segmentInputStream2.getSegmentId()).thenReturn(Segment.fromScopedName("Foo/test/0")); Mockito.when(segmentInputStream2.getOffset()).thenReturn(10L);
@Override public void close() { synchronized (readers) { if (!closed) { log.info("Closing reader {} ", this); closed = true; groupState.readerShutdown(getPosition()); for (EventSegmentReader reader : readers) { reader.close(); } readers.clear(); groupState.close(); } } }
@Override public long getOffset() { return input.getOffset(); }
List<EventSegmentReader> readers = reader.getReaders(); assertEquals(2, readers.size()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); Assert.assertEquals(segment2, readers.get(1).getSegmentId()); readers = reader.getReaders(); assertEquals(1, readers.size()); Assert.assertEquals(segment1, readers.get(0).getSegmentId()); reader.close();
@Override @SneakyThrows(EndOfSegmentException.class) //endingOffset should make this impossible. public T next() { if (!hasNext()) { throw new NoSuchElementException(); } try { return deserializer.deserialize(input.read()); } catch (NoSuchSegmentException | SegmentTruncatedException e) { throw new TruncatedDataException("Segment " + segment + " has been truncated."); } }
@Override public void close() { synchronized (lock) { try { out.close(); } catch (SegmentSealedException e) { log.warn("Error closing segment writer {}", out); } conditional.close(); meta.close(); in.close(); } } }