@Override public Iterator<Entry<Revision, T>> readFrom(Revision start) { synchronized (lock) { long startOffset = start.asImpl().getOffsetInSegment(); SegmentInfo segmentInfo = meta.getSegmentInfo(); long endOffset = segmentInfo.getWriteOffset(); if (startOffset < segmentInfo.getStartingOffset()) { throw new TruncatedDataException("Data at the supplied revision has been truncated."); } log.trace("Creating iterator from {} until {}", startOffset, endOffset); return new StreamIterator(startOffset, endOffset); } }
@Override @Synchronized public SegmentInfo getSegmentInfo() { return new SegmentInfo(segment, startingOffset, writeOffset, false, System.currentTimeMillis()); }
@Test(timeout = 5000) public void testTruncate() { MockSegmentStreamFactory factory = new MockSegmentStreamFactory(); Segment segment = new Segment("Scope", "Stream", 1); EventWriterConfig config = EventWriterConfig.builder().build(); SegmentOutputStream outputStream = factory.createOutputStreamForSegment(segment, c -> { }, config, ""); sendData("1", outputStream); sendData("2", outputStream); sendData("3", outputStream); SegmentMetadataClient metadataClient = factory.createSegmentMetadataClient(segment, ""); long length = metadataClient.getSegmentInfo().getWriteOffset(); @Cleanup SegmentIteratorImpl<String> iter = new SegmentIteratorImpl<>(factory, segment, stringSerializer, 0, length); assertEquals("1", iter.next()); long segmentLength = metadataClient.fetchCurrentSegmentLength(); assertEquals(0, segmentLength % 3); metadataClient.truncateSegment(segmentLength * 2 / 3); AssertExtensions.assertThrows(TruncatedDataException.class, () -> iter.next()); @Cleanup SegmentIteratorImpl<String> iter2 = new SegmentIteratorImpl<>(factory, segment, stringSerializer, segmentLength * 2 / 3, length); assertTrue(iter2.hasNext()); assertEquals("3", iter2.next()); assertFalse(iter.hasNext()); }
@Override public Revision fetchOldestRevision() { long startingOffset = meta.getSegmentInfo().getStartingOffset(); return new RevisionImpl(segment, startingOffset, 0); }
@Test(timeout = 5000) public void testHasNext() { MockSegmentStreamFactory factory = new MockSegmentStreamFactory(); Segment segment = new Segment("Scope", "Stream", 1); EventWriterConfig config = EventWriterConfig.builder().build(); SegmentOutputStream outputStream = factory.createOutputStreamForSegment(segment, c -> { }, config, ""); sendData("1", outputStream); sendData("2", outputStream); sendData("3", outputStream); SegmentMetadataClient metadataClient = factory.createSegmentMetadataClient(segment, ""); long length = metadataClient.getSegmentInfo().getWriteOffset(); @Cleanup SegmentIteratorImpl<String> iter = new SegmentIteratorImpl<>(factory, segment, stringSerializer, 0, length); assertTrue(iter.hasNext()); assertTrue(iter.hasNext()); assertEquals("1", iter.next()); assertEquals("2", iter.next()); assertTrue(iter.hasNext()); assertEquals("3", iter.next()); assertFalse(iter.hasNext()); assertThrows(NoSuchElementException.class, () -> iter.next()); assertFalse(iter.hasNext()); }
private void handleSegmentTruncated(EventSegmentReader segmentReader) throws ReaderNotInReaderGroupException, TruncatedDataException { Segment segmentId = segmentReader.getSegmentId(); log.info("{} encountered truncation for segment {} ", this, segmentId); String delegationToken = groupState.getOrRefreshDelegationTokenFor(segmentId); @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segmentId, delegationToken); try { long startingOffset = metadataClient.getSegmentInfo().getStartingOffset(); segmentReader.setOffset(startingOffset); } catch (NoSuchSegmentException e) { handleEndOfSegment(segmentReader, true); } throw new TruncatedDataException(); }
private SegmentRange getSegmentRange(final Segment segment, final StreamCut startStreamCut, final StreamCut endStreamCut) { SegmentRangeImpl.SegmentRangeImplBuilder segmentRangeBuilder = SegmentRangeImpl.builder() .segment(segment); if (startStreamCut.asImpl().getPositions().containsKey(segment) && endStreamCut.asImpl().getPositions().containsKey(segment)) { //use the meta data present in startStreamCut and endStreamCuts. segmentRangeBuilder.startOffset(startStreamCut.asImpl().getPositions().get(segment)) .endOffset(endStreamCut.asImpl().getPositions().get(segment)); } else { //use segment meta data client to fetch the segment offsets. SegmentInfo r = segmentToInfo(segment); segmentRangeBuilder.startOffset(startStreamCut.asImpl().getPositions().getOrDefault(segment, r.getStartingOffset())) .endOffset(endStreamCut.asImpl().getPositions().getOrDefault(segment, r.getWriteOffset())); } return segmentRangeBuilder.build(); }
@Test(timeout = 5000) public void testOffset() { MockSegmentStreamFactory factory = new MockSegmentStreamFactory(); Segment segment = new Segment("Scope", "Stream", 1); EventWriterConfig config = EventWriterConfig.builder().build(); SegmentOutputStream outputStream = factory.createOutputStreamForSegment(segment, c -> { }, config, ""); sendData("1", outputStream); sendData("2", outputStream); sendData("3", outputStream); SegmentMetadataClient metadataClient = factory.createSegmentMetadataClient(segment, ""); long length = metadataClient.getSegmentInfo().getWriteOffset(); @Cleanup SegmentIteratorImpl<String> iter = new SegmentIteratorImpl<>(factory, segment, stringSerializer, 0, length); assertEquals(0, iter.getOffset()); assertEquals("1", iter.next()); assertEquals(length / 3, iter.getOffset()); assertEquals("2", iter.next()); assertEquals(length / 3 * 2, iter.getOffset()); assertTrue(iter.hasNext()); assertEquals(length / 3 * 2, iter.getOffset()); assertEquals("3", iter.next()); assertEquals(length, iter.getOffset()); assertThrows(NoSuchElementException.class, () -> iter.next()); assertFalse(iter.hasNext()); assertEquals(length, iter.getOffset()); }
@Override public SegmentInfo getSegmentInfo() { val future = RETRY_SCHEDULE.retryingOn(ConnectionFailedException.class) .throwingOn(NoSuchSegmentException.class) .runAsync(() -> getStreamSegmentInfo(delegationToken), connectionFactory.getInternalExecutor()); StreamSegmentInfo info = Futures.getThrowingException(future); return new SegmentInfo(segmentId, info.getStartOffset(), info.getWriteOffset(), info.isSealed(), info.getLastModified()); }
Segment segment = new Segment(scope, streamName, 0); SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segment, ""); assertEquals(0, metadataClient.getSegmentInfo().getStartingOffset()); long writeOffset = metadataClient.getSegmentInfo().getWriteOffset(); assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength()); assertTrue(metadataClient.getSegmentInfo().getWriteOffset() > testString.length()); metadataClient.truncateSegment(writeOffset); assertEquals(writeOffset, metadataClient.getSegmentInfo().getStartingOffset()); assertEquals(writeOffset, metadataClient.getSegmentInfo().getWriteOffset()); assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength());