@Override public long fetchTailOffset() { return meta.fetchCurrentSegmentLength(); }
private SegmentInfo segmentToInfo(Segment s) { @Cleanup SegmentMetadataClient client = segmentMetadataClientFactory.createSegmentMetadataClient(s, latestDelegationToken.get()); return client.getSegmentInfo(); } }
@Override public void truncateToRevision(Revision newStart) { meta.truncateSegment(newStart.asImpl().getOffsetInSegment()); }
@Override public void closeAndSeal() throws IOException { out.close(); meta.sealSegment(); meta.close(); }
Segment segment = new Segment(scope, streamName, 0); SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segment, ""); assertEquals(0, metadataClient.getSegmentInfo().getStartingOffset()); long writeOffset = metadataClient.getSegmentInfo().getWriteOffset(); assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength()); assertTrue(metadataClient.getSegmentInfo().getWriteOffset() > testString.length()); metadataClient.truncateSegment(writeOffset); assertEquals(writeOffset, metadataClient.getSegmentInfo().getStartingOffset()); assertEquals(writeOffset, metadataClient.getSegmentInfo().getWriteOffset()); assertEquals(writeOffset, metadataClient.fetchCurrentSegmentLength());
ByteBuffer buffer2 = writeInt(stream, 2); ByteBuffer buffer3 = writeInt(stream, 3); long length = metadataClient.fetchCurrentSegmentLength(); assertEquals(0, length % 3); EventRead<byte[]> event1 = reader.readNextEvent(0); assertEquals(buffer1, ByteBuffer.wrap(event1.getEvent())); metadataClient.truncateSegment(length / 3); assertEquals(buffer2, ByteBuffer.wrap(reader.readNextEvent(0).getEvent())); metadataClient.truncateSegment(length); ByteBuffer buffer4 = writeInt(stream, 4); AssertExtensions.assertThrows(TruncatedDataException.class, () -> reader.readNextEvent(0));
@Override public void close() throws IOException { out.close(); meta.close(); }
@Override public Revision getMark() { synchronized (lock) { long value = meta.fetchProperty(RevisionStreamClientMark); return value == NULL_VALUE ? null : new RevisionImpl(segment, value, 0); } }
@Override public boolean compareAndSetMark(Revision expected, Revision newLocation) { long expectedValue = expected == null ? NULL_VALUE : expected.asImpl().getOffsetInSegment(); long newValue = newLocation == null ? NULL_VALUE : newLocation.asImpl().getOffsetInSegment(); synchronized (lock) { return meta.compareAndSetAttribute(RevisionStreamClientMark, expectedValue, newValue); } }
@Test(timeout = 5000) public void testTruncate() { MockSegmentStreamFactory factory = new MockSegmentStreamFactory(); Segment segment = new Segment("Scope", "Stream", 1); EventWriterConfig config = EventWriterConfig.builder().build(); SegmentOutputStream outputStream = factory.createOutputStreamForSegment(segment, c -> { }, config, ""); sendData("1", outputStream); sendData("2", outputStream); sendData("3", outputStream); SegmentMetadataClient metadataClient = factory.createSegmentMetadataClient(segment, ""); long length = metadataClient.getSegmentInfo().getWriteOffset(); @Cleanup SegmentIteratorImpl<String> iter = new SegmentIteratorImpl<>(factory, segment, stringSerializer, 0, length); assertEquals("1", iter.next()); long segmentLength = metadataClient.fetchCurrentSegmentLength(); assertEquals(0, segmentLength % 3); metadataClient.truncateSegment(segmentLength * 2 / 3); AssertExtensions.assertThrows(TruncatedDataException.class, () -> iter.next()); @Cleanup SegmentIteratorImpl<String> iter2 = new SegmentIteratorImpl<>(factory, segment, stringSerializer, segmentLength * 2 / 3, length); assertTrue(iter2.hasNext()); assertEquals("3", iter2.next()); assertFalse(iter.hasNext()); }
@Override public void close() { synchronized (lock) { try { out.close(); } catch (SegmentSealedException e) { log.warn("Error closing segment writer {}", out); } conditional.close(); meta.close(); in.close(); } } }
private SegmentInfo segmentToInfo(Segment s) { String delegationToken; synchronized (this) { delegationToken = latestDelegationToken.get(); } @Cleanup SegmentMetadataClient client = segmentMetadataClientFactory.createSegmentMetadataClient(s, delegationToken); return client.getSegmentInfo(); }
@Override public long fetchTailOffset() { return meta.fetchCurrentSegmentLength(); }
@Override public Revision fetchOldestRevision() { long startingOffset = meta.getSegmentInfo().getStartingOffset(); return new RevisionImpl(segment, startingOffset, 0); }
@Override public Revision fetchLatestRevision() { synchronized (lock) { long streamLength = meta.fetchCurrentSegmentLength(); return new RevisionImpl(segment, streamLength, 0); } }
@Override public Iterator<Entry<Revision, T>> readFrom(Revision start) { synchronized (lock) { long startOffset = start.asImpl().getOffsetInSegment(); SegmentInfo segmentInfo = meta.getSegmentInfo(); long endOffset = segmentInfo.getWriteOffset(); if (startOffset < segmentInfo.getStartingOffset()) { throw new TruncatedDataException("Data at the supplied revision has been truncated."); } log.trace("Creating iterator from {} until {}", startOffset, endOffset); return new StreamIterator(startOffset, endOffset); } }
private long getRemainingBytes(SegmentMetadataClientFactory metaFactory, StreamCut fromStreamCut, StreamCut toStreamCut) { long totalLength = 0; //fetch StreamSegmentSuccessors final CompletableFuture<StreamSegmentSuccessors> unread; final Map<Segment, Long> endPositions; if (toStreamCut.equals(StreamCut.UNBOUNDED)) { unread = controller.getSuccessors(fromStreamCut); endPositions = Collections.emptyMap(); } else { unread = controller.getSegments(fromStreamCut, toStreamCut); endPositions = toStreamCut.asImpl().getPositions(); } StreamSegmentSuccessors unreadVal = Futures.getAndHandleExceptions(unread, RuntimeException::new); //compute remaining bytes. for (Segment s : unreadVal.getSegments()) { if (endPositions.containsKey(s)) { totalLength += endPositions.get(s); } else { @Cleanup SegmentMetadataClient metadataClient = metaFactory.createSegmentMetadataClient(s, unreadVal.getDelegationToken()); totalLength += metadataClient.fetchCurrentSegmentLength(); } } for (long bytesRead : fromStreamCut.asImpl().getPositions().values()) { totalLength -= bytesRead; } log.debug("Remaining bytes from position: {} to position: {} is {}", fromStreamCut, toStreamCut, totalLength); return totalLength; }
private void handleSegmentTruncated(EventSegmentReader segmentReader) throws ReaderNotInReaderGroupException, TruncatedDataException { Segment segmentId = segmentReader.getSegmentId(); log.info("{} encountered truncation for segment {} ", this, segmentId); String delegationToken = groupState.getOrRefreshDelegationTokenFor(segmentId); @Cleanup SegmentMetadataClient metadataClient = metadataClientFactory.createSegmentMetadataClient(segmentId, delegationToken); try { long startingOffset = metadataClient.getSegmentInfo().getStartingOffset(); segmentReader.setOffset(startingOffset); } catch (NoSuchSegmentException e) { handleEndOfSegment(segmentReader, true); } throw new TruncatedDataException(); }
ack2.get(5, TimeUnit.SECONDS); SegmentMetadataClient metadataClient = new SegmentMetadataClientFactoryImpl(controller, clientCF).createSegmentMetadataClient(segment, ""); assertEquals(payload.length * 2, metadataClient.fetchCurrentSegmentLength());
Mockito.when(metadataClient.getSegmentInfo()).thenThrow(NoSuchSegmentException.class);