private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) { for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) { String segmentName = e.getKey(); byte[] expectedData = e.getValue().toByteArray(); long segmentLength = store.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength(); Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, segmentLength); AtomicLong expectedCurrentOffset = new AtomicLong(0); // We retry a number of times on StreamSegmentNotExists. It is possible that waitForSegmentsInStorage may have // returned successfully because it detected the Segment was complete there, but the internal callback to the // ReadIndex (completeMerge) may not yet have been executed. The ReadIndex has a mechanism to cope with this, // but it only retries once, after a fixed time interval, which is more than generous on any system. // However, on very slow systems, it is possible that that callback may take a significant amount of time to even // begin executing, hence the trying to read data that was merged from a Transaction may result in a spurious // StreamSegmentNotExistsException. // This is gracefully handled by retries in AppendProcessor and/or Client, but in this case, we simply have to // do the retries ourselves, hoping that the callback eventually executes. Retry.withExpBackoff(100, 2, 10, TIMEOUT.toMillis() / 5) .retryWhen(ex -> Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) .run(() -> { checkSegmentReads(segmentName, expectedCurrentOffset, segmentLength, store, expectedData); return null; }); } }