/** * Creates a new instance of the StreamSegmentMapOperation class for a non-transaction Segment. * * @param streamSegmentProperties Information about the StreamSegment. */ public StreamSegmentMapOperation(SegmentProperties streamSegmentProperties) { this.streamSegmentId = ContainerMetadata.NO_STREAM_SEGMENT_ID; this.streamSegmentName = streamSegmentProperties.getName(); this.startOffset = streamSegmentProperties.getStartOffset(); this.length = streamSegmentProperties.getLength(); this.sealed = streamSegmentProperties.isSealed(); this.attributes = streamSegmentProperties.getAttributes(); this.pinned = false; }
@SneakyThrows private Void attemptReconcile(Throwable ex, String segmentName, Duration timeout) { ex = Exceptions.unwrap(ex); boolean reconciled = false; if (isPossibleEndOfSegment(ex)) { // If we get a Sealed/Merged/NotExists exception, verify that the segment really is in that state. try { SegmentProperties sp = this.streamSegmentStore.getStreamSegmentInfo(segmentName, timeout) .get(timeout.toMillis(), TimeUnit.MILLISECONDS); reconciled = sp.isSealed() || sp.isDeleted(); } catch (Throwable ex2) { reconciled = isPossibleEndOfSegment(Exceptions.unwrap(ex2)); } } if (reconciled) { return null; } else { throw ex; } }
private void checkIfEmptyAndNotSealed(StreamSegmentExistsException ex, String chunkName) throws StreamSegmentException { // SegmentChunk exists, check if it's empty and not sealed. try { val si = this.baseStorage.getStreamSegmentInfo(chunkName); if (si.getLength() > 0 || si.isSealed()) { throw ex; } } catch (StreamSegmentNotExistsException notExists) { // nothing to do. } }
@Override public CompletableReadResultEntry apply(Long readOffset, Integer readLength) { if (readOffset < this.segmentInfo.getStartOffset()) { // We attempted to read from a truncated portion of the Segment. return new TruncatedReadResultEntry(readOffset, readLength, this.segmentInfo.getStartOffset()); } else if (readOffset >= this.segmentInfo.getLength()) { // We've reached the end of a Sealed Segment. return new EndOfStreamSegmentReadResultEntry(readOffset, readLength); } else { // Execute the read from Storage. if (readOffset + readLength > segmentInfo.getLength()) { // Adjust max read length to not exceed the last known offset of the Segment. readLength = (int) (segmentInfo.getLength() - readOffset); } readLength = Math.min(this.readBlockSize, readLength); return new StorageReadResultEntry(readOffset, readLength, this::fetchContents); } }
private void validateProperties(String stage, String segmentName, SegmentProperties sp, long expectedLength, boolean expectedSealed) { Assert.assertNotNull("No result from GetInfoOperation (" + stage + ").", sp); Assert.assertEquals("Unexpected name (" + stage + ").", segmentName, sp.getName()); Assert.assertEquals("Unexpected length (" + stage + ").", expectedLength, sp.getLength()); Assert.assertEquals("Unexpected sealed status (" + stage + ").", expectedSealed, sp.isSealed()); }
@Override public String toString() { return String.format("Metadata.StorageLength = %d, Storage.Length = %d", this.initialStorageLength, this.storageInfo.getLength()); } }
private void recordStatForTransaction(SegmentProperties sourceInfo, String targetSegmentName) { try { if (sourceInfo != null && sourceInfo.getAttributes().containsKey(Attributes.CREATION_TIME) && sourceInfo.getAttributes().containsKey(Attributes.EVENT_COUNT)) { long creationTime = sourceInfo.getAttributes().get(Attributes.CREATION_TIME); int numOfEvents = sourceInfo.getAttributes().get(Attributes.EVENT_COUNT).intValue(); long len = sourceInfo.getLength(); if (statsRecorder != null) { statsRecorder.merge(targetSegmentName, len, numOfEvents, creationTime); } // If source segment is a transaction, add its length and event count onto metrics of target segment if (StreamSegmentNameUtils.isTransactionSegment(sourceInfo.getName())) { dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_BYTES, targetSegmentName), sourceInfo.getLength()); dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_EVENTS, targetSegmentName), sourceInfo.getAttributes().get(EVENT_COUNT)); } } } catch (Exception ex) { // gobble up any errors from stat recording so we do not affect rest of the flow. log.error("exception while computing stats while merging txn {}", sourceInfo.getName(), ex); } }
sp.isSealed(), storageProps.isSealed()); storageProps.getLength()); byte[] actualData = new byte[expectedData.length]; int actualLength = 0; } catch (Exception ex) { ex = (Exception) Exceptions.unwrap(ex); if (!(ex instanceof StreamSegmentTruncatedException) || metadataProps.getStartOffset() == 0) { expectedLength = (int) (storageProps.getLength() - metadataProps.getStartOffset()); if (metadataProps.getStartOffset() < storageProps.getLength()) { @Cleanup ReadResult readResult = readOnlySegmentStore.read(segmentName, metadataProps.getStartOffset(), expectedLength, TIMEOUT).join(); actualLength = readResult.readRemaining(actualData, TIMEOUT);
private long getStartOffset(SegmentProperties segmentInfo, TestContext context) { if (segmentInfo.getLength() == 0) { return 0; } long offset = 0; val handle = context.storage.openRead(segmentInfo.getName()).join(); byte[] buffer = new byte[1]; while (offset < segmentInfo.getLength()) { try { context.storage.read(handle, offset, buffer, 0, buffer.length, TIMEOUT).join(); break; // If we make it here, we stumbled across a valid offset. } catch (Exception ex) { if (!(Exceptions.unwrap(ex) instanceof StreamSegmentTruncatedException)) { throw ex; } } offset += SEGMENT_ROLLING_SIZE; } return offset - SEGMENT_ROLLING_SIZE; }
long expectedLength = lengths.get(segmentName); Assert.assertEquals("Unexpected StartOffset for segment " + segmentName, expectedStartOffset, sp.getStartOffset()); Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength, sp.getLength()); Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted()); Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted());
private CompletableFuture<SegmentHandle> getHandle() { SegmentHandle h = this.handle.get(); if (h != null) { return CompletableFuture.completedFuture(h); } else { return this.storage.openRead(this.segmentInfo.getName()) .thenApply(sh -> { this.handle.set(sh); return sh; }); } } }
Assert.assertEquals("Unexpected length for segment " + segmentName, expectedLength.get(), sp.getLength()); Assert.assertFalse("Unexpected value for isDeleted for segment " + segmentName, sp.isDeleted()); Assert.assertFalse("Unexpected value for isSealed for segment " + segmentName, sp.isDeleted()); expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeAccumulate, Attributes.NULL_ATTRIBUTE_VALUE));
/** * Invokes the {@link Connector#getMapSegmentId()} callback in order to assign an Id to a Segment. Upon completion, * this operation will have mapped the given Segment to a new internal Segment Id if none was provided in the given * SegmentInfo. If the given SegmentInfo already has a SegmentId set, then all efforts will be made to map that Segment * with the requested Segment Id. * * @param segmentInfo The SegmentInfo for the StreamSegment to generate and persist. * @param pin If true, this Segment's metadata will be pinned to memory. * @param timeout Timeout for the operation. * @return A CompletableFuture that, when completed, will contain the internal SegmentId that was assigned (or the * one supplied via SegmentInfo, if any). If the operation failed, then this Future will complete with that exception. */ protected CompletableFuture<Long> submitAssignment(SegmentInfo segmentInfo, boolean pin, Duration timeout) { SegmentProperties properties = segmentInfo.getProperties(); if (properties.isDeleted()) { // Stream does not exist. Fail the request with the appropriate exception. failAssignment(properties.getName(), new StreamSegmentNotExistsException("StreamSegment does not exist.")); return Futures.failedFuture(new StreamSegmentNotExistsException(properties.getName())); } long existingSegmentId = this.connector.containerMetadata.getStreamSegmentId(properties.getName(), true); if (isValidSegmentId(existingSegmentId)) { // Looks like someone else beat us to it. completeAssignment(properties.getName(), existingSegmentId); return CompletableFuture.completedFuture(existingSegmentId); } else { return this.connector.getMapSegmentId() .apply(segmentInfo.getSegmentId(), segmentInfo.getProperties(), pin, timeout) .thenApply(id -> completeAssignment(properties.getName(), id)); } }
protected void verifyFinalWriteOperationsFail(SegmentHandle handle, Storage storage) { AssertExtensions.assertSuppliedFutureThrows( "Seal was allowed on fenced Storage.", () -> storage.seal(handle, TIMEOUT), ex -> ex instanceof StorageNotPrimaryException); val si = storage.getStreamSegmentInfo(handle.getSegmentName(), TIMEOUT).join(); Assert.assertFalse("Segment was sealed after rejected call to seal.", si.isSealed()); AssertExtensions.assertSuppliedFutureThrows( "Delete was allowed on fenced Storage.", () -> storage.delete(handle, TIMEOUT), ex -> ex instanceof StorageNotPrimaryException); boolean exists = storage.exists(handle.getSegmentName(), TIMEOUT).join(); Assert.assertTrue("Segment was deleted after rejected call to delete.", exists); }
long lastIndexedOffset = context.segment().getInfo().getAttributes().get(Attributes.TABLE_INDEX_OFFSET); long segmentLength = context.segment().getInfo().getLength(); AssertExtensions.assertGreaterThan("Expected some unindexed data.", lastIndexedOffset, segmentLength);
/** * Gets the offset (from the given {@link SegmentProperties}'s Attributes up to which all Table Entries have been indexed. * * @param segmentInfo A {@link SegmentProperties} from which to extract the information. * @return The offset. */ long getLastIndexedOffset(SegmentProperties segmentInfo) { return segmentInfo.getAttributes().getOrDefault(Attributes.TABLE_INDEX_OFFSET, 0L); }
Map<UUID, Long> expectedAttributes = segmentState == null ? null : segmentState.getAttributes(); SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in metadata for StreamSegment " + name, expectedAttributes, sm); long expectedStartOffset = segmentState == null ? 0 : segmentState.getStartOffset(); Assert.assertEquals("Unexpected StartOffset in metadata for " + name, expectedStartOffset, sm.getStartOffset());
connection.send(new SegmentIsTruncated(nonCachedEntry.getStreamSegmentOffset(), segment, info.getStartOffset(), EMPTY_STACK_TRACE))) .exceptionally(e -> handleException(nonCachedEntry.getStreamSegmentOffset(), segment, operation, wrapCancellationException(e))); } else {
Assert.assertFalse("Not-deleted segment was marked as deleted in metadata.", props.isDeleted());
private RollingSegmentHandle readHeader(SegmentProperties headerInfo, SegmentHandle headerHandle) throws StreamSegmentException { byte[] readBuffer = new byte[(int) headerInfo.getLength()]; this.baseStorage.read(headerHandle, 0, readBuffer, 0, readBuffer.length); RollingSegmentHandle handle = HandleSerializer.deserialize(readBuffer, headerHandle); if (headerInfo.isSealed()) { handle.markSealed(); } return handle; }