private CompletableFuture<SegmentHandle> getHandle() { SegmentHandle h = this.handle.get(); if (h != null) { return CompletableFuture.completedFuture(h); } else { return this.storage.openRead(this.segmentInfo.getName()) .thenApply(sh -> { this.handle.set(sh); return sh; }); } } }
private CompletableFuture<Void> validateConditionalUpdateFailures(DirectSegmentAccess segment, Map<TableKey, Long> expectedVersions, TimeoutTimer timer) { assert !expectedVersions.isEmpty(); val bucketReader = TableBucketReader.key(segment, this::getBackpointerOffset, this.executor); val searches = expectedVersions.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> bucketReader.find(e.getKey().getKey(), e.getValue(), timer))); return Futures .allOf(searches.values()) .thenRun(() -> { val failed = new HashMap<TableKey, Long>(); for (val e : searches.entrySet()) { val actual = e.getValue().join(); boolean isValid = actual == null ? e.getKey().getVersion() == TableKey.NOT_EXISTS : e.getKey().getVersion() == actual.getVersion(); if (!isValid) { failed.put(e.getKey(), actual == null ? TableKey.NOT_EXISTS : actual.getVersion()); } } if (!failed.isEmpty()) { throw new CompletionException(new BadKeyVersionException(segment.getInfo().getName(), failed)); } }); }
/** * Reads a range of bytes from a Segment in Storage. * * @param segmentInfo A SegmentProperties describing the Segment to read. * @param startOffset The first offset within the Segment to read from. * @param maxReadLength The maximum number of bytes to read. * @param readBlockSize The maximum number of bytes to read at once (the returned StreamSegmentReadResult will be * broken down into Entries smaller than or equal to this size). * @param storage A ReadOnlyStorage to execute the reads against. * @return A StreamSegmentReadResult that can be used to process the data. This will be made up of ReadResultEntries * of the following types: Storage, Truncated or EndOfSegment. */ public static StreamSegmentReadResult read(SegmentProperties segmentInfo, long startOffset, int maxReadLength, int readBlockSize, ReadOnlyStorage storage) { Exceptions.checkArgument(startOffset >= 0, "startOffset", "startOffset must be a non-negative number."); Exceptions.checkArgument(maxReadLength >= 0, "maxReadLength", "maxReadLength must be a non-negative number."); Preconditions.checkNotNull(segmentInfo, "segmentInfo"); Preconditions.checkNotNull(storage, "storage"); String traceId = String.format("Read[%s]", segmentInfo.getName()); return new StreamSegmentReadResult(startOffset, maxReadLength, new SegmentReader(segmentInfo, null, readBlockSize, storage), traceId); }
/** * Invokes the {@link Connector#getMapSegmentId()} callback in order to assign an Id to a Segment. Upon completion, * this operation will have mapped the given Segment to a new internal Segment Id if none was provided in the given * SegmentInfo. If the given SegmentInfo already has a SegmentId set, then all efforts will be made to map that Segment * with the requested Segment Id. * * @param segmentInfo The SegmentInfo for the StreamSegment to generate and persist. * @param pin If true, this Segment's metadata will be pinned to memory. * @param timeout Timeout for the operation. * @return A CompletableFuture that, when completed, will contain the internal SegmentId that was assigned (or the * one supplied via SegmentInfo, if any). If the operation failed, then this Future will complete with that exception. */ protected CompletableFuture<Long> submitAssignment(SegmentInfo segmentInfo, boolean pin, Duration timeout) { SegmentProperties properties = segmentInfo.getProperties(); if (properties.isDeleted()) { // Stream does not exist. Fail the request with the appropriate exception. failAssignment(properties.getName(), new StreamSegmentNotExistsException("StreamSegment does not exist.")); return Futures.failedFuture(new StreamSegmentNotExistsException(properties.getName())); } long existingSegmentId = this.connector.containerMetadata.getStreamSegmentId(properties.getName(), true); if (isValidSegmentId(existingSegmentId)) { // Looks like someone else beat us to it. completeAssignment(properties.getName(), existingSegmentId); return CompletableFuture.completedFuture(existingSegmentId); } else { return this.connector.getMapSegmentId() .apply(segmentInfo.getSegmentId(), segmentInfo.getProperties(), pin, timeout) .thenApply(id -> completeAssignment(properties.getName(), id)); } }
/** * Creates a new instance of the StreamSegmentMapOperation class for a non-transaction Segment. * * @param streamSegmentProperties Information about the StreamSegment. */ public StreamSegmentMapOperation(SegmentProperties streamSegmentProperties) { this.streamSegmentId = ContainerMetadata.NO_STREAM_SEGMENT_ID; this.streamSegmentName = streamSegmentProperties.getName(); this.startOffset = streamSegmentProperties.getStartOffset(); this.length = streamSegmentProperties.getLength(); this.sealed = streamSegmentProperties.isSealed(); this.attributes = streamSegmentProperties.getAttributes(); this.pinned = false; }
private void recordStatForTransaction(SegmentProperties sourceInfo, String targetSegmentName) { try { if (sourceInfo != null && sourceInfo.getAttributes().containsKey(Attributes.CREATION_TIME) && sourceInfo.getAttributes().containsKey(Attributes.EVENT_COUNT)) { long creationTime = sourceInfo.getAttributes().get(Attributes.CREATION_TIME); int numOfEvents = sourceInfo.getAttributes().get(Attributes.EVENT_COUNT).intValue(); long len = sourceInfo.getLength(); if (statsRecorder != null) { statsRecorder.merge(targetSegmentName, len, numOfEvents, creationTime); } // If source segment is a transaction, add its length and event count onto metrics of target segment if (StreamSegmentNameUtils.isTransactionSegment(sourceInfo.getName())) { dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_BYTES, targetSegmentName), sourceInfo.getLength()); dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_EVENTS, targetSegmentName), sourceInfo.getAttributes().get(EVENT_COUNT)); } } } catch (Exception ex) { // gobble up any errors from stat recording so we do not affect rest of the flow. log.error("exception while computing stats while merging txn {}", sourceInfo.getName(), ex); } }
.collect(Collectors.toList()); CompletableFuture<Void> result = getBucketOffsets(segment, hashes, timer) .thenAccept(offsets -> validateConditionalUpdate(batch.getVersionedItems(), offsets, segment.getInfo().getName())); return Futures.exceptionallyCompose( result,
@Override public CompletableFuture<DirectSegmentAccess> forSegment(String segmentName, Duration timeout) { Exceptions.checkNotClosed(this.closed.get(), this); SegmentMock segment = this.segment.get(); if (segment == null) { return Futures.failedFuture(new StreamSegmentNotExistsException(segmentName)); } Assert.assertEquals("Unexpected segment name.", segment.getInfo().getName(), segmentName); return CompletableFuture.supplyAsync(() -> segment, executorService()); }
@Override public CompletableFuture<Void> deleteStreamSegment(String segmentName, Duration timeout) { SegmentMock segment = this.segment.get(); if (segment == null) { return Futures.failedFuture(new StreamSegmentNotExistsException(segmentName)); } Assert.assertEquals("Unexpected segment name.", segment.getInfo().getName(), segmentName); Assert.assertTrue(this.segment.compareAndSet(segment, null)); return CompletableFuture.completedFuture(null); }
private void checkActiveSegments(SegmentContainer container, int expectedCount) { val initialActiveSegments = container.getActiveSegments(); int ignoredSegments = 0; for (SegmentProperties sp : initialActiveSegments) { if (sp.getName().equals(EXPECTED_METADATA_SEGMENT_NAME)) { ignoredSegments++; continue; } val expectedSp = container.getStreamSegmentInfo(sp.getName(), TIMEOUT).join(); Assert.assertEquals("Unexpected length (from getActiveSegments) for segment " + sp.getName(), expectedSp.getLength(), sp.getLength()); Assert.assertEquals("Unexpected sealed (from getActiveSegments) for segment " + sp.getName(), expectedSp.isSealed(), sp.isSealed()); Assert.assertEquals("Unexpected deleted (from getActiveSegments) for segment " + sp.getName(), expectedSp.isDeleted(), sp.isDeleted()); SegmentMetadataComparer.assertSameAttributes("Unexpected attributes (from getActiveSegments) for segment " + sp.getName(), expectedSp.getAttributes(), sp); } Assert.assertEquals("Unexpected result from getActiveSegments with freshly created segments.", expectedCount + ignoredSegments, initialActiveSegments.size()); }
private long getStartOffset(SegmentProperties segmentInfo, TestContext context) { if (segmentInfo.getLength() == 0) { return 0; } long offset = 0; val handle = context.storage.openRead(segmentInfo.getName()).join(); byte[] buffer = new byte[1]; while (offset < segmentInfo.getLength()) { try { context.storage.read(handle, offset, buffer, 0, buffer.length, TIMEOUT).join(); break; // If we make it here, we stumbled across a valid offset. } catch (Exception ex) { if (!(Exceptions.unwrap(ex) instanceof StreamSegmentTruncatedException)) { throw ex; } } offset += SEGMENT_ROLLING_SIZE; } return offset - SEGMENT_ROLLING_SIZE; }
private CompletableFuture<Void> waitForSegmentInStorage(SegmentProperties sp, StreamSegmentStore readOnlyStore) { if (sp.getLength() == 0) { // Empty segments may or may not exist in Storage, so don't bother complicating ourselves with this. return CompletableFuture.completedFuture(null); } TimeoutTimer timer = new TimeoutTimer(TIMEOUT); AtomicBoolean tryAgain = new AtomicBoolean(true); return Futures.loop( tryAgain::get, () -> Futures .exceptionallyExpecting(readOnlyStore.getStreamSegmentInfo(sp.getName(), TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException, StreamSegmentInformation.builder().name(sp.getName()).build()) .thenCompose(storageProps -> { if (sp.isSealed()) { tryAgain.set(!storageProps.isSealed()); } else { tryAgain.set(sp.getLength() != storageProps.getLength()); } if (tryAgain.get() && !timer.hasRemaining()) { return Futures.<Void>failedFuture(new TimeoutException( String.format("Segment %s did not complete in Storage in the allotted time.", sp.getName()))); } else { return Futures.delayedFuture(Duration.ofMillis(100), executorService()); } }), executorService()); }
private CompletableFuture<Void> waitForSegmentInStorage(SegmentProperties metadataProps, TestContext context) { if (metadataProps.getLength() == 0) { // Empty segments may or may not exist in Storage, so don't bother complicating ourselves with this. return CompletableFuture.completedFuture(null); } Function<SegmentProperties, Boolean> meetsConditions = storageProps -> storageProps.isSealed() == metadataProps.isSealed() && storageProps.getLength() >= metadataProps.getLength() && context.storageFactory.truncationOffsets.getOrDefault(metadataProps.getName(), 0L) >= metadataProps.getStartOffset(); AtomicBoolean canContinue = new AtomicBoolean(true); TimeoutTimer timer = new TimeoutTimer(TIMEOUT); return Futures.loop( canContinue::get, () -> Futures.exceptionallyExpecting( context.storage.getStreamSegmentInfo(metadataProps.getName(), TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException, StreamSegmentInformation.builder().name(metadataProps.getName()).build()) .thenCompose(storageProps -> { if (meetsConditions.apply(storageProps)) { canContinue.set(false); return CompletableFuture.completedFuture(null); } else if (!timer.hasRemaining()) { return Futures.failedFuture(new TimeoutException()); } else { return Futures.delayedFuture(Duration.ofMillis(10), executorService()); } }).thenRun(Runnables.doNothing()), executorService()); }
private void write00(SegmentInfo s, RevisionDataOutput output) throws IOException { output.writeLong(s.getSegmentId()); SegmentProperties sp = s.getProperties(); output.writeUTF(sp.getName()); output.writeLong(sp.getLength()); output.writeLong(sp.getStartOffset()); output.writeBoolean(sp.isSealed()); // We only serialize Core Attributes. Extended Attributes can be retrieved from the AttributeIndex. output.writeMap(Attributes.getCoreNonNullAttributes(sp.getAttributes()), RevisionDataOutput::writeUUID, RevisionDataOutput::writeLong); }
private void validateProperties(String stage, String segmentName, SegmentProperties sp, long expectedLength, boolean expectedSealed) { Assert.assertNotNull("No result from GetInfoOperation (" + stage + ").", sp); Assert.assertEquals("Unexpected name (" + stage + ").", segmentName, sp.getName()); Assert.assertEquals("Unexpected length (" + stage + ").", expectedLength, sp.getLength()); Assert.assertEquals("Unexpected sealed status (" + stage + ").", expectedSealed, sp.isSealed()); }
protected void assertEquals(String message, SegmentProperties expected, SegmentProperties actual) { Assert.assertEquals(message + " getName() mismatch.", expected.getName(), actual.getName()); Assert.assertEquals(message + " isDeleted() mismatch.", expected.isDeleted(), actual.isDeleted()); Assert.assertEquals(message + " getLength() mismatch.", expected.getLength(), actual.getLength()); Assert.assertEquals(message + " isSealed() mismatch.", expected.isSealed(), actual.isSealed()); Assert.assertEquals(message + " getStartOffset() mismatch.", expected.getStartOffset(), actual.getStartOffset()); }
@Override public void getStreamSegmentInfo(GetStreamSegmentInfo getStreamSegmentInfo) { String segmentName = getStreamSegmentInfo.getSegmentName(); final String operation = "getStreamSegmentInfo"; if (!verifyToken(segmentName, getStreamSegmentInfo.getRequestId(), getStreamSegmentInfo.getDelegationToken(), operation)) { return; } segmentStore.getStreamSegmentInfo(segmentName, TIMEOUT) .thenAccept(properties -> { if (properties != null) { StreamSegmentInfo result = new StreamSegmentInfo(getStreamSegmentInfo.getRequestId(), properties.getName(), true, properties.isSealed(), properties.isDeleted(), properties.getLastModified().getTime(), properties.getLength(), properties.getStartOffset()); log.trace("Read stream segment info: {}", result); connection.send(result); } else { log.trace("getStreamSegmentInfo could not find segment {}", segmentName); connection.send(new StreamSegmentInfo(getStreamSegmentInfo.getRequestId(), segmentName, false, true, true, 0, 0, 0)); } }) .exceptionally(e -> handleException(getStreamSegmentInfo.getRequestId(), segmentName, operation, e)); }
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
/** * Tests the getStreamSegmentInfo() method. */ @Test public void testGetStreamSegmentInfo() { @Cleanup val context = new TestContext(); context.container.startAsync().awaitRunning(); // Non-existent segment. AssertExtensions.assertSuppliedFutureThrows( "Unexpected exception when the segment does not exist.", () -> context.container.getStreamSegmentInfo(SEGMENT_NAME, TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException); // Create a segment, add some data, set some attributes, "truncate" it and then seal it. val storageInfo = context.storage.create(SEGMENT_NAME, TIMEOUT) .thenCompose(handle -> context.storage.write(handle, 0, new ByteArrayInputStream(new byte[10]), 10, TIMEOUT)) .thenCompose(v -> context.storage.getStreamSegmentInfo(SEGMENT_NAME, TIMEOUT)).join(); val expectedInfo = StreamSegmentInformation.from(storageInfo) .startOffset(storageInfo.getLength() / 2) .attributes(ImmutableMap.of(UUID.randomUUID(), 100L, Attributes.EVENT_COUNT, 1L)) .build(); // Fetch the SegmentInfo from the ReadOnlyContainer and verify it is as expected. val actual = context.container.getStreamSegmentInfo(SEGMENT_NAME, TIMEOUT).join(); Assert.assertEquals("Unexpected Name.", expectedInfo.getName(), actual.getName()); Assert.assertEquals("Unexpected Length.", expectedInfo.getLength(), actual.getLength()); Assert.assertEquals("Unexpected Sealed status.", expectedInfo.isSealed(), actual.isSealed()); }
private StreamSegmentMetadata toMetadata(long segmentId, SegmentProperties sp) { StreamSegmentMetadata sm = new StreamSegmentMetadata(sp.getName(), segmentId, CONTAINER_ID); sm.updateAttributes(sp.getAttributes()); sm.setLength(sp.getLength()); sm.setStartOffset(sp.getStartOffset()); if (sp.isSealed()) { sm.markSealed(); } if (sp.isDeleted()) { sm.markDeleted(); } return sm; }