/** * Gets the offset (from the given {@link SegmentProperties}'s Attributes up to which all Table Entries have been indexed. * * @param segmentInfo A {@link SegmentProperties} from which to extract the information. * @return The offset. */ long getLastIndexedOffset(SegmentProperties segmentInfo) { return segmentInfo.getAttributes().getOrDefault(Attributes.TABLE_INDEX_OFFSET, 0L); }
/** * Gets the number of Table Entries indexed in the Segment for the given {@link SegmentProperties}. * * @param segmentInfo A {@link SegmentProperties} from which to extract the information. * @return The count. */ long getEntryCount(SegmentProperties segmentInfo) { return segmentInfo.getAttributes().getOrDefault(Attributes.TABLE_ENTRY_COUNT, 0L); }
/** * Gets the number of Table Buckets in the Segment for the given {@link SegmentProperties}. * * @param segmentInfo A {@link SegmentProperties} from which to extract the information. * @return The count. */ long getBucketCount(SegmentProperties segmentInfo) { return segmentInfo.getAttributes().getOrDefault(Attributes.TABLE_BUCKET_COUNT, 0L); }
private void loadAsynchronously(String streamSegmentName) { if (!pendingCacheLoads.contains(streamSegmentName)) { pendingCacheLoads.add(streamSegmentName); if (store != null) { store.getStreamSegmentInfo(streamSegmentName, TIMEOUT) .thenAcceptAsync(prop -> { if (prop != null && prop.getAttributes().containsKey(Attributes.SCALE_POLICY_TYPE) && prop.getAttributes().containsKey(Attributes.SCALE_POLICY_RATE)) { byte type = prop.getAttributes().get(Attributes.SCALE_POLICY_TYPE).byteValue(); int rate = prop.getAttributes().get(Attributes.SCALE_POLICY_RATE).intValue(); cache.put(streamSegmentName, new SegmentAggregates(type, rate)); } pendingCacheLoads.remove(streamSegmentName); }, executor); } } }
private void recordStatForTransaction(SegmentProperties sourceInfo, String targetSegmentName) { try { if (sourceInfo != null && sourceInfo.getAttributes().containsKey(Attributes.CREATION_TIME) && sourceInfo.getAttributes().containsKey(Attributes.EVENT_COUNT)) { long creationTime = sourceInfo.getAttributes().get(Attributes.CREATION_TIME); int numOfEvents = sourceInfo.getAttributes().get(Attributes.EVENT_COUNT).intValue(); long len = sourceInfo.getLength(); if (statsRecorder != null) { statsRecorder.merge(targetSegmentName, len, numOfEvents, creationTime); } // If source segment is a transaction, add its length and event count onto metrics of target segment if (StreamSegmentNameUtils.isTransactionSegment(sourceInfo.getName())) { dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_BYTES, targetSegmentName), sourceInfo.getLength()); dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_EVENTS, targetSegmentName), sourceInfo.getAttributes().get(EVENT_COUNT)); } } } catch (Exception ex) { // gobble up any errors from stat recording so we do not affect rest of the flow. log.error("exception while computing stats while merging txn {}", sourceInfo.getName(), ex); } }
/** * Creates a new instance of the StreamSegmentMapOperation class for a non-transaction Segment. * * @param streamSegmentProperties Information about the StreamSegment. */ public StreamSegmentMapOperation(SegmentProperties streamSegmentProperties) { this.streamSegmentId = ContainerMetadata.NO_STREAM_SEGMENT_ID; this.streamSegmentName = streamSegmentProperties.getName(); this.startOffset = streamSegmentProperties.getStartOffset(); this.length = streamSegmentProperties.getLength(); this.sealed = streamSegmentProperties.isSealed(); this.attributes = streamSegmentProperties.getAttributes(); this.pinned = false; }
/** * Verifies that the given SegmentMetadata instance has the expected Attributes. * * @param message The message to include in the assertion failures. * @param expected The expected set of Attributes. * @param toCheck The SegmentProperties instance to verify. */ public static void assertSameAttributes(String message, Map<UUID, Long> expected, SegmentProperties toCheck) { val actual = toCheck.getAttributes(); if (expected == null) { Assert.assertEquals(message + " No attributes expected.", 0, actual.size()); return; } Assert.assertEquals(message + " Counts differ.", expected.size(), actual.size()); for (Map.Entry<UUID, Long> e : expected.entrySet()) { Assert.assertTrue(" attribute not found " + e.getKey(), actual.containsKey(e.getKey())); long actualValue = actual.get(e.getKey()); Assert.assertEquals(message + " value differs.", (long) e.getValue(), actualValue); } } }
/** * Tests the ability of the MetadataStore to create a new Segment if the Segment already exists. */ @Test public void testCreateSegmentAlreadyExists() { final String segmentName = "NewSegment"; final Map<UUID, Long> originalAttributes = ImmutableMap.of(UUID.randomUUID(), 123L, Attributes.EVENT_COUNT, 1L); final Map<UUID, Long> expectedAttributes = Attributes.getCoreNonNullAttributes(originalAttributes); final Collection<AttributeUpdate> correctAttributeUpdates = originalAttributes.entrySet().stream() .map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())) .collect(Collectors.toList()); final Map<UUID, Long> badAttributes = Collections.singletonMap(UUID.randomUUID(), 456L); final Collection<AttributeUpdate> badAttributeUpdates = badAttributes.entrySet().stream() .map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())) .collect(Collectors.toList()); @Cleanup TestContext context = createTestContext(); // Create a segment. context.getMetadataStore().createSegment(segmentName, correctAttributeUpdates, TIMEOUT).join(); // Try to create it again. AssertExtensions.assertSuppliedFutureThrows( "createSegment did not fail when Segment already exists.", () -> context.getMetadataStore().createSegment(segmentName, badAttributeUpdates, TIMEOUT), ex -> ex instanceof StreamSegmentExistsException); val si = context.getMetadataStore().getSegmentInfo(segmentName, TIMEOUT).join(); AssertExtensions.assertMapEquals("Unexpected attributes after failed attempt to recreate correctly created segment", expectedAttributes, si.getAttributes()); }
@Override public void getSegmentAttribute(GetSegmentAttribute getSegmentAttribute) { long requestId = getSegmentAttribute.getRequestId(); String segmentName = getSegmentAttribute.getSegmentName(); UUID attributeId = getSegmentAttribute.getAttributeId(); final String operation = "getSegmentAttribute"; if (!verifyToken(segmentName, getSegmentAttribute.getRequestId(), getSegmentAttribute.getDelegationToken(), operation)) { return; } long trace = LoggerHelpers.traceEnter(log, operation, getSegmentAttribute); segmentStore.getStreamSegmentInfo(segmentName, TIMEOUT) .thenAccept(properties -> { LoggerHelpers.traceLeave(log, operation, trace, properties); if (properties == null) { connection.send(new NoSuchSegment(requestId, segmentName, EMPTY_STACK_TRACE)); } else { Map<UUID, Long> attributes = properties.getAttributes(); Long value = attributes.get(attributeId); if (value == null) { value = WireCommands.NULL_ATTRIBUTE_VALUE; } connection.send(new SegmentAttribute(requestId, value)); } }) .exceptionally(e -> handleException(requestId, segmentName, operation, e)); }
@SneakyThrows private void assertSegmentCreated(String segmentName, Collection<AttributeUpdate> attributeUpdates, TestContext context) { SegmentProperties sp; try { sp = context.getMetadataStore().getSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); } catch (Throwable ex) { ex = Exceptions.unwrap(ex); if (ex instanceof StreamSegmentNotExistsException) { Assert.fail("No segment has been created in the Storage for " + segmentName); } throw ex; } long segmentId = context.getMetadata().getStreamSegmentId(segmentName, false); Assert.assertEquals("Segment '" + segmentName + "' has been registered in the metadata.", ContainerMetadata.NO_STREAM_SEGMENT_ID, segmentId); val attributes = Attributes.getCoreNonNullAttributes(toAttributes(attributeUpdates)); AssertExtensions.assertMapEquals("Wrong attributes.", attributes, sp.getAttributes()); }
long lastIndexedOffset = context.segment().getInfo().getAttributes().get(Attributes.TABLE_INDEX_OFFSET); long segmentLength = context.segment().getInfo().getLength(); AssertExtensions.assertGreaterThan("Expected some unindexed data.", lastIndexedOffset, segmentLength);
EXPECTED_ATTRIBUTE_VALUE, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE)); } else { val extAttrValue = sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE); Assert.assertTrue("Unexpected extended attribute value from getInfo()", extAttrValue == Attributes.NULL_ATTRIBUTE_VALUE || extAttrValue == EXPECTED_ATTRIBUTE_VALUE);
private void write00(SegmentInfo s, RevisionDataOutput output) throws IOException { output.writeLong(s.getSegmentId()); SegmentProperties sp = s.getProperties(); output.writeUTF(sp.getName()); output.writeLong(sp.getLength()); output.writeLong(sp.getStartOffset()); output.writeBoolean(sp.isSealed()); // We only serialize Core Attributes. Extended Attributes can be retrieved from the AttributeIndex. output.writeMap(Attributes.getCoreNonNullAttributes(sp.getAttributes()), RevisionDataOutput::writeUUID, RevisionDataOutput::writeLong); }
expectedAttributeValue.get(), (long) attributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE)); Assert.assertEquals("Unexpected value for metadata attribute " + attributeId + " for segment " + segmentName, expectedAttributeValue.get(), (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
Assert.assertEquals("Unexpected length.", appendData.length, info.getLength()); Assert.assertEquals("Unexpected startOffset.", 1, info.getStartOffset()); Assert.assertEquals("Unexpected attribute count.", 2, info.getAttributes().size()); Assert.assertEquals("Unexpected attribute 1.", 1L, (long) info.getAttributes().get(attributeId1)); Assert.assertEquals("Unexpected attribute 2.", 2L, (long) info.getAttributes().get(attributeId2)); Assert.assertTrue("Unexpected isSealed.", info.isSealed());
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
Map<UUID, Long> expectedAttributes = segmentState == null ? null : segmentState.getAttributes(); SegmentMetadataComparer.assertSameAttributes("Unexpected attributes in metadata for StreamSegment " + name, expectedAttributes, sm); long expectedStartOffset = segmentState == null ? 0 : segmentState.getStartOffset();
private void checkActiveSegments(SegmentContainer container, int expectedCount) { val initialActiveSegments = container.getActiveSegments(); int ignoredSegments = 0; for (SegmentProperties sp : initialActiveSegments) { if (sp.getName().equals(EXPECTED_METADATA_SEGMENT_NAME)) { ignoredSegments++; continue; } val expectedSp = container.getStreamSegmentInfo(sp.getName(), TIMEOUT).join(); Assert.assertEquals("Unexpected length (from getActiveSegments) for segment " + sp.getName(), expectedSp.getLength(), sp.getLength()); Assert.assertEquals("Unexpected sealed (from getActiveSegments) for segment " + sp.getName(), expectedSp.isSealed(), sp.isSealed()); Assert.assertEquals("Unexpected deleted (from getActiveSegments) for segment " + sp.getName(), expectedSp.isDeleted(), sp.isDeleted()); SegmentMetadataComparer.assertSameAttributes("Unexpected attributes (from getActiveSegments) for segment " + sp.getName(), expectedSp.getAttributes(), sp); } Assert.assertEquals("Unexpected result from getActiveSegments with freshly created segments.", expectedCount + ignoredSegments, initialActiveSegments.size()); }
private StreamSegmentMetadata toMetadata(long segmentId, SegmentProperties sp) { StreamSegmentMetadata sm = new StreamSegmentMetadata(sp.getName(), segmentId, CONTAINER_ID); sm.updateAttributes(sp.getAttributes()); sm.setLength(sp.getLength()); sm.setStartOffset(sp.getStartOffset()); if (sp.isSealed()) { sm.markSealed(); } if (sp.isDeleted()) { sm.markDeleted(); } return sm; }
sm.markPinned(); sm.updateAttributes(sp.getAttributes()); return CompletableFuture.completedFuture(id); },