/** * Returns a new Map of Attribute Ids to Values containing only those Core Attributes from the given Map that do not * have a null value. * * @param attributes The Map of Attribute Ids to Values to filter from. * @return A new Map containing only Core Attribute Ids and Values (from the original map). */ public static Map<UUID, Long> getCoreNonNullAttributes(Map<UUID, Long> attributes) { return attributes.entrySet().stream() .filter(e -> Attributes.isCoreAttribute(e.getKey()) && e.getValue() != NULL_ATTRIBUTE_VALUE) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } }
/** * Tests the ability of the MetadataStore to create a new Segment if the Segment already exists. */ @Test public void testCreateSegmentAlreadyExists() { final String segmentName = "NewSegment"; final Map<UUID, Long> originalAttributes = ImmutableMap.of(UUID.randomUUID(), 123L, Attributes.EVENT_COUNT, 1L); final Map<UUID, Long> expectedAttributes = Attributes.getCoreNonNullAttributes(originalAttributes); final Collection<AttributeUpdate> correctAttributeUpdates = originalAttributes.entrySet().stream() .map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())) .collect(Collectors.toList()); final Map<UUID, Long> badAttributes = Collections.singletonMap(UUID.randomUUID(), 456L); final Collection<AttributeUpdate> badAttributeUpdates = badAttributes.entrySet().stream() .map(e -> new AttributeUpdate(e.getKey(), AttributeUpdateType.Replace, e.getValue())) .collect(Collectors.toList()); @Cleanup TestContext context = createTestContext(); // Create a segment. context.getMetadataStore().createSegment(segmentName, correctAttributeUpdates, TIMEOUT).join(); // Try to create it again. AssertExtensions.assertSuppliedFutureThrows( "createSegment did not fail when Segment already exists.", () -> context.getMetadataStore().createSegment(segmentName, badAttributeUpdates, TIMEOUT), ex -> ex instanceof StreamSegmentExistsException); val si = context.getMetadataStore().getSegmentInfo(segmentName, TIMEOUT).join(); AssertExtensions.assertMapEquals("Unexpected attributes after failed attempt to recreate correctly created segment", expectedAttributes, si.getAttributes()); }
private void write00(SegmentInfo s, RevisionDataOutput output) throws IOException { output.writeLong(s.getSegmentId()); SegmentProperties sp = s.getProperties(); output.writeUTF(sp.getName()); output.writeLong(sp.getLength()); output.writeLong(sp.getStartOffset()); output.writeBoolean(sp.isSealed()); // We only serialize Core Attributes. Extended Attributes can be retrieved from the AttributeIndex. output.writeMap(Attributes.getCoreNonNullAttributes(sp.getAttributes()), RevisionDataOutput::writeUUID, RevisionDataOutput::writeLong); }
@Override public synchronized void updateAttributes(Map<UUID, Long> attributes) { attributes.forEach((id, value) -> { if (Attributes.isCoreAttribute(id)) { this.coreAttributes.put(id, value); } else { this.extendedAttributes.put(id, new ExtendedAttributeValue(value)); } }); }
@SneakyThrows private void assertSegmentCreated(String segmentName, Collection<AttributeUpdate> attributeUpdates, TestContext context) { SegmentProperties sp; try { sp = context.getMetadataStore().getSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); } catch (Throwable ex) { ex = Exceptions.unwrap(ex); if (ex instanceof StreamSegmentNotExistsException) { Assert.fail("No segment has been created in the Storage for " + segmentName); } throw ex; } long segmentId = context.getMetadata().getStreamSegmentId(segmentName, false); Assert.assertEquals("Segment '" + segmentName + "' has been registered in the metadata.", ContainerMetadata.NO_STREAM_SEGMENT_ID, segmentId); val attributes = Attributes.getCoreNonNullAttributes(toAttributes(attributeUpdates)); AssertExtensions.assertMapEquals("Wrong attributes.", attributes, sp.getAttributes()); }
/** * Collects the extended Attributes from the AttributeUpdates of the given operation. */ private Map<UUID, Long> getExtendedAttributes(AttributeUpdaterOperation operation) { Collection<AttributeUpdate> updates = operation.getAttributeUpdates(); if (updates == null) { return Collections.emptyMap(); } return updates.stream() .filter(au -> !Attributes.isCoreAttribute(au.getAttributeId())) .collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue)); }
val coreAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes); // We expect extended attributes to be dropped in this case. SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction.", coreAttributes, sp);
} else if (!Attributes.isCoreAttribute(attributeId)) { extendedAttributeIds.add(attributeId);
expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes); // We expect extended attributes to be dropped in this case. localContainer.createStreamSegment(segmentName, initialAttributes, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); SegmentProperties sp = localContainer.getStreamSegmentInfo(segmentName, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes); // We expect extended attributes to be dropped in this case. SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after eviction & resurrection.", expectedAttributes, sp); expectedAttributes = Attributes.getCoreNonNullAttributes(expectedAttributes); // We expect extended attributes to be dropped in this case. SegmentMetadataComparer.assertSameAttributes("Unexpected attributes after deletion and re-creation.", expectedAttributes, sp);
Collection<UUID> attributeIds = updates.stream() .map(AttributeUpdate::getAttributeId) .filter(id -> !Attributes.isCoreAttribute(id)) .collect(Collectors.toList()); if (!attributeIds.isEmpty()) {
private void writeSegmentMetadata00(RevisionDataOutput output, SegmentMetadata sm) throws IOException { output.writeLong(sm.getId()); output.writeUTF(sm.getName()); output.writeLong(sm.getLength()); output.writeLong(sm.getStorageLength()); output.writeBoolean(sm.isMerged()); output.writeBoolean(sm.isSealed()); output.writeBoolean(sm.isSealedInStorage()); output.writeBoolean(sm.isDeleted()); output.writeBoolean(sm.isDeletedInStorage()); output.writeLong(sm.getLastModified().getTime()); output.writeLong(sm.getStartOffset()); // We only serialize Core Attributes. Extended Attributes can be retrieved from the AttributeIndex. output.writeMap(Attributes.getCoreNonNullAttributes(sm.getAttributes()), RevisionDataOutput::writeUUID, RevisionDataOutput::writeLong); }
private void verifyAttributes(SegmentMetadata metadata, TestContext context) { val persistedAttributes = context.dataSource.getPersistedAttributes(metadata.getId()); int extendedAttributeCount = 0; if (context.transactionIds.containsKey(metadata.getId()) && metadata.isMerged()) { Assert.assertEquals("Unexpected number of attributes in attribute index for merged transaction " + metadata.getId(), 0, persistedAttributes.size()); AssertExtensions.assertFutureThrows( "Merged transaction attribute index still exists.", context.dataSource.persistAttributes(metadata.getId(), Collections.singletonMap(UUID.randomUUID(), 0L), TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException); } else { for (val e : metadata.getAttributes().entrySet()) { if (Attributes.isCoreAttribute(e.getKey())) { Assert.assertFalse("Not expecting Core Attribute in Attribute Index for Segment " + metadata.getId(), persistedAttributes.containsKey(e.getKey())); } else { extendedAttributeCount++; Assert.assertEquals("Unexpected attribute value for Segment " + metadata.getId(), e.getValue(), persistedAttributes.get(e.getKey())); } } Assert.assertEquals("Unexpected number of attributes in attribute index for Segment " + metadata.getId(), extendedAttributeCount, persistedAttributes.size()); if (metadata.isSealedInStorage()) { AssertExtensions.assertFutureThrows( "Sealed segment attribute index accepted new values.", context.dataSource.persistAttributes(metadata.getId(), Collections.singletonMap(UUID.randomUUID(), 0L), TIMEOUT), ex -> ex instanceof StreamSegmentSealedException); } } }
val expectedAttributes = Attributes.getCoreNonNullAttributes(segmentInfo.getAttributes()); SegmentMetadataComparer.assertSameAttributes("Unexpected attributes when Segment exists in Storage", expectedAttributes, inStorageInfo); Assert.assertEquals("Not expecting any segments to be mapped.", 0, context.getNonPinnedMappedSegmentCount());
private void checkAttributes(SegmentMetadata metadata, TestContext context) { val persistedAttributes = context.dataSource.getPersistedAttributes(metadata.getId()); int extendedAttributeCount = 0; if (context.transactionIds.containsKey(metadata.getId()) && metadata.isMerged()) { Assert.assertEquals("Unexpected number of attributes in attribute index for merged transaction " + metadata.getId(), 0, persistedAttributes.size()); AssertExtensions.assertFutureThrows( "Merged transaction attribute index still exists.", context.dataSource.persistAttributes(metadata.getId(), Collections.singletonMap(UUID.randomUUID(), 0L), TIMEOUT), ex -> ex instanceof StreamSegmentNotExistsException); } else { for (val e : metadata.getAttributes().entrySet()) { if (Attributes.isCoreAttribute(e.getKey())) { Assert.assertFalse("Not expecting Core Attribute in Attribute Index for " + metadata.getId(), persistedAttributes.containsKey(e.getKey())); } else { extendedAttributeCount++; Assert.assertEquals("Unexpected attribute value for " + metadata.getId(), e.getValue(), persistedAttributes.get(e.getKey())); } } Assert.assertEquals("Unexpected number of attributes in attribute index for " + metadata.getId(), extendedAttributeCount, persistedAttributes.size()); if (metadata.isSealedInStorage()) { AssertExtensions.assertFutureThrows( "Sealed segment attribute index accepted new values.", context.dataSource.persistAttributes(metadata.getId(), Collections.singletonMap(UUID.randomUUID(), 0L), TIMEOUT), ex -> ex instanceof StreamSegmentSealedException); } } }
Assert.assertEquals("Unexpected value for attribute " + attributeId + " via getAttributes() after recovery for segment " + segmentName, expectedAttributeValue, (long) allAttributeValues.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE)); if (Attributes.isCoreAttribute(attributeId)) { Assert.assertEquals("Expecting core attribute to be loaded in memory.", expectedAttributeValue, (long) sp.getAttributes().getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE));
EXPECTED_ATTRIBUTE_VALUE, (long) allAttributes.getOrDefault(attributeId, Attributes.NULL_ATTRIBUTE_VALUE)); if (Attributes.isCoreAttribute(attributeId)) {