/** * Helper method to merge disjoint metadata objects. This is used to merge the metadata fro (up to) two * metadata records returned by the datasets (one per scope) into a single Metadata object. */ private static Metadata mergeDisjointMetadata(Metadata meta, Metadata other) { return new Metadata(Sets.union(meta.getTags(), other.getTags()), // this will not conflict because the two metadata are mutually disjoint ImmutableMap.<ScopedName, String>builder() .putAll(meta.getProperties()) .putAll(other.getProperties()).build()); } }
@Override public Set<MetadataRecord> getMetadata(MetadataEntity metadataEntity) { Metadata metadata = performRead(new Read(metadataEntity)); return MetadataScope.ALL.stream() .map(scope -> new MetadataRecord(metadataEntity, scope, metadata.getProperties(scope), metadata.getTags(scope))) .collect(Collectors.toSet()); }
@Override public void addProperties(MetadataScope scope, Map<MetadataEntity, Map<String, String>> toUpdate) { List<MetadataMutation> mutations = toUpdate.entrySet().stream() .map(entry -> new MetadataMutation.Update(entry.getKey(), new Metadata(scope, entry.getValue()))) .collect(Collectors.toList()); applyBatch(mutations); }
@Test public void testSearchOnTagsUpdate() throws IOException { MetadataStorage mds = getMetadataStorage(); MetadataEntity entity = NamespaceId.DEFAULT.app("appX").workflow("wtf").toMetadataEntity(); Metadata meta = new Metadata(SYSTEM, tags("tag1", "tag2")); mds.apply(new Update(entity, meta)); Assert.assertEquals(meta.getTags(SYSTEM), mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM)); assertResults(mds, SearchRequest.of("tag1").build(), new MetadataRecord(entity, meta)); // add an more tags mds.apply(new Update(entity, new Metadata(SYSTEM, tags("tag3", "tag4")))); Set<String> newTags = tags("tag1", "tag2", "tag3", "tag4"); Metadata newMeta = new Metadata(SYSTEM, newTags); Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM)); for (String expectedTag : newTags) { assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta)); } // add an empty set of tags. This should have no effect on retrieval or search of tags mds.apply(new Update(entity, new Metadata(SYSTEM, tags()))); Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM)); for (String expectedTag : newTags) { assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta)); } // clean up mds.apply(new Drop(entity)); }
@Override public Map<String, String> getProperties(MetadataEntity entity) { Metadata metadata = performRead(new Read(entity, MetadataKind.PROPERTY)); return metadata.getProperties().entrySet().stream() .collect(Collectors.toMap(entry -> entry.getKey().getName(), Map.Entry::getValue)); }
@Override public Set<String> getTags(MetadataEntity entity) { Metadata metadata = performRead(new Read(entity, MetadataKind.TAG)); return metadata.getTags().stream().map(ScopedName::getName).collect(Collectors.toSet()); }
@Override public MetadataRecord getMetadata(MetadataScope scope, MetadataEntity metadataEntity) { Metadata metadata = performRead(new Read(metadataEntity, scope)); return new MetadataRecord(metadataEntity, scope, metadata.getProperties(scope), metadata.getTags(scope)); }
@Override public void addTags(MetadataScope scope, MetadataEntity entity, Set<String> tagsToAdd) { applyMutation(new MetadataMutation.Update(entity, new Metadata(scope, tagsToAdd))); }
@Override public Map<String, String> getProperties(MetadataScope scope, MetadataEntity entity) { Metadata metadata = performRead(new Read(entity, scope, MetadataKind.PROPERTY)); return metadata.getProperties(scope); }
@Override public Set<String> getTags(MetadataScope scope, MetadataEntity entity) { Metadata metadata = performRead(new Read(entity, scope, MetadataKind.TAG)); return metadata.getTags(scope); }
protected static Metadata union(Metadata meta, Metadata other) { return new Metadata(Sets.union(meta.getTags(), other.getTags()), ImmutableMap.<ScopedName, String>builder() .putAll(meta.getProperties()) .putAll(other.getProperties()).build()); }
private MetadataChange update(MetadataDatasetContext context, MetadataEntity entity, Metadata updates) { Set<String> userTagsToAdd = new HashSet<>(); Set<String> systemTagsToAdd = new HashSet<>(); Map<String, String> userPropertiesToAdd = new HashMap<>(); Map<String, String> systemPropertiesToAdd = new HashMap<>(); updates.getTags().forEach(tag -> (USER == tag.getScope() ? userTagsToAdd : systemTagsToAdd).add(tag.getName())); updates.getProperties().forEach( (key, value) -> (USER == key.getScope() ? userPropertiesToAdd : systemPropertiesToAdd).put(key.getName(), value)); MetadataDataset.Change userChange = addInScope(context, USER, entity, userTagsToAdd, userPropertiesToAdd); MetadataDataset.Change systemChange = addInScope(context, SYSTEM, entity, systemTagsToAdd, systemPropertiesToAdd); return combineChanges(entity, userChange, systemChange); }
@Override public void addProperties(MetadataScope scope, MetadataEntity entity, Map<String, String> properties) { applyMutation(new MetadataMutation.Update(entity, new Metadata(scope, properties))); }
private Metadata filterBy(Metadata metadata, MetadataScope scope, MetadataKind kind) { //noinspection ConstantConditions return new Metadata( kind == PROPERTY ? Collections.emptySet() : Sets.filter(metadata.getTags(), tag -> scope == null || scope.equals(tag.getScope())), kind == MetadataKind.TAG ? Collections.EMPTY_MAP : Maps.filterKeys(metadata.getProperties(), key -> scope == null || scope.equals(key.getScope()))); }
private MetadataChange create(MetadataDatasetContext context, MetadataEntity entity, Metadata metadata, Map<ScopedNameOfKind, MetadataDirective> directives) { Set<String> newUserTags = new HashSet<>(); Set<String> newSystemTags = new HashSet<>(); Map<String, String> newUserProperties = new HashMap<>(); Map<String, String> newSystemProperties = new HashMap<>(); metadata.getTags().forEach(tag -> (USER == tag.getScope() ? newUserTags : newSystemTags).add(tag.getName())); metadata.getProperties().forEach( (key, value) -> (USER == key.getScope() ? newUserProperties : newSystemProperties).put(key.getName(), value)); MetadataDataset.Change userChange = replaceInScope(context, USER, entity, newUserTags, newUserProperties, directives); MetadataDataset.Change systemChange = replaceInScope(context, SYSTEM, entity, newSystemTags, newSystemProperties, directives); return combineChanges(entity, userChange, systemChange); }
@Override public void replaceMetadata(MetadataScope scope, MetadataDataset.Record metadata, Set<String> propertiesToKeep, Set<String> propertiesToPreserve) { Map<ScopedNameOfKind, MetadataDirective> directives = new HashMap<>(); propertiesToKeep.forEach(name -> directives.put(new ScopedNameOfKind(MetadataKind.PROPERTY, scope, name), MetadataDirective.KEEP)); propertiesToPreserve.forEach(name -> directives.put(new ScopedNameOfKind(MetadataKind.PROPERTY, scope, name), MetadataDirective.PRESERVE)); MetadataMutation mutation = new MetadataMutation.Create( metadata.getMetadataEntity(), new Metadata(scope, metadata.getTags(), metadata.getProperties()), directives); applyMutation(mutation); }
private Metadata filterBy(Metadata metadata, ImmutableSet<ScopedNameOfKind> selection) { //noinspection ConstantConditions return new Metadata( Sets.filter(metadata.getTags(), tag -> selection.contains(new ScopedNameOfKind(MetadataKind.TAG, tag.getScope(), tag.getName()))), Maps.filterKeys(metadata.getProperties(), key -> selection.contains(new ScopedNameOfKind(PROPERTY, key.getScope(), key.getName())))); }
private void publishAudit(MetadataChange change, MetadataScope scope) { Map<String, String> propsBefore = change.getBefore().getProperties(scope); Map<String, String> propsAfter = change.getAfter().getProperties(scope); Set<String> tagsBefore = change.getBefore().getTags(scope); Set<String> tagsAfter = change.getAfter().getTags(scope);
private Metadata read(MetadataDatasetContext context, Read read) { MetadataDataset.Record userMetadata = readScope(context, MetadataScope.USER, read); MetadataDataset.Record systemMetadata = readScope(context, MetadataScope.SYSTEM, read); return mergeDisjointMetadata(new Metadata(USER, userMetadata.getTags(), userMetadata.getProperties()), new Metadata(SYSTEM, systemMetadata.getTags(), systemMetadata.getProperties())); }
Map<MetadataScope, co.cask.cdap.api.metadata.Metadata> metaMap = new HashMap<>(); MetadataScope.ALL.forEach(scope -> { Map<String, String> props = record.getMetadata().getProperties(scope); Set<String> tags = record.getMetadata().getTags(scope); if (!props.isEmpty() || !tags.isEmpty()) { metaMap.put(scope, new co.cask.cdap.api.metadata.Metadata(props, tags));