@After public void ensureCleanUp() throws IOException { assertEmpty(getMetadataStorage(), SearchRequest.of("*").build()); }
private void verifyMetadataSelection(MetadataStorage mds, MetadataEntity entity, Metadata metadata, ImmutableSet<ScopedNameOfKind> selection) throws IOException { Assert.assertEquals(filterBy(metadata, selection), mds.read(new Read(entity, selection))); }
private void verifyMetadata(MetadataStorage mds, MetadataEntity entity, Metadata metadata) throws IOException { // verify entire metadata Assert.assertEquals(metadata, mds.read(new Read(entity))); // filter by scope verifyFilteredMetadata(mds, entity, metadata, SYSTEM, null); verifyFilteredMetadata(mds, entity, metadata, USER, null); // filter by kind verifyFilteredMetadata(mds, entity, metadata, null, PROPERTY); verifyFilteredMetadata(mds, entity, metadata, null, MetadataKind.TAG); // filter by kind and scope verifyFilteredMetadata(mds, entity, metadata, SYSTEM, PROPERTY); verifyFilteredMetadata(mds, entity, metadata, SYSTEM, MetadataKind.TAG); verifyFilteredMetadata(mds, entity, metadata, USER, PROPERTY); verifyFilteredMetadata(mds, entity, metadata, USER, MetadataKind.TAG); }
@Test public void testUpdateSearch() throws IOException { MetadataStorage mds = getMetadataStorage(); final String ns = "ns"; final MetadataEntity program = new NamespaceId(ns).app("app1").worker("wk1").toMetadataEntity(); Metadata meta = new Metadata(USER, tags("tag1", "tag2"), props("key1", "value1", "key2", "value2")); MetadataRecord programRecord = new MetadataRecord(program, meta); mds.apply(new Update(program, meta)); assertResults(mds, SearchRequest.of("value1").addNamespace(ns).build(), programRecord); assertResults(mds, SearchRequest.of("value2").addNamespace(ns).build(), programRecord); assertResults(mds, SearchRequest.of("tag2").addNamespace(ns).build(), programRecord); mds.apply(new Update(program, new Metadata(USER, props("key1", "value3")))); mds.apply(new Remove(program, ImmutableSet.of(new ScopedNameOfKind(PROPERTY, USER, "key2"), new ScopedNameOfKind(TAG, USER, "tag2")))); programRecord = new MetadataRecord(program, new Metadata(USER, tags("tag1"), props("key1", "value3"))); // Searching for value1 should be empty assertEmpty(mds, SearchRequest.of("value1").addNamespace(ns).build()); // Instead key1 has value value3 now assertResults(mds, SearchRequest.of("value3").addNamespace(ns).build(), programRecord); // key2 and tag2 were deleted assertEmpty(mds, SearchRequest.of("value2").addNamespace(ns).build()); assertEmpty(mds, SearchRequest.of("tag2").addNamespace(ns).build()); // tag1 is still here assertResults(mds, SearchRequest.of("tag1").addNamespace(ns).build(), programRecord); // clean up mds.apply(new Drop(program)); }
@Test public void testCrossNamespaceDefaultSearch() throws IOException { MetadataStorage mds = getMetadataStorage(); NamespaceId ns1 = new NamespaceId("ns1"); NamespaceId ns2 = new NamespaceId("ns2"); MetadataEntity ns1app = ns1.app("a").toMetadataEntity(); MetadataEntity ns2app = ns2.app("a").toMetadataEntity(); MetadataRecord app1Record = new MetadataRecord(ns1app, new Metadata(USER, props("k1", "v1", "k2", "v2"))); MetadataRecord app2Record = new MetadataRecord(ns2app, new Metadata(USER, props("k1", "v1"))); mds.apply(new Update(ns1app, app1Record.getMetadata())); mds.apply(new Update(ns2app, app2Record.getMetadata())); assertResults(mds, SearchRequest.of("v1").build(), app1Record, app2Record); assertResults(mds, SearchRequest.of("v2").build(), app1Record); assertResults(mds, SearchRequest.of("*").build(), app1Record, app2Record); // clean up mds.batch(ImmutableList.of(new Drop(ns1app), new Drop(ns2app))); }
@Test public void testUpdateRemove() throws IOException { MetadataStorage mds = getMetadataStorage(); verifyMetadata(mds, entity, Metadata.EMPTY); MetadataChange change = mds.apply(new Update(entity, metadata)); Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change); verifyMetadata(mds, entity, metadata); verifyMetadata(mds, entity, Metadata.EMPTY); verifyMetadata(mds, entity, metadata); change = mds.apply(new Remove(entity, USER)); Metadata newMetadata = filterBy(metadata, SYSTEM, null); Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change); verifyMetadata(mds, entity, newMetadata); verifyMetadata(mds, entity, metadata); change = mds.apply(new Remove(entity, SYSTEM)); newMetadata = filterBy(metadata, USER, null); Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change); verifyMetadata(mds, entity, newMetadata); verifyMetadata(mds, entity, metadata); change = mds.apply(new Remove(entity, TAG)); newMetadata = filterBy(metadata, null, PROPERTY); Assert.assertEquals(new MetadataChange(entity, metadata, newMetadata), change); verifyMetadata(mds, entity, newMetadata);
@Test public void testCrossNamespacePagination() throws IOException { MetadataStorage mds = getMetadataStorage(); MetadataEntity ns2app2 = ns2Id.app("a2").toMetadataEntity(); mds.apply(new Update(ns1app1, new Metadata(USER, tags("v1")))); mds.apply(new Update(ns1app2, new Metadata(USER, tags("v1")))); mds.apply(new Update(ns1app3, new Metadata(USER, tags("v1")))); mds.apply(new Update(ns2app1, new Metadata(USER, tags("v1")))); mds.apply(new Update(ns2app2, new Metadata(USER, tags("v1")))); MetadataRecord record11 = new MetadataRecord(ns1app1, new Metadata(USER, tags("v1"))); MetadataRecord record12 = new MetadataRecord(ns1app2, new Metadata(USER, tags("v1"))); MetadataRecord record13 = new MetadataRecord(ns1app3, new Metadata(USER, tags("v1"))); MetadataRecord record21 = new MetadataRecord(ns2app1, new Metadata(USER, tags("v1"))); MetadataRecord record22 = new MetadataRecord(ns2app2, new Metadata(USER, tags("v1"))); assertResults(mds, SearchRequest.of("*").setLimit(Integer.MAX_VALUE).setCursorRequested(true).build(), record11, record12, record13, record21, record22); assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(4).build(), results[1], results[2], results[3], results[4]); assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(0).setLimit(4).build(), results[0], results[1], results[2], results[3]); assertResults(mds, SearchRequest.of("*").setCursorRequested(true).setOffset(1).setLimit(3).build(), results[1], results[2], results[3], results[3]);
new MetadataRecord(app, new Metadata(SYSTEM, props(ENTITY_NAME_KEY, appName))); MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(SYSTEM, props(ENTITY_NAME_KEY, datasetName))); MetadataRecord programRecord = new MetadataRecord(program, new Metadata(SYSTEM, props(ENTITY_NAME_KEY, programName))); MetadataStorage mds = getMetadataStorage(); mds.apply(new Update(app, appRecord.getMetadata())); mds.apply(new Update(dataset, datasetRecord.getMetadata())); mds.apply(new Update(program, programRecord.getMetadata())); assertResults(mds, SearchRequest.of("name*").addNamespace(ns1).setLimit(3).build(), appRecord, datasetRecord, programRecord); assertInOrder(mds, SearchRequest.of("*").addNamespace(ns1).setLimit(2) .setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.ASC)).build(), programRecord, datasetRecord); assertInOrder(mds, SearchRequest.of("*").addNamespace(ns1).setOffset(1).setLimit(2) .setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.ASC)).build(), datasetRecord, appRecord); assertInOrder(mds, SearchRequest.of("*").addNamespace(ns1).setLimit(2) .setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.DESC)).build(), appRecord, datasetRecord); assertInOrder(mds, SearchRequest.of("*").addNamespace(ns1).setOffset(2).setLimit(1) .setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.DESC)).build(), programRecord);
@Test public void testCrossNamespaceCustomSearch() throws Exception { MetadataStorage mds = getMetadataStorage(); String appName = "app"; MetadataEntity ns1App = new NamespaceId("ns1").app(appName).toMetadataEntity(); MetadataEntity ns2App = new NamespaceId("ns2").app(appName).toMetadataEntity(); Metadata meta = new Metadata(SYSTEM, props(ENTITY_NAME_KEY, appName)); MetadataRecord app1Record = new MetadataRecord(ns1App, meta); MetadataRecord app2Record = new MetadataRecord(ns2App, meta); mds.apply(new Update(ns1App, meta)); mds.apply(new Update(ns2App, meta)); assertInOrder(mds, SearchRequest.of("*").setSorting(new Sorting(ENTITY_NAME_KEY, Sorting.Order.ASC)).build(), app1Record, app2Record); // clean up mds.batch(ImmutableList.of(new Drop(ns1App), new Drop(ns2App))); }
@Test public void testMutations() throws IOException { MetadataStorage mds = getMetadataStorage(); verifyMetadata(mds, entity, Metadata.EMPTY); verifyMetadata(mds, entity, Metadata.EMPTY); new ScopedNameOfKind(PROPERTY, USER, "up2")))); Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, Metadata.EMPTY), change); verifyMetadata(mds, entity, Metadata.EMPTY); change = mds.apply(new Update(entity, metadata)); Assert.assertEquals(new MetadataChange(entity, Metadata.EMPTY, metadata), change); verifyMetadata(mds, entity, metadata); verifyMetadata(mds, entity, metadata); verifyMetadata(mds, entity, metadata); verifyMetadataSelection(mds, entity, metadata, ImmutableSet.of( new ScopedNameOfKind(PROPERTY, SYSTEM, "sp1"), new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), verifyMetadataSelection(mds, entity, metadata, ImmutableSet.of( new ScopedNameOfKind(PROPERTY, SYSTEM, "nosuch"), new ScopedNameOfKind(MetadataKind.TAG, USER, "nosuch"))); change = mds.apply(recreate); Assert.assertEquals(new MetadataChange(entity, previousMetadata, metadata), change);
@Test public void testCrossNamespaceSearch() throws IOException { MetadataStorage mds = getMetadataStorage(); record11 = new MetadataRecord(ns1app1, new Metadata(USER, tags("v1"), props("k1", "v1"))), record12 = new MetadataRecord(ns1app2, new Metadata(USER, props("k1", "v1", "k2", "v2"))), record13 = new MetadataRecord(ns1app3, new Metadata(USER, props("k1", "v1", "k3", "v3"))), record21 = new MetadataRecord(ns2app1, new Metadata(USER, props("k1", "v1", "k2", "v2"))), record22 = new MetadataRecord(ns2app2, new Metadata(USER, tags("v2", "v3"), props("k1", "v1"))); MetadataRecord[] records = { record11, record12, record13, record21, record22 }; assertResults(mds, SearchRequest.of("v1").setLimit(10).build(), record11, record12, record13, record21, record22); assertResults(mds, SearchRequest.of("v2").setLimit(10).build(), record12, record21, record22); assertResults(mds, SearchRequest.of("v3").setLimit(10).build(), record13, record22);
@Test public void testSearchOnTypes() throws Exception { MetadataStorage mds = getMetadataStorage(); MetadataEntity myDs = NamespaceId.DEFAULT.dataset("myDs").toMetadataEntity(); MetadataEntity myField1 = MetadataEntity.builder(myDs).appendAsType("field", "myField1").build(); MetadataEntity myField2 = MetadataEntity.builder(myDs).appendAsType("field", "myField2").build(); MetadataRecord record1 = new MetadataRecord(myField1, new Metadata(USER, props("testKey1", "testValue1"))); MetadataRecord record2 = new MetadataRecord(myField2, new Metadata(USER, props("testKey2", "testValue2"))); mds.apply(new Update(myField1, record1.getMetadata())); mds.apply(new Update(myField2, record2.getMetadata())); // Search for it based on value assertResults(mds, SearchRequest.of("field:myField1").build(), record1); // should return both fields assertResults(mds, SearchRequest.of("field:myFie*").build(), record1, record2); assertResults(mds, SearchRequest.of("field*").build(), record1, record2); // clean up mds.batch(ImmutableList.of(new Drop(myField1), new Drop(myField2))); }
@Test public void testSearchOnTagsUpdate() throws IOException { MetadataStorage mds = getMetadataStorage(); MetadataEntity entity = NamespaceId.DEFAULT.app("appX").workflow("wtf").toMetadataEntity(); Metadata meta = new Metadata(SYSTEM, tags("tag1", "tag2")); mds.apply(new Update(entity, meta)); Assert.assertEquals(meta.getTags(SYSTEM), mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM)); assertResults(mds, SearchRequest.of("tag1").build(), new MetadataRecord(entity, meta)); // add an more tags mds.apply(new Update(entity, new Metadata(SYSTEM, tags("tag3", "tag4")))); Set<String> newTags = tags("tag1", "tag2", "tag3", "tag4"); Metadata newMeta = new Metadata(SYSTEM, newTags); Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM)); for (String expectedTag : newTags) { assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta)); } // add an empty set of tags. This should have no effect on retrieval or search of tags mds.apply(new Update(entity, new Metadata(SYSTEM, tags()))); Assert.assertEquals(newTags, mds.read(new Read(entity, SYSTEM)).getTags(SYSTEM)); for (String expectedTag : newTags) { assertResults(mds, SearchRequest.of(expectedTag).build(), new MetadataRecord(entity, newMeta)); } // clean up mds.apply(new Drop(entity)); }
@Test public void testSearchOnKeyValue() throws Exception { MetadataStorage mds = getMetadataStorage(); program, new Metadata(USER, props("key1", "value1", "key2", "value2", "multiword", multiWordValue))); MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(ImmutableSet.of(), ImmutableMap.of(new ScopedName(SYSTEM, "sKey1"), "sValue1", assertResults(mds, SearchRequest.of("key1:value1").addType(TYPE_PROGRAM).build(), programRecord); assertResults(mds, SearchRequest.of("key1:value1").build(), programRecord, datasetRecord); assertResults(mds, SearchRequest.of("multiword:aV5").build(), programRecord); assertResults(mds, SearchRequest.of(" multiword:aV1 ").build(), programRecord); programRecord = new MetadataRecord(program, new Metadata(USER, props("key1", "value1", "key2", "value2"))); assertEmpty(mds, SearchRequest.of("multiword:aV5").build()); assertEmpty(mds, SearchRequest.of("key1:value1").addNamespace("ns12").build()); assertResults(mds, SearchRequest.of(" value1 av2 ").build(), programRecord, datasetRecord); assertResults(mds, SearchRequest.of(" value1 sValue1 ").build(), programRecord, datasetRecord); assertResults(mds, SearchRequest.of(" valu* sVal* ").build(), programRecord, datasetRecord); assertResults(mds, SearchRequest.of(" valu* sVal* ").addType(TYPE_PROGRAM).addType(TYPE_DATASET).build(), programRecord, datasetRecord); assertResults(mds, SearchRequest.of(" valu* sVal* ").build(), programRecord, datasetRecord);
@Test public void testSearchDifferentNamespaces() throws IOException { MetadataStorage mds = getMetadataStorage(); final String ns1 = "ns1"; final NamespaceId ns1Id = new NamespaceId(ns1); final MetadataEntity artifact = ns1Id.artifact("artifact", "1.0").toMetadataEntity(); final MetadataEntity sysArtifact = NamespaceId.SYSTEM.artifact("artifact", "1.0").toMetadataEntity(); final String multiWordKey = "multiword"; final String multiWordValue = "aV1 av2 , - , av3 - av4_av5 av6"; Metadata meta = new Metadata(SYSTEM, props(multiWordKey, multiWordValue)); MetadataRecord artifactRecord = new MetadataRecord(artifact, meta); MetadataRecord sysArtifactRecord = new MetadataRecord(sysArtifact, meta); mds.apply(new Update(artifact, meta)); mds.apply(new Update(sysArtifact, meta)); // searching only user namespace should not return system entity assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).build(), artifactRecord); // searching only user namespace and system should return only the system entity assertResults(mds, SearchRequest.of("aV5").addSystemNamespace().build(), sysArtifactRecord); // searching only user namespace and system should return both entities assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addSystemNamespace().build(), artifactRecord, sysArtifactRecord); // clean up mds.batch(ImmutableList.of(new Drop(artifact), new Drop(sysArtifact))); }
private void verifyFilteredMetadata(MetadataStorage mds, MetadataEntity entity, Metadata metadata, MetadataScope scope, MetadataKind kind) throws IOException { Assert.assertEquals(filterBy(metadata, scope, kind), mds.read(new Read(entity, scope, kind))); }
@Test public void testSearchOnTags() throws Exception { MetadataStorage mds = getMetadataStorage(); app1Record = new MetadataRecord(app1, new Metadata(USER, tags("tag1", "tag2", "tag3"))), app2Record = new MetadataRecord(app2, new Metadata(USER, tags("tag1", "tag2", "tag3_more"))), program1Record = new MetadataRecord(program1, new Metadata(USER, tags("tag1"))), dataset1Record = new MetadataRecord(dataset1, new Metadata(USER, tags("tag3", "tag2", "tag12-tag33"))), dataset2Record = new MetadataRecord(dataset2, new Metadata(USER, tags("tag2", "tag4"))), file1Record = new MetadataRecord(file1, new Metadata(USER, tags("tag2", "tag5"))); assertResults(mds, SearchRequest.of("tags:*").addNamespace(ns1).build(), app1Record, program1Record, dataset1Record, dataset2Record, file1Record); assertResults(mds, SearchRequest.of("tags:tag1*").addNamespace(ns1).build(), app1Record, program1Record, dataset1Record); assertResults(mds, SearchRequest.of(" tAGS : tag1 ").addNamespace(ns1).build(), app1Record, program1Record); assertResults(mds, SearchRequest.of("tags:tag5").addNamespace(ns1).build(), file1Record); assertResults(mds, SearchRequest.of("tags:tag2").addNamespace(ns1).build(), app1Record, dataset1Record, dataset2Record, file1Record); assertResults(mds, SearchRequest.of("tags:tag4").addNamespace(ns1).build(), dataset2Record); assertResults(mds, SearchRequest.of("tags:tag33").addNamespace(ns1).build(),
@Test public void testSearchOnValue() throws Exception { MetadataStorage mds = getMetadataStorage(); program, new Metadata(USER, props("key1", "value1", "key2", "value2", "multiword", multiWordValue))); mds.apply(new Update(program, programRecord.getMetadata())); assertResults(mds, SearchRequest.of("value1").build(), programRecord); assertResults(mds, SearchRequest.of(" aV1 ").addType(TYPE_PROGRAM).build(), programRecord); assertEmpty(mds, SearchRequest.of("-").build()); assertEmpty(mds, SearchRequest.of(",").build()); assertEmpty(mds, SearchRequest.of("_").build()); assertEmpty(mds, SearchRequest.of(", ,").build()); assertEmpty(mds, SearchRequest.of(", - ,").build()); assertResults(mds, SearchRequest.of("av5").addType(TYPE_PROGRAM).build(), programRecord); assertResults(mds, SearchRequest.of("ValUe1").addType(TYPE_PROGRAM).build(), programRecord); mds.apply(new Update(program, new Metadata(SYSTEM, props("key3", "value1")))); programRecord = new MetadataRecord( program, new Metadata(ImmutableSet.of(), ImmutableMap.of(new ScopedName(USER, "key1"), "value1", assertResults(mds, SearchRequest.of("value1").addType(TYPE_PROGRAM).build(), programRecord); MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, props("key21", "value21"))); mds.apply(new Update(dataset, datasetRecord.getMetadata())); assertResults(mds, SearchRequest.of("value2*").build(), programRecord, datasetRecord);
@Test public void testSearchPagination() throws IOException { MetadataStorage mds = getMetadataStorage(); mds.apply(new Update(service, new Metadata(USER, tags("tag", "tag1")))); mds.apply(new Update(worker, new Metadata(USER, tags("tag2", "tag3 tag4")))); mds.apply(new Update(dataset, new Metadata(USER, tags("tag5 tag6", "tag7 tag8")))); mds.apply(new Update(hidden, new Metadata(USER, tags("tag9", "tag10", "tag11", "tag12", "tag13")))); MetadataRecord serviceRecord = new MetadataRecord(service, new Metadata(USER, tags("tag", "tag1"))); MetadataRecord streamRecord = new MetadataRecord(worker, new Metadata(USER, tags("tag2", "tag3 tag4"))); MetadataRecord datasetRecord = new MetadataRecord(dataset, new Metadata(USER, tags("tag5 tag6", "tag7 tag8"))); MetadataRecord hiddenRecord = new MetadataRecord(hidden, new Metadata(USER, tags("tag9", "tag10", "tag11", "tag12", "tag13"))); assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(Integer.MAX_VALUE).build(), datasetRecord, streamRecord, serviceRecord); assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setShowHidden(true).build(), hiddenRecord, datasetRecord, streamRecord, serviceRecord); assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setLimit(2).setCursorRequested(true).build(), datasetRecord, streamRecord); assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(2).build(), streamRecord, serviceRecord); assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(1).setLimit(3).setShowHidden(true).build(), datasetRecord, streamRecord, serviceRecord); assertResults(mds, SearchRequest.of("tag*").addNamespace(ns).setOffset(2).setLimit(2).build(),
@Test public void testSearchIncludesSystemEntities() throws IOException { MetadataStorage mds = getMetadataStorage(); Metadata meta = new Metadata(SYSTEM, props(multiWordKey, multiWordValue)); assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addSystemNamespace().build(), programRecord, sysArtifactRecord); assertResults(mds, SearchRequest.of("aV5").addNamespace(ns1).addType(TYPE_PROGRAM).build(), programRecord); assertResults(mds, SearchRequest.of("multiword:aV5").addNamespace(ns1).addType(TYPE_PROGRAM).build(), programRecord); assertResults(mds, SearchRequest.of("multiword:" + multiWordValue) .addNamespace(ns1).addSystemNamespace().addType(TYPE_ARTIFACT).build(), sysArtifactRecord); assertResults(mds, SearchRequest.of("multiword:aV4").addNamespace(ns2).addSystemNamespace().build(), artifactRecord, sysArtifactRecord); assertEmpty(mds, SearchRequest.of("aV*").addNamespace(ns2).addSystemNamespace().addType(TYPE_PROGRAM).build()); assertResults(mds, SearchRequest.of("av*").addNamespace("ns3").addSystemNamespace().build(), sysArtifactRecord); assertResults(mds, SearchRequest.of("av*").addSystemNamespace().build(), sysArtifactRecord);