@Test public void testGetChildren() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-children"; String childDatasetName = "childDataset"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(new Path(datasetPath, childDatasetName)); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); Collection<ConfigKeyPath> children = this._simpleHadoopFilesystemConfigStore.getChildren(datasetConfigKey, VERSION); Assert.assertEquals(children.size(), 1); Assert.assertEquals(children.iterator().next().getOwnPathName(), childDatasetName); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } }
@Test public void testGetOwnConfig() throws ConfigStoreCreationException, URISyntaxException, IOException { String datasetName = "dataset-test-get-own-config"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(datasetPath); this.fs.create(new Path(datasetPath, "main.conf")).close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); Config config = this._simpleHadoopFilesystemConfigStore.getOwnConfig(datasetConfigKey, VERSION); Assert.assertTrue(config.isEmpty()); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } }
@Test public void testGetOwnImportsWithResolution() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-own-imports-resolution"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(datasetPath); BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8)); writer.write("/path/to/${?" + TAG_NAME_SYS_PROP_KEY + "}"); writer.close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); List<ConfigKeyPath> imports = this._simpleHadoopFilesystemConfigStore.getOwnImports(datasetConfigKey, VERSION); Assert.assertEquals(imports.size(), 1); Assert.assertEquals(imports.get(0).getAbsolutePathString(), "/path/to/" + TAG_NAME_SYS_PROP_VALUE); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } }
@Test public void testHash() { ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data"); ConfigKeyPath profile1 = data.createChild("databases").createChild("identity").createChild("profile"); ConfigKeyPath profile2 = data.createChild("databases").createChild("identity").createChild("profile"); Assert.assertFalse(profile1 == profile2); Assert.assertTrue(profile1.equals(profile2)); Assert.assertEquals(profile1.hashCode(), profile2.hashCode()); Set<ConfigKeyPath> testSet = new HashSet<ConfigKeyPath>(); testSet.add(profile1); Assert.assertTrue(testSet.contains(profile2)); } }
@Test public void testGetOwnImportsWithRuntimeConfigResolution() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-own-imports-resolution"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); Properties prop = new Properties(); prop.put(TAG_NAME_RUNTIME_PROP_KEY, TAG_NAME_RUNTIME_PROP_VALUE); Optional<Config> runtimeConfig = Optional.fromNullable(ConfigUtils.propertiesToConfig(prop)); try { this.fs.mkdirs(datasetPath); BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8)); writer.write("/path/to/${?" + TAG_NAME_RUNTIME_PROP_KEY + "}"); writer.close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); List<ConfigKeyPath> imports = this._simpleHadoopFilesystemConfigStore.getOwnImports(datasetConfigKey, VERSION, runtimeConfig); Assert.assertEquals(imports.size(), 1); Assert.assertEquals(imports.get(0).getAbsolutePathString(), "/path/to/" + TAG_NAME_RUNTIME_PROP_VALUE); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } }
@Test public void testGetOwnImports() throws IOException, URISyntaxException, ConfigStoreCreationException { String datasetName = "dataset-test-get-own-imports"; String tagKey1 = "/path/to/tag1"; String tagKey2 = "/path/to/tag2"; Path datasetPath = new Path(CONFIG_DIR_PATH, datasetName); try { this.fs.mkdirs(datasetPath); BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(this.fs.create(new Path(datasetPath, "includes.conf")), Charsets.UTF_8)); writer.write(tagKey1); writer.newLine(); writer.write(tagKey2); writer.close(); ConfigKeyPath datasetConfigKey = SingleLinkedListConfigKeyPath.ROOT.createChild(datasetName); List<ConfigKeyPath> imports = this._simpleHadoopFilesystemConfigStore.getOwnImports(datasetConfigKey, VERSION); Assert.assertEquals(imports.size(), 2); Assert.assertEquals(imports.get(0).getAbsolutePathString(), tagKey2); Assert.assertEquals(imports.get(1).getAbsolutePathString(), tagKey1); } finally { if (this.fs.exists(datasetPath)) { this.fs.delete(datasetPath, true); } } }
@Test public void testNonRoot() { ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data"); Assert.assertEquals(data.getAbsolutePathString(), "/data"); ConfigKeyPath profile = data.createChild("databases").createChild("identity").createChild("profile"); Assert.assertEquals(profile.toString(), "/data/databases/identity/profile"); }
@Test public void testSystemPropertyResolution() { ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); ConfigStoreTopologyInspector mockTopology = mock(ConfigStoreTopologyInspector.class, Mockito.RETURNS_SMART_NULLS); ConfigStoreBackedValueInspector valueInspector = new ConfigStoreBackedValueInspector(mockConfigStore, version, mockTopology); ConfigKeyPath testConfigKeyPath = SingleLinkedListConfigKeyPath.ROOT.createChild("a"); when(mockConfigStore.getOwnConfig(testConfigKeyPath.getParent(), version)).thenReturn(ConfigFactory.empty()); when(mockConfigStore.getOwnConfig(testConfigKeyPath, version)).thenReturn( ConfigFactory.parseString("configProp = ${?" + VALUE_INSPECTOR_SYS_PROP_KEY + "}")); Assert.assertEquals(valueInspector.getResolvedConfig(testConfigKeyPath).getString("configProp"), VALUE_INSPECTOR_SYS_PROP_VALUE); }
@Test public void testGetConfigKeyPath() throws URISyntaxException{ String expected = "/datasets/a1/a2"; URI clientAbsURI = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); ConfigKeyPath result = ConfigClientUtils.buildConfigKeyPath(clientAbsURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); URI clientRelativeURI = new URI("etl-hdfs:///datasets/a1/a2"); result = ConfigClientUtils.buildConfigKeyPath(clientRelativeURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); clientRelativeURI = new URI("etl-hdfs:/datasets/a1/a2"); result = ConfigClientUtils.buildConfigKeyPath(clientRelativeURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); ConfigKeyPath configKey = SingleLinkedListConfigKeyPath.ROOT.createChild("data").createChild("databases").createChild("Identity"); // client app pass URI without authority URI adjusted = ConfigClientUtils.buildUriInClientFormat(configKey, mockConfigStore, false); Assert.assertTrue(adjusted.toString().equals("etl-hdfs:/data/databases/Identity")); // client app pass URI with authority adjusted = ConfigClientUtils.buildUriInClientFormat(configKey, mockConfigStore, true); Assert.assertTrue(adjusted.toString().equals("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/data/databases/Identity")); }
@Test public void testIsAncestorOrSame() throws Exception{ //Path ancestor = new Path("/"); //Path descendant = new Path("/"); URI ancestor = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/"); URI descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); // ends with "/" descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2/"); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); // wrong authority descendant = new URI("etl-hdfs://ltx1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); // wrong path descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/sahil/HdfsBasedConfigTest/datasets/a1/a2"); Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data"); ConfigKeyPath data2 = SingleLinkedListConfigKeyPath.ROOT.createChild("data2"); ConfigKeyPath identity = SingleLinkedListConfigKeyPath.ROOT.createChild("data").createChild("databases").createChild("Identity"); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(identity, data)); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(identity, SingleLinkedListConfigKeyPath.ROOT)); Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(identity, data2)); }
@Test public void testSelfImportSelf() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); // self import self addConfigStoreImports(mockConfigStore, version, tag, tag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag") > 0); } }
@Test public void testResolveConfigOverridingInChild() { ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); ConfigStoreTopologyInspector mockTopology = mock(ConfigStoreTopologyInspector.class, Mockito.RETURNS_SMART_NULLS); ConfigStoreBackedValueInspector valueInspector = new ConfigStoreBackedValueInspector(mockConfigStore, version, mockTopology); ConfigKeyPath keyPathA = SingleLinkedListConfigKeyPath.ROOT.createChild("a"); ConfigKeyPath keyPathA_Slash_B = keyPathA.createChild("b"); when(mockConfigStore.getOwnConfig(keyPathA.getParent(), version)).thenReturn(ConfigFactory.empty()); when(mockConfigStore.getOwnConfig(keyPathA, version)).thenReturn( ConfigFactory.parseString("key1 = value1InA \n key2 = ${key1}")); when(mockConfigStore.getOwnConfig(keyPathA_Slash_B, version)).thenReturn( ConfigFactory.parseString("key1 = value1InB")); Assert.assertEquals(valueInspector.getResolvedConfig(keyPathA_Slash_B).getString("key2"), "value1InB"); } }
@Test public void testNoCircular() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag"); ConfigKeyPath tag2 = SingleLinkedListConfigKeyPath.ROOT.createChild("tag2");
@Test public void testSelfImportChild() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag); // parent import direct child addConfigStoreImports(mockConfigStore, version, tag, highPriorityTag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag") > 0 && e.getMessage().indexOf("/tag ") > 0); } }
@Test public void testSelfIndirectlyImportDescendant() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag"); ConfigKeyPath tag2 = SingleLinkedListConfigKeyPath.ROOT.createChild("tag2"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag, tag2); addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag); addConfigStoreChildren(mockConfigStore, version, highPriorityTag, nertzHighPriorityTag); // self import descendant // formed the loop /tag -> /tag2 -> /tag/highPriorityTag/nertzHighPriorityTag -> /tag/highPriorityTag -> /tag addConfigStoreImports(mockConfigStore, version, tag, tag2); addConfigStoreImports(mockConfigStore, version, tag2, nertzHighPriorityTag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag/nertzHighPriorityTag") > 0 && e.getMessage().indexOf("/tag/highPriorityTag ") > 0 && e.getMessage().indexOf("/tag ") > 0 && e.getMessage().indexOf("/tag2 ") > 0); } }
@Test public void testSelfImportDescendant() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag); addConfigStoreChildren(mockConfigStore, version, highPriorityTag, nertzHighPriorityTag); // self import descendant // formed the loop /tag -> /tag/highPriorityTag/nertzHighPriorityTag -> /tag/highPriorityTag -> /tag addConfigStoreImports(mockConfigStore, version, tag, nertzHighPriorityTag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag/nertzHighPriorityTag") > 0 && e.getMessage().indexOf("/tag/highPriorityTag ") > 0 && e.getMessage().indexOf("/tag ") > 0); } }
@Test public void testLoops() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath subTag1 = tag.createChild("subTag1"); ConfigKeyPath subTag2 = tag.createChild("subTag2"); ConfigKeyPath subTag3 = tag.createChild("subTag3"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); addConfigStoreChildren(mockConfigStore, version, tag, subTag1, subTag2, subTag3); // self import descendant // formed loop /tag/subTag1 -> /tag/subTag2 -> /tag/subTag3 -> /tag/subTag1 addConfigStoreImports(mockConfigStore, version, subTag1, subTag2); addConfigStoreImports(mockConfigStore, version, subTag2, subTag3); addConfigStoreImports(mockConfigStore, version, subTag3, subTag1); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(subTag1); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/subTag1") > 0 && e.getMessage().indexOf("/tag/subTag2") > 0 && e.getMessage().indexOf("/tag/subTag3") > 0); } }