@Override public ConfigKeyPath apply(String input) { if (input == null) { return null; } ConfigKeyPath configKey = SingleLinkedListConfigKeyPath.ROOT; for (String file : Splitter.on(SingleLinkedListConfigKeyPath.PATH_DELIMETER).omitEmptyStrings().split(input)) { configKey = configKey.createChild(file); } return configKey; } }
/** * Build the {@link ConfigKeyPath} based on the absolute/relative path * @param input - absolute/relative file path * @return - {@link ConfigKeyPath} corresponding to the input */ public static ConfigKeyPath getConfigKeyPath(String input) { ConfigKeyPath result = SingleLinkedListConfigKeyPath.ROOT; String[] paths = input.split("/"); for (String p : paths) { // in case input start with "/", some elements could be "", which should be skip if (p.equals("")) { continue; } result = result.createChild(p); } return result; }
children.add(configKey.createChild(fileStatus.getPath().getName()));
/** * Retrieves all the children of the given {@link ConfigKeyPath} using {@link Files#walk} to list files */ @Override public Collection<ConfigKeyPath> getChildren(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException { Preconditions.checkNotNull(configKey, "configKey cannot be null!"); Preconditions.checkArgument(version.equals(getCurrentVersion())); List<ConfigKeyPath> children = new ArrayList<>(); Path datasetDir = getDatasetDirForKey(configKey); try { if (!Files.exists(this.fs.getPath(datasetDir.toString()))) { return children; } Stream<Path> files = Files.walk(datasetDir, 1); for (Iterator<Path> it = files.iterator(); it.hasNext();) { Path path = it.next(); if (Files.isDirectory(path) && !path.equals(datasetDir)) { children.add(configKey.createChild(StringUtils.removeEnd(path.getName(path.getNameCount() - 1).toString(), SingleLinkedListConfigKeyPath.PATH_DELIMETER))); } } return children; } catch (IOException e) { throw new RuntimeException(String.format("Error while getting children for configKey: \"%s\"", configKey), e); } }
@Test public void testHash() { ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data"); ConfigKeyPath profile1 = data.createChild("databases").createChild("identity").createChild("profile"); ConfigKeyPath profile2 = data.createChild("databases").createChild("identity").createChild("profile"); Assert.assertFalse(profile1 == profile2); Assert.assertTrue(profile1.equals(profile2)); Assert.assertEquals(profile1.hashCode(), profile2.hashCode()); Set<ConfigKeyPath> testSet = new HashSet<ConfigKeyPath>(); testSet.add(profile1); Assert.assertTrue(testSet.contains(profile2)); } }
@Test public void testNonRoot() { ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data"); Assert.assertEquals(data.getAbsolutePathString(), "/data"); ConfigKeyPath profile = data.createChild("databases").createChild("identity").createChild("profile"); Assert.assertEquals(profile.toString(), "/data/databases/identity/profile"); }
@Test public void testGetConfigKeyPath() throws URISyntaxException{ String expected = "/datasets/a1/a2"; URI clientAbsURI = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); ConfigKeyPath result = ConfigClientUtils.buildConfigKeyPath(clientAbsURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); URI clientRelativeURI = new URI("etl-hdfs:///datasets/a1/a2"); result = ConfigClientUtils.buildConfigKeyPath(clientRelativeURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); clientRelativeURI = new URI("etl-hdfs:/datasets/a1/a2"); result = ConfigClientUtils.buildConfigKeyPath(clientRelativeURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); ConfigKeyPath configKey = SingleLinkedListConfigKeyPath.ROOT.createChild("data").createChild("databases").createChild("Identity"); // client app pass URI without authority URI adjusted = ConfigClientUtils.buildUriInClientFormat(configKey, mockConfigStore, false); Assert.assertTrue(adjusted.toString().equals("etl-hdfs:/data/databases/Identity")); // client app pass URI with authority adjusted = ConfigClientUtils.buildUriInClientFormat(configKey, mockConfigStore, true); Assert.assertTrue(adjusted.toString().equals("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/data/databases/Identity")); }
@Test public void testIsAncestorOrSame() throws Exception{ //Path ancestor = new Path("/"); //Path descendant = new Path("/"); URI ancestor = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/"); URI descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); // ends with "/" descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2/"); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); // wrong authority descendant = new URI("etl-hdfs://ltx1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); // wrong path descendant = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/sahil/HdfsBasedConfigTest/datasets/a1/a2"); Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(descendant, ancestor)); ConfigKeyPath data = SingleLinkedListConfigKeyPath.ROOT.createChild("data"); ConfigKeyPath data2 = SingleLinkedListConfigKeyPath.ROOT.createChild("data2"); ConfigKeyPath identity = SingleLinkedListConfigKeyPath.ROOT.createChild("data").createChild("databases").createChild("Identity"); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(identity, data)); Assert.assertTrue(ConfigClientUtils.isAncestorOrSame(identity, SingleLinkedListConfigKeyPath.ROOT)); Assert.assertTrue(!ConfigClientUtils.isAncestorOrSame(identity, data2)); }
@Test public void testResolveConfigOverridingInChild() { ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); ConfigStoreTopologyInspector mockTopology = mock(ConfigStoreTopologyInspector.class, Mockito.RETURNS_SMART_NULLS); ConfigStoreBackedValueInspector valueInspector = new ConfigStoreBackedValueInspector(mockConfigStore, version, mockTopology); ConfigKeyPath keyPathA = SingleLinkedListConfigKeyPath.ROOT.createChild("a"); ConfigKeyPath keyPathA_Slash_B = keyPathA.createChild("b"); when(mockConfigStore.getOwnConfig(keyPathA.getParent(), version)).thenReturn(ConfigFactory.empty()); when(mockConfigStore.getOwnConfig(keyPathA, version)).thenReturn( ConfigFactory.parseString("key1 = value1InA \n key2 = ${key1}")); when(mockConfigStore.getOwnConfig(keyPathA_Slash_B, version)).thenReturn( ConfigFactory.parseString("key1 = value1InB")); Assert.assertEquals(valueInspector.getResolvedConfig(keyPathA_Slash_B).getString("key2"), "value1InB"); } }
@Test public void testNoCircular() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag");
@Test public void testLoops() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath subTag1 = tag.createChild("subTag1"); ConfigKeyPath subTag2 = tag.createChild("subTag2"); ConfigKeyPath subTag3 = tag.createChild("subTag3"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); addConfigStoreChildren(mockConfigStore, version, tag, subTag1, subTag2, subTag3); // self import descendant // formed loop /tag/subTag1 -> /tag/subTag2 -> /tag/subTag3 -> /tag/subTag1 addConfigStoreImports(mockConfigStore, version, subTag1, subTag2); addConfigStoreImports(mockConfigStore, version, subTag2, subTag3); addConfigStoreImports(mockConfigStore, version, subTag3, subTag1); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(subTag1); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/subTag1") > 0 && e.getMessage().indexOf("/tag/subTag2") > 0 && e.getMessage().indexOf("/tag/subTag3") > 0); } }
@Test public void testSelfImportDescendant() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag); addConfigStoreChildren(mockConfigStore, version, highPriorityTag, nertzHighPriorityTag); // self import descendant // formed the loop /tag -> /tag/highPriorityTag/nertzHighPriorityTag -> /tag/highPriorityTag -> /tag addConfigStoreImports(mockConfigStore, version, tag, nertzHighPriorityTag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag/nertzHighPriorityTag") > 0 && e.getMessage().indexOf("/tag/highPriorityTag ") > 0 && e.getMessage().indexOf("/tag ") > 0); } }
@Test public void testSelfImportChild() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag); addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag); // parent import direct child addConfigStoreImports(mockConfigStore, version, tag, highPriorityTag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag") > 0 && e.getMessage().indexOf("/tag ") > 0); } }
@Test public void testSelfIndirectlyImportDescendant() { ConfigKeyPath tag = SingleLinkedListConfigKeyPath.ROOT.createChild("tag"); ConfigKeyPath highPriorityTag = tag.createChild("highPriorityTag"); ConfigKeyPath nertzHighPriorityTag = highPriorityTag.createChild("nertzHighPriorityTag"); ConfigKeyPath tag2 = SingleLinkedListConfigKeyPath.ROOT.createChild("tag2"); ConfigStore mockConfigStore = mock(ConfigStore.class, Mockito.RETURNS_SMART_NULLS); when(mockConfigStore.getCurrentVersion()).thenReturn(version); addConfigStoreChildren(mockConfigStore, version, SingleLinkedListConfigKeyPath.ROOT, tag, tag2); addConfigStoreChildren(mockConfigStore, version, tag, highPriorityTag); addConfigStoreChildren(mockConfigStore, version, highPriorityTag, nertzHighPriorityTag); // self import descendant // formed the loop /tag -> /tag2 -> /tag/highPriorityTag/nertzHighPriorityTag -> /tag/highPriorityTag -> /tag addConfigStoreImports(mockConfigStore, version, tag, tag2); addConfigStoreImports(mockConfigStore, version, tag2, nertzHighPriorityTag); ConfigStoreBackedTopology csTopology = new ConfigStoreBackedTopology(mockConfigStore, this.version); InMemoryTopology inMemory = new InMemoryTopology(csTopology); try { inMemory.getImportsRecursively(tag); Assert.fail("Did not catch expected CircularDependencyException"); } catch (CircularDependencyException e) { Assert.assertTrue(e.getMessage().indexOf("/tag/highPriorityTag/nertzHighPriorityTag") > 0 && e.getMessage().indexOf("/tag/highPriorityTag ") > 0 && e.getMessage().indexOf("/tag ") > 0 && e.getMessage().indexOf("/tag2 ") > 0); } }
@Override public ConfigKeyPath apply(String input) { if (input == null) { return null; } ConfigKeyPath configKey = SingleLinkedListConfigKeyPath.ROOT; for (String file : Splitter.on(SingleLinkedListConfigKeyPath.PATH_DELIMETER).omitEmptyStrings().split(input)) { configKey = configKey.createChild(file); } return configKey; } }
/** * Build the {@link ConfigKeyPath} based on the absolute/relative path * @param input - absolute/relative file path * @return - {@link ConfigKeyPath} corresponding to the input */ public static ConfigKeyPath getConfigKeyPath(String input) { ConfigKeyPath result = SingleLinkedListConfigKeyPath.ROOT; String[] paths = input.split("/"); for (String p : paths) { // in case input start with "/", some elements could be "", which should be skip if (p.equals("")) { continue; } result = result.createChild(p); } return result; }
/** * Retrieves all the children of the given {@link ConfigKeyPath} using {@link Files#walk} to list files */ @Override public Collection<ConfigKeyPath> getChildren(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException { Preconditions.checkNotNull(configKey, "configKey cannot be null!"); Preconditions.checkArgument(version.equals(getCurrentVersion())); List<ConfigKeyPath> children = new ArrayList<>(); Path datasetDir = getDatasetDirForKey(configKey); try { if (!Files.exists(this.fs.getPath(datasetDir.toString()))) { return children; } Stream<Path> files = Files.walk(datasetDir, 1); for (Iterator<Path> it = files.iterator(); it.hasNext();) { Path path = it.next(); if (Files.isDirectory(path) && !path.equals(datasetDir)) { children.add(configKey.createChild(StringUtils.removeEnd(path.getName(path.getNameCount() - 1).toString(), SingleLinkedListConfigKeyPath.PATH_DELIMETER))); } } return children; } catch (IOException e) { throw new RuntimeException(String.format("Error while getting children for configKey: \"%s\"", configKey), e); } }
children.add(configKey.createChild(fileStatus.getPath().getName()));