public static Optional<Config> getConfigForTopic(Properties properties, String topicKey, ConfigClient configClient) { Optional<String> configStoreUri = getConfigStoreUri(properties); Optional<Config> config = Optional.<Config>absent(); if (!configStoreUri.isPresent()) { return config; } try { Preconditions.checkArgument(properties.containsKey(GOBBLIN_CONFIG_COMMONPATH), "Missing required property " + GOBBLIN_CONFIG_COMMONPATH); Preconditions.checkArgument(properties.containsKey(topicKey), "Missing required property " + topicKey); String topicName = properties.getProperty(topicKey); String commonPath = properties.getProperty(GOBBLIN_CONFIG_COMMONPATH); config = Optional.fromNullable( getConfig(configClient, getUriStringForTopic(topicName, commonPath, configStoreUri.get()), ConfigClientUtils.getOptionalRuntimeConfig(properties))); } catch (URISyntaxException e) { log.error("Unable to get config", e); } return config; }
public Collection<URI> getImports(URI configKeyUri, boolean recursive, Optional<Config> runtimeConfig) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { ConfigStoreAccessor accessor = this.getConfigStoreAccessor(configKeyUri); ConfigKeyPath configKeypath = ConfigClientUtils.buildConfigKeyPath(configKeyUri, accessor.configStore); Collection<ConfigKeyPath> result; if (!recursive) { result = accessor.topologyInspector.getOwnImports(configKeypath, runtimeConfig); } else { result = accessor.topologyInspector.getImportsRecursively(configKeypath, runtimeConfig); } return ConfigClientUtils.buildUriInClientFormat(result, accessor.configStore, configKeyUri.getAuthority() != null); }
/** * Utility method to check whether one URI is the ancestor of the other * * return true iff both URI's scheme/authority name match and ancestor's path is the prefix of the descendant's path * @param descendant: the descendant URI to check * @param ancestor : the ancestor URI to check * @return */ public static boolean isAncestorOrSame(URI descendant, URI ancestor) { Preconditions.checkNotNull(descendant, "input can not be null"); Preconditions.checkNotNull(ancestor, "input can not be null"); if (!stringSame(descendant.getScheme(), ancestor.getScheme())) { return false; } if (!stringSame(descendant.getAuthority(), ancestor.getAuthority())) { return false; } return isAncestorOrSame(getConfigKeyPath(descendant.getPath()), getConfigKeyPath(ancestor.getPath())); }
/** * * @param configKeyURI - URI provided by client , which could missing authority/store root directory * @param cs - ConfigStore corresponding to the input URI. Require input URI's scheme/authority name * match ConfigStore's scheme/authority * @return - {@link ConfigKeyPath} for the relative path */ public static ConfigKeyPath buildConfigKeyPath(URI configKeyURI, ConfigStore cs) { checkMatchingSchemeAndAuthority(configKeyURI, cs); // Example store root is etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest URI relative = cs.getStoreURI().relativize(configKeyURI); return getConfigKeyPath(relative.getPath()); }
private URI getMatchedFloorKeyFromCache(URI configKeyURI) { URI floorKey = this.configStoreAccessorMap.floorKey(configKeyURI); if (floorKey == null) { return null; } // both scheme name and authority name, if present, should match // or both authority should be null if (ConfigClientUtils.isAncestorOrSame(configKeyURI, floorKey)) { return floorKey; } return null; }
public Config getConfig(URI configKeyUri, Optional<Config> runtimeConfig) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { ConfigStoreAccessor accessor = this.getConfigStoreAccessor(configKeyUri); ConfigKeyPath configKeypath = ConfigClientUtils.buildConfigKeyPath(configKeyUri, accessor.configStore); return accessor.valueInspector.getResolvedConfig(configKeypath, runtimeConfig); }
public static Collection<URI> buildUriInClientFormat(Collection<ConfigKeyPath> configKeyPaths, ConfigStore cs, boolean returnURIWithAuthority) { Collection<URI> result = new ArrayList<>(); if (configKeyPaths == null) { return result; } for (ConfigKeyPath p : configKeyPaths) { result.add(buildUriInClientFormat(p, cs, returnURIWithAuthority)); } return result; }
public static List<ConfigKeyPath> getConfigKeyPath(List<String> input) { List<ConfigKeyPath> result = new ArrayList<>(); for (String s : input) { result.add(getConfigKeyPath(s)); } return result; }
public static boolean isAncestorOrSame(ConfigKeyPath descendant, ConfigKeyPath ancestor) { Preconditions.checkNotNull(descendant, "input can not be null"); Preconditions.checkNotNull(ancestor, "input can not be null"); if (descendant.equals(ancestor)) return true; if (descendant.isRootPath()) return false; return isAncestorOrSame(descendant.getParent(), ancestor); }
@Test (expectedExceptions = java.lang.IllegalArgumentException.class ) public void testInvalidSchemeURI() throws URISyntaxException { URI clientURI = new URI("hdfs:///datasets/a1/a2"); ConfigClientUtils.buildConfigKeyPath(clientURI, mockConfigStore); }
/** * * @param configKeyURI - URI provided by client , which could missing authority/store root directory * @param cs - ConfigStore corresponding to the input URI. Require input URI's scheme/authority name * match ConfigStore's scheme/authority * @return - {@link ConfigKeyPath} for the relative path */ public static ConfigKeyPath buildConfigKeyPath(URI configKeyURI, ConfigStore cs) { checkMatchingSchemeAndAuthority(configKeyURI, cs); // Example store root is etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest URI relative = cs.getStoreURI().relativize(configKeyURI); return getConfigKeyPath(relative.getPath()); }
public static Collection<URI> buildUriInClientFormat(Collection<ConfigKeyPath> configKeyPaths, ConfigStore cs, boolean returnURIWithAuthority) { Collection<URI> result = new ArrayList<>(); if (configKeyPaths == null) { return result; } for (ConfigKeyPath p : configKeyPaths) { result.add(buildUriInClientFormat(p, cs, returnURIWithAuthority)); } return result; }
public static List<ConfigKeyPath> getConfigKeyPath(List<String> input) { List<ConfigKeyPath> result = new ArrayList<>(); for (String s : input) { result.add(getConfigKeyPath(s)); } return result; }
public Collection<URI> getImportedBy(URI configKeyUri, boolean recursive, Optional<Config> runtimeConfig) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { ConfigStoreAccessor accessor = this.getConfigStoreAccessor(configKeyUri); ConfigKeyPath configKeypath = ConfigClientUtils.buildConfigKeyPath(configKeyUri, accessor.configStore); Collection<ConfigKeyPath> result; if (!recursive) { result = accessor.topologyInspector.getImportedBy(configKeypath, runtimeConfig); } else { result = accessor.topologyInspector.getImportedByRecursively(configKeypath, runtimeConfig); } return ConfigClientUtils.buildUriInClientFormat(result, accessor.configStore, configKeyUri.getAuthority() != null); }
Optional<Config> runtimeConfig = ConfigClientUtils.getOptionalRuntimeConfig(properties);
/** * Utility method to check whether one URI is the ancestor of the other * * return true iff both URI's scheme/authority name match and ancestor's path is the prefix of the descendant's path * @param descendant: the descendant URI to check * @param ancestor : the ancestor URI to check * @return */ public static boolean isAncestorOrSame(URI descendant, URI ancestor) { Preconditions.checkNotNull(descendant, "input can not be null"); Preconditions.checkNotNull(ancestor, "input can not be null"); if (!stringSame(descendant.getScheme(), ancestor.getScheme())) { return false; } if (!stringSame(descendant.getAuthority(), ancestor.getAuthority())) { return false; } return isAncestorOrSame(getConfigKeyPath(descendant.getPath()), getConfigKeyPath(ancestor.getPath())); }
private ConfigStoreAccessor getConfigStoreAccessor(URI configKeyURI) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { URI matchedFloorKey = getMatchedFloorKeyFromCache(configKeyURI); ConfigStoreAccessor result; if (matchedFloorKey != null) { result = this.configStoreAccessorMap.get(matchedFloorKey); return result; } result = createNewConfigStoreAccessor(configKeyURI); ConfigStore cs = result.configStore; // put default root URI in cache as well for the URI which missing authority if (configKeyURI.getAuthority() == null) { // configKeyURI is missing authority/configstore root "etl-hdfs:///datasets/a1/a2" try { this.configStoreAccessorMap.put(new URI(configKeyURI.getScheme(), null, "/", null, null), result); } catch (URISyntaxException e) { // should not come here throw new RuntimeException("Can not build URI based on " + configKeyURI); } } else { // need to check Config Store's root is the prefix of input configKeyURI if (!ConfigClientUtils.isAncestorOrSame(configKeyURI, cs.getStoreURI())) { throw new RuntimeException( String.format("Config Store root URI %s is not the prefix of input %s", cs.getStoreURI(), configKeyURI)); } } // put to cache this.configStoreAccessorMap.put(cs.getStoreURI(), result); return result; }
@Test (expectedExceptions = java.lang.IllegalArgumentException.class ) public void testInvalidAuthortiyURI() throws URISyntaxException { URI clientURI = new URI("etl-hdfs://ltx1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); ConfigClientUtils.buildConfigKeyPath(clientURI, mockConfigStore); } }
@Test public void testGetConfigKeyPath() throws URISyntaxException{ String expected = "/datasets/a1/a2"; URI clientAbsURI = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/datasets/a1/a2"); ConfigKeyPath result = ConfigClientUtils.buildConfigKeyPath(clientAbsURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); URI clientRelativeURI = new URI("etl-hdfs:///datasets/a1/a2"); result = ConfigClientUtils.buildConfigKeyPath(clientRelativeURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); clientRelativeURI = new URI("etl-hdfs:/datasets/a1/a2"); result = ConfigClientUtils.buildConfigKeyPath(clientRelativeURI, mockConfigStore); Assert.assertEquals(result.toString(), expected); ConfigKeyPath configKey = SingleLinkedListConfigKeyPath.ROOT.createChild("data").createChild("databases").createChild("Identity"); // client app pass URI without authority URI adjusted = ConfigClientUtils.buildUriInClientFormat(configKey, mockConfigStore, false); Assert.assertTrue(adjusted.toString().equals("etl-hdfs:/data/databases/Identity")); // client app pass URI with authority adjusted = ConfigClientUtils.buildUriInClientFormat(configKey, mockConfigStore, true); Assert.assertTrue(adjusted.toString().equals("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/data/databases/Identity")); }
Optional<Config> runtimeConfig = ConfigClientUtils.getOptionalRuntimeConfig(properties);