/** * Get the URIs which imports the input URI * * @param configKeyUri - The URI for the configuration key. * @param recursive - Specify whether to get direct or recursively imported by links * @return the URIs which imports the input URI * * @throws ConfigStoreFactoryDoesNotExistsException: if missing scheme name or the scheme name is invalid * @throws ConfigStoreCreationException: Specified {@link ConfigStoreFactory} can not create required {@link ConfigStore} * @throws VersionDoesNotExistException: Required version does not exist anymore ( may get deleted by retention job ) */ public Collection<URI> getImportedBy(URI configKeyUri, boolean recursive) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { return getImportedBy(configKeyUri, recursive, Optional.<Config>absent()); }
private void enhanceDisabledURIsWithBlackListTag(Set<URI> disabledURIs) throws URISyntaxException, ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { if (this.blacklistTags.isPresent()) { for (Path s : this.blacklistTags.get()) { disabledURIs.addAll(configClient.getImportedBy(new URI(s.toString()), true)); } } }
/** * Will return the list of URIs given which are importing tag {@param tagUri} */ public static Collection<URI> getTopicsURIFromConfigStore(ConfigClient configClient, Path tagUri, String filterString, Optional<Config> runtimeConfig) { try { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true, runtimeConfig); return importedBy.stream().filter((URI u) -> u.toString().contains(filterString)).collect(Collectors.toList()); } catch (URISyntaxException | ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException e) { throw new Error(e); } }
/** * Semantic of black/whitelist: * - Whitelist always respect blacklist. * - Job-level blacklist is reponsible for dataset filtering instead of dataset discovery. i.e. * There's no implementation of job-level whitelist currently. */ protected Set<URI> getValidDatasetURIs(Path datasetCommonRoot) { Collection<URI> allDatasetURIs; Set<URI> disabledURISet = new HashSet(); // This try block basically populate the Valid dataset URI set. try { allDatasetURIs = configClient.getImportedBy(new URI(whitelistTag.toString()), true); enhanceDisabledURIsWithBlackListTag(disabledURISet); } catch ( ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException | URISyntaxException e) { log.error("Caught error while getting all the datasets URIs " + e.getMessage()); throw new RuntimeException(e); } return getValidDatasetURIsHelper(allDatasetURIs, disabledURISet, datasetCommonRoot); }
importedBys.addAll(client.getImportedBy(new URI(tag), false));
Path tagUri = PathUtils.mergePaths(new Path(this.configStoreUri.get()), new Path(tag)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true); for (URI uri : importedBy) { String dbName = new Path(uri).getParent().getName();
Collection<URI> importedBy = client.getImportedBy(nertzTagURI, false); Assert.assertEquals(importedBy.size(), 1); Assert.assertEquals(importedBy.iterator().next().toString(), expectedImportedBy[0]); importedBy = client.getImportedBy(nertzTagURI, true); Assert.assertEquals(importedBy.size(), 2); for(URI u: importedBy){ "etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/data/databases/identity"}; nertzTagURI = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/tag2/nertzTag2"); importedBy = client.getImportedBy(nertzTagURI, false); Assert.assertEquals(importedBy.size(), 1); Assert.assertEquals(importedBy.iterator().next().toString(), expectedImportedBy_abs[0]); importedBy = client.getImportedBy(nertzTagURI, true); Assert.assertEquals(importedBy.size(), 2); for(URI u: importedBy){
/** * Get the URIs which imports the input URI * * @param configKeyUri - The URI for the configuration key. * @param recursive - Specify whether to get direct or recursively imported by links * @return the URIs which imports the input URI * * @throws ConfigStoreFactoryDoesNotExistsException: if missing scheme name or the scheme name is invalid * @throws ConfigStoreCreationException: Specified {@link ConfigStoreFactory} can not create required {@link ConfigStore} * @throws VersionDoesNotExistException: Required version does not exist anymore ( may get deleted by retention job ) */ public Collection<URI> getImportedBy(URI configKeyUri, boolean recursive) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { return getImportedBy(configKeyUri, recursive, Optional.<Config>absent()); }
private void enhanceDisabledURIsWithBlackListTag(Set<URI> disabledURIs) throws URISyntaxException, ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { if (this.blacklistTags.isPresent()) { for (Path s : this.blacklistTags.get()) { disabledURIs.addAll(configClient.getImportedBy(new URI(s.toString()), true)); } } }
/** * Will return the list of URIs given which are importing tag {@param tagUri} */ public static Collection<URI> getTopicsURIFromConfigStore(ConfigClient configClient, Path tagUri, String filterString, Optional<Config> runtimeConfig) { try { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true, runtimeConfig); return importedBy.stream().filter((URI u) -> u.toString().contains(filterString)).collect(Collectors.toList()); } catch (URISyntaxException | ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException e) { throw new Error(e); } }
/** * Semantic of black/whitelist: * - Whitelist always respect blacklist. * - Job-level blacklist is reponsible for dataset filtering instead of dataset discovery. i.e. * There's no implementation of job-level whitelist currently. */ protected Set<URI> getValidDatasetURIs(Path datasetCommonRoot) { Collection<URI> allDatasetURIs; Set<URI> disabledURISet = new HashSet(); // This try block basically populate the Valid dataset URI set. try { allDatasetURIs = configClient.getImportedBy(new URI(whitelistTag.toString()), true); enhanceDisabledURIsWithBlackListTag(disabledURISet); } catch ( ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException | URISyntaxException e) { log.error("Caught error while getting all the datasets URIs " + e.getMessage()); throw new RuntimeException(e); } return getValidDatasetURIsHelper(allDatasetURIs, disabledURISet, datasetCommonRoot); }
importedBys.addAll(client.getImportedBy(new URI(tag), false));
Path tagUri = PathUtils.mergePaths(new Path(this.configStoreUri.get()), new Path(tag)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true); for (URI uri : importedBy) { String dbName = new Path(uri).getParent().getName();