/** * * @param jobConfDirPath The directory path for job cofniguration files. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path jobConfDirPath, URI uri) { return PathUtils.mergePaths(jobConfDirPath, new Path(uri)); }
public static URI getUriStringForTopic(String topicName, String commonPath, String configStoreUri) throws URISyntaxException { URI storeUri = new URI(configStoreUri); Path path = PathUtils.mergePaths(new Path(storeUri.getPath()), PathUtils.mergePaths(new Path(commonPath), new Path(topicName))); URI topicUri = new URI(storeUri.getScheme(), storeUri.getAuthority(), path.toString(), storeUri.getQuery(), storeUri.getFragment()); log.info("URI for topic is : " + topicUri.toString()); return topicUri; }
/** * Helper funcition for converting datasetURN into URI * Note that here the URN can possibly being specified with pattern, i.e. with wildcards like `*` * It will be resolved by configStore. */ private URI datasetURNtoURI(String datasetURN) { try { return new URI(PathUtils.mergePaths(new Path(this.storeRoot), new Path(datasetURN)).toString()); }catch (URISyntaxException e) { log.error("Dataset with URN:" + datasetURN + " cannot be converted into URI. Skip the dataset"); return null; } }
/** * * @param fsSpecStoreDirPath The directory path for specs. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path fsSpecStoreDirPath, URI uri, String version) { return PathUtils.addExtension(PathUtils.mergePaths(fsSpecStoreDirPath, new Path(uri)), version); } }
@Override public JobTemplate getTemplate(URI uri) throws SpecNotFoundException, JobTemplate.TemplateException { if (!uri.getScheme().equals(FS_SCHEME)) { throw new RuntimeException("Expected scheme " + FS_SCHEME + " got unsupported scheme " + uri.getScheme()); } // path of uri is location of template file relative to the job configuration root directory Path templateFullPath = PathUtils.mergePaths(jobConfDirPath, new Path(uri.getPath())); try (InputStream is = fs.open(templateFullPath)) { return new HOCONInputStreamJobTemplate(is, uri, this); } catch (IOException ioe) { throw new SpecNotFoundException(uri, ioe); } }
/** * * @param flowTemplateDirURI URI of the flow template directory * @return a {@link FlowTemplate} * @throws SpecNotFoundException * @throws JobTemplate.TemplateException * @throws IOException */ public FlowTemplate getFlowTemplate(URI flowTemplateDirURI) throws SpecNotFoundException, JobTemplate.TemplateException, IOException, URISyntaxException { if (!this.sysConfig.hasPath(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY)) { throw new RuntimeException("Missing config " + ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY); } if (!flowTemplateDirURI.getScheme().equals(FS_SCHEME)) { throw new RuntimeException( "Expected scheme " + FS_SCHEME + " got unsupported scheme " + flowTemplateDirURI.getScheme()); } String templateCatalogDir = this.sysConfig.getString(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY); // path of uri is location of template file relative to the job configuration root directory Path templateDirPath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(flowTemplateDirURI.getPath())); Path templateFullPath = PathUtils.mergePaths(templateDirPath, new Path(FLOW_CONF_FILE_NAME)); FileSystem fs = FileSystem.get(templateFullPath.toUri(), new Configuration()); try (InputStream is = fs.open(templateFullPath)) { return new HOCONInputStreamFlowTemplate(is, flowTemplateDirURI, this); } }
/** * Move a path to trash. The absolute path of the input path will be replicated under the trash directory. * @param path {@link org.apache.hadoop.fs.FileSystem} path to move to trash. * @return true if move to trash was done successfully. * @throws IOException */ @Override public boolean moveToTrash(Path path) throws IOException { Path fullyResolvedPath = path.isAbsolute() ? path : new Path(this.fs.getWorkingDirectory(), path); Path targetPathInTrash = PathUtils.mergePaths(this.trashLocation, fullyResolvedPath); if (!this.fs.exists(targetPathInTrash.getParent())) { this.fs.mkdirs(targetPathInTrash.getParent()); } else if (this.fs.exists(targetPathInTrash)) { targetPathInTrash = targetPathInTrash.suffix("_" + System.currentTimeMillis()); } return this.fs.rename(fullyResolvedPath, targetPathInTrash); }
this.commonRoot = PathUtils.mergePaths(new Path(this.storeRoot), new Path(jobProps.getProperty(GOBBLIN_CONFIG_STORE_DATASET_COMMON_ROOT))); this.whitelistTag = PathUtils.mergePaths(new Path(this.storeRoot), new Path(jobProps.getProperty(GOBBLIN_CONFIG_STORE_WHITELIST_TAG))); this.threadPoolSize = jobProps.containsKey(CopySource.MAX_CONCURRENT_LISTING_SERVICES) List<Path> disablePaths = new ArrayList<Path>(); for (String s : disableStrs) { disablePaths.add(PathUtils.mergePaths(new Path(this.storeRoot), new Path(s)));
@Test public void testMergePaths() throws Exception { Path path1 = new Path("/some/path"); Path path2 = new Path("/path2/file"); Assert.assertEquals(PathUtils.mergePaths(path1, path2).toString(), "/some/path/path2/file"); }
Path flowTemplateDirPath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(flowTemplateDirURI)); "Expected scheme " + FS_SCHEME + " got unsupported scheme " + flowTemplateDirURI.getScheme()); Path fullJobTemplatePath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(jobTemplateRelativeUri)); jobConfig = jobConfig.withFallback(loadHoconFileAtPath(fullJobTemplatePath, true));
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
"Missing required property " + GOBBLIN_CONFIG_FILTER); String filterString = properties.getProperty(GOBBLIN_CONFIG_FILTER); Path whiteListTagUri = PathUtils.mergePaths(new Path(configStoreUri.get()), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST))); getTopicsURIFromConfigStore(configClient, whiteListTagUri, filterString, runtimeConfig).stream() "Missing required property " + GOBBLIN_CONFIG_FILTER); String filterString = properties.getProperty(GOBBLIN_CONFIG_FILTER); Path blackListTagUri = PathUtils.mergePaths(new Path(configStoreUri.get()), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST))); getTopicsURIFromConfigStore(configClient, blackListTagUri, filterString, runtimeConfig).stream()
"Missing required property " + GOBBLIN_CONFIG_FILTER); String filterString = properties.getProperty(GOBBLIN_CONFIG_FILTER); Path whiteListTagUri = PathUtils.mergePaths(new Path(configStoreUri), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST))); List<String> whitelistedTopics = new ArrayList<>(); "Missing required property " + GOBBLIN_CONFIG_FILTER); String filterString = properties.getProperty(GOBBLIN_CONFIG_FILTER); Path blackListTagUri = PathUtils.mergePaths(new Path(configStoreUri), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST))); List<String> blacklistedTopics = new ArrayList<>();
String tag = this.props.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST); ConfigClient configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.WEAK_LOCAL_STABILITY); Path tagUri = PathUtils.mergePaths(new Path(this.configStoreUri.get()), new Path(tag)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true);
/** * * @param jobConfDirPath The directory path for job cofniguration files. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path jobConfDirPath, URI uri) { return PathUtils.mergePaths(jobConfDirPath, new Path(uri)); }
public static URI getUriStringForTopic(String topicName, String commonPath, String configStoreUri) throws URISyntaxException { URI storeUri = new URI(configStoreUri); Path path = PathUtils.mergePaths(new Path(storeUri.getPath()), PathUtils.mergePaths(new Path(commonPath), new Path(topicName))); URI topicUri = new URI(storeUri.getScheme(), storeUri.getAuthority(), path.toString(), storeUri.getQuery(), storeUri.getFragment()); log.info("URI for topic is : " + topicUri.toString()); return topicUri; }
/** * Helper funcition for converting datasetURN into URI * Note that here the URN can possibly being specified with pattern, i.e. with wildcards like `*` * It will be resolved by configStore. */ private URI datasetURNtoURI(String datasetURN) { try { return new URI(PathUtils.mergePaths(new Path(this.storeRoot), new Path(datasetURN)).toString()); }catch (URISyntaxException e) { log.error("Dataset with URN:" + datasetURN + " cannot be converted into URI. Skip the dataset"); return null; } }
/** * * @param fsSpecStoreDirPath The directory path for specs. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path fsSpecStoreDirPath, URI uri, String version) { return PathUtils.addExtension(PathUtils.mergePaths(fsSpecStoreDirPath, new Path(uri)), version); } }
@Override public JobTemplate getTemplate(URI uri) throws SpecNotFoundException, JobTemplate.TemplateException { if (!uri.getScheme().equals(FS_SCHEME)) { throw new RuntimeException("Expected scheme " + FS_SCHEME + " got unsupported scheme " + uri.getScheme()); } // path of uri is location of template file relative to the job configuration root directory Path templateFullPath = PathUtils.mergePaths(jobConfDirPath, new Path(uri.getPath())); try (InputStream is = fs.open(templateFullPath)) { return new HOCONInputStreamJobTemplate(is, uri, this); } catch (IOException ioe) { throw new SpecNotFoundException(uri, ioe); } }
/** * Move a path to trash. The absolute path of the input path will be replicated under the trash directory. * @param path {@link org.apache.hadoop.fs.FileSystem} path to move to trash. * @return true if move to trash was done successfully. * @throws IOException */ @Override public boolean moveToTrash(Path path) throws IOException { Path fullyResolvedPath = path.isAbsolute() ? path : new Path(this.fs.getWorkingDirectory(), path); Path targetPathInTrash = PathUtils.mergePaths(this.trashLocation, fullyResolvedPath); if (!this.fs.exists(targetPathInTrash.getParent())) { this.fs.mkdirs(targetPathInTrash.getParent()); } else if (this.fs.exists(targetPathInTrash)) { targetPathInTrash = targetPathInTrash.suffix("_" + System.currentTimeMillis()); } return this.fs.rename(fullyResolvedPath, targetPathInTrash); }