/** * * @param jobConfDirPath The directory path for job cofniguration files. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path jobConfDirPath, URI uri) { return PathUtils.mergePaths(jobConfDirPath, new Path(uri)); }
/** * Helper funcition for converting datasetURN into URI */ private URI datasetURNtoURI(String datasetURN) { try { return new URI(PathUtils.mergePaths(new Path(this.storeRoot), new Path(datasetURN)).toString()); }catch (URISyntaxException e) { log.error("Dataset with URN:" + datasetURN + " cannot be converted into URI. Skip the dataset"); return null; } }
/** * * @param fsSpecStoreDirPath The directory path for specs. * @param uri Uri as the identifier of JobSpec * @return */ protected Path getPathForURI(Path fsSpecStoreDirPath, URI uri, String version) { return PathUtils.addExtension(PathUtils.mergePaths(fsSpecStoreDirPath, new Path(uri)), version); } }
@Override public JobTemplate getTemplate(URI uri) throws SpecNotFoundException, JobTemplate.TemplateException { if (!uri.getScheme().equals(FS_SCHEME)) { throw new RuntimeException("Expected scheme " + FS_SCHEME + " got unsupported scheme " + uri.getScheme()); } // path of uri is location of template file relative to the job configuration root directory Path templateFullPath = PathUtils.mergePaths(jobConfDirPath, new Path(uri.getPath())); try (InputStream is = fs.open(templateFullPath)) { return new HOCONInputStreamJobTemplate(is, uri, this); } catch (IOException ioe) { throw new SpecNotFoundException(uri, ioe); } }
/** * Move a path to trash. The absolute path of the input path will be replicated under the trash directory. * @param path {@link org.apache.hadoop.fs.FileSystem} path to move to trash. * @return true if move to trash was done successfully. * @throws IOException */ @Override public boolean moveToTrash(Path path) throws IOException { Path fullyResolvedPath = path.isAbsolute() ? path : new Path(this.fs.getWorkingDirectory(), path); Path targetPathInTrash = PathUtils.mergePaths(this.trashLocation, fullyResolvedPath); if (!this.fs.exists(targetPathInTrash.getParent())) { this.fs.mkdirs(targetPathInTrash.getParent()); } else if (this.fs.exists(targetPathInTrash)) { targetPathInTrash = targetPathInTrash.suffix("_" + System.currentTimeMillis()); } return this.fs.rename(fullyResolvedPath, targetPathInTrash); }
this.commonRoot = PathUtils.mergePaths(new Path(this.storeRoot), new Path(jobProps.getProperty(GOBBLIN_CONFIG_STORE_DATASET_COMMON_ROOT))); this.whitelistTag = PathUtils.mergePaths(new Path(this.storeRoot), new Path(jobProps.getProperty(GOBBLIN_CONFIG_STORE_WHITELIST_TAG))); this.threadPoolSize = jobProps.containsKey(CopySource.MAX_CONCURRENT_LISTING_SERVICES) List<Path> disablePaths = new ArrayList<Path>(); for (String s : disableStrs) { disablePaths.add(PathUtils.mergePaths(new Path(this.storeRoot), new Path(s)));
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
String tag = this.props.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST); ConfigClient configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.WEAK_LOCAL_STABILITY); Path tagUri = PathUtils.mergePaths(new Path(this.configStoreUri.get()), new Path(tag)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true);