public NamespaceKey(List<String> pathComponents) { checkNotNull(pathComponents); this.pathComponents = ImmutableList.copyOf(pathComponents); this.schemaPath = PathUtils.constructFullPath(pathComponents); }
/** * Convert fs path to dotted schema path. * /a/b/c -> a.b.c * @param fsPath filesystem path * @return schema path. */ public static String toDottedPath(Path fsPath) { return constructFullPath(toPathComponents(fsPath)); }
public String getJobResultsTableName(JobId jobId) { // return String.format("TABLE(%s(type => 'arrow'))", PathUtils.constructFullPath(getOutputTablePath(jobId))); }
public String getPreviewDataUrlPath() { List<String> pathList = toPathList(); URI uri = UriBuilder.fromUri("/dataset/{fullPath}/preview") .build(PathUtils.constructFullPath(pathList)); return uri.toString(); }
public String toParentPath() { return PathUtils.constructFullPath(toParentPathList()); }
public static BatchSchema fromDataset(DatasetConfig config){ try{ ByteString bytes = DatasetHelper.getSchemaBytes(config); if(bytes == null){ throw new IllegalStateException(String.format("Schema is currently unavailable for dataset %s.", PathUtils.constructFullPath(config.getFullPathList()))); } return deserialize(bytes); }catch(Exception ex){ throw new IllegalStateException(String.format("Schema for dataset %s is corrupt.", PathUtils.constructFullPath(config.getFullPathList())), ex); } }
public String getReflectionDatasetPath(String layoutId) { try { return PathUtils.constructFullPath(relationshipMap.get(layoutId).getDataset().getPathList()); }catch (Exception e) { logger.warn("failed to get reflection dataset path", e); return ""; } }
/** * Convert fs path relative to parent to dotted schema path. * parent: /a/b, child: /a/b/c/d -> c.d * @param parent parent path * @param child full path of child inside parent path * @return dotted schema name of child relative to parent. * @throws IOException if child does not belong under parent */ public static String toDottedPath(final Path parent, final Path child) throws IOException { final List<String> parentPathComponents = toPathComponents(parent); final List<String> childPathComponents = toPathComponents(child); for (int i = 0; i < parentPathComponents.size(); ++i) { if (!parentPathComponents.get(i).equals(childPathComponents.get(i))) { throw new IOException(String.format("Invalid file/directory %s listed under %s", child, parent)); } } return constructFullPath(childPathComponents.subList(parentPathComponents.size(), childPathComponents.size())); }
public static String describeDependencies(ReflectionId reflectionId, Iterable<DependencyEntry> dependencyEntries) { final StringBuilder builder = new StringBuilder(); builder.append("reflection ").append(reflectionId.getId()).append(" depends on : {\n"); for (DependencyEntry dependency : dependencyEntries) { if (dependency.getType() == DependencyType.REFLECTION) { final ReflectionId rId = ((ReflectionDependency) dependency).getReflectionId(); builder.append(" reflection ").append(rId.getId()).append("\n"); } else { final List<String> path = ((DatasetDependency) dependency).getPath(); builder.append(" dataset ").append(PathUtils.constructFullPath(path)).append("\n"); } } builder.append("}\n"); return builder.toString(); }
private void addParents(DocumentWriter writer, List<ParentDataset> parentDatasetList) { if (notEmpty(parentDatasetList)) { final String[] parents = new String[parentDatasetList.size()]; int i = 0; for (ParentDataset parent : parentDatasetList) { parents[i++] = PathUtils.constructFullPath(parent.getDatasetPathList()); } writer.write(DATASET_PARENTS, parents); } }
public static String slugify(Collection<String> pathComponents) { Slugify slg = new Slugify(); return slg.slugify(constructFullPath(pathComponents)); }
private void addAllParents(DocumentWriter writer, List<ParentDataset> parents, List<ParentDataset> grandParents) { if (notEmpty(parents)) { grandParents = listNotNull(grandParents); int i = 0; final String[] allParents = new String[parents.size() + grandParents.size()]; for (ParentDataset parent : parents) { allParents[i++] = PathUtils.constructFullPath(parent.getDatasetPathList()); } for (ParentDataset grandParent : grandParents) { allParents[i++] = PathUtils.constructFullPath(grandParent.getDatasetPathList()); } writer.write(DATASET_ALLPARENTS, allParents); } } }
/** * Generates special url for querying datasets. * This should be temporary (famous last words) until dataset urls are made consistent. */ public String getQueryUrlPath() { List<String> pathList = toPathList(); final URITemplateBuilder builder = new URITemplateBuilder(); builder.add(root.getRootUrl()); builder.add(root.getName()); builder.add(PathUtils.constructFullPath(pathList.subList(1, pathList.size()))); return builder.build(); }
private static String getSchemaPath(String file) throws IOException { return "dacfs_test." + PathUtils.constructFullPath(PathUtils.toPathComponents(new Path(FileUtils.getResourceAsFile(file).getAbsolutePath()))); }
private static String toTableName(String schema1, String schema2, final AttemptId id) { return PathUtils.constructFullPath(Arrays.asList(schema1, schema2, QueryIdHelper.getQueryId(id.toQueryId()))); } }
public static Map<String, NameSpaceContainer> listHome(NamespaceService ns, String parent) throws Exception { Map<String, NameSpaceContainer> children = new HashMap<>(); for (NameSpaceContainer container : ns.list(new NamespaceKey(parent))) { children.put(PathUtils.constructFullPath(container.getFullPathList()), container); } return children; }
@Override public String toString() { final JobAttemptUI jobAttempt = getJobAttempt(); return format("{JobId: %s, SQL: %s, Dataset: %s, DatasetVersion: %s}", getJobId(), jobAttempt.getInfo().getSql(), PathUtils.constructFullPath(jobAttempt.getInfo().getDatasetPathList()), jobAttempt.getInfo().getDatasetVersion()); //todo }
public static Map<String, NameSpaceContainer> listFolder(NamespaceService ns, String parent) throws Exception { Map<String, NameSpaceContainer> children = new HashMap<>(); for (NameSpaceContainer container : ns.list(new NamespaceKey(PathUtils.parseFullPath(parent)))) { children.put(PathUtils.constructFullPath(container.getFullPathList()), container); } return children; }
@Override public String toString() { final JobAttempt jobAttempt = getJobAttempt(); return format("{JobId: %s, SQL: %s, Dataset: %s, DatasetVersion: %s}", jobId.getId(), jobAttempt.getInfo().getSql(), PathUtils.constructFullPath(jobAttempt.getInfo().getDatasetPathList()), jobAttempt.getInfo().getDatasetVersion()); //todo }
private void createAndSaveRefresh(final JobDetails details, final RefreshDecision decision) { final boolean isFull = decision.getAccelerationSettings().getMethod() == RefreshMethod.FULL; final long updateId = isFull ? -1L : getUpdateId(job.getJobId(), job.getData()); final MaterializationMetrics metrics = ReflectionUtils.computeMetrics(job); final List<DataPartition> dataPartitions = ReflectionUtils.computeDataPartitions(job.getJobAttempt().getInfo()); final List<String> refreshPath = ReflectionUtils.getRefreshPath(job.getJobId(), job.getData(), accelerationBasePath); final Refresh refresh = ReflectionUtils.createRefresh(reflection.getId(), refreshPath, decision.getSeriesId(), decision.getSeriesOrdinal(), updateId, details, metrics, dataPartitions); logger.trace("Refresh created: {}", refresh); materializationStore.save(refresh); logger.debug("materialization {} was written to {}", ReflectionUtils.getId(materialization), PathUtils.constructFullPath(refreshPath)); }