final NamespaceKey parentKey = new NamespaceKey(parentDataset.getDatasetPathList()); if (!parentsToLevelMap.containsKey(parentKey)) { parentsToLevelMap.put(parentKey, parentDataset.getLevel() + 1); final NamespaceKey parentKey = new NamespaceKey(grandParentDataset.getDatasetPathList()); if (!parentsToLevelMap.containsKey(parentKey)) { parentsToLevelMap.put(parentKey, grandParentDataset.getLevel() + 1); grandParents.add(new ParentDataset().setDatasetPathList(entry.getKey().getPathComponents()).setLevel(entry.getValue()));
@Nullable @Override public String apply(@Nullable final ParentDataset parent) { return makePathString(parent.getDatasetPathList()); } })
public static void applyQueryMetadata(final VirtualDatasetUI dataset, final QueryMetadata metadata) { final List<ViewFieldType> viewFieldTypesList = Views.viewToFieldTypes(Views.relDataTypeToFieldType(metadata.getRowType())); dataset.setCalciteFieldsList(viewFieldTypesList); if (metadata.getBatchSchema() != null) { dataset.setSqlFieldsList(ViewFieldsHelper.getBatchSchemaFields(metadata.getBatchSchema())); dataset.setRecordSchema(metadata.getBatchSchema().toByteString()); } else { dataset.setSqlFieldsList(viewFieldTypesList); } Optional<List<ParentDatasetInfo>> parents = metadata.getParents(); if (parents.isPresent()) { List<ParentDataset> otherParents = new ArrayList<>(); for(ParentDatasetInfo parent : parents.get()){ otherParents.add(new ParentDataset() .setDatasetPathList(parent.getDatasetPathList()) .setType(parent.getType()) .setLevel(1)); } dataset.setParentsList(otherParents); } Optional<List<FieldOrigin>> fieldOrigins = metadata.getFieldOrigins(); if (fieldOrigins.isPresent()) { dataset.setFieldOriginsList(fieldOrigins.get()); } Optional<List<ParentDataset>> grandParents = metadata.getGrandParents(); if (grandParents.isPresent()) { dataset.setGrandParentsList(grandParents.get()); } updateDerivationAfterLearningOriginsAndAncestors(dataset); }
if(message.parents == null) message.parents = new ArrayList<ParentDataset>(); message.parents.add(input.mergeObject(null, ParentDataset.getSchema())); break; if(message.grandParents == null) message.grandParents = new ArrayList<ParentDataset>(); message.grandParents.add(input.mergeObject(null, ParentDataset.getSchema())); break;
if (isUnsavedDirectPhysicalDataset && atHistoryTip) { // example select * mongo.yelp.review ParentDataset parentDataset = vds.getParentsList().get(0); displayFullPath = parentDataset.getDatasetPathList(); // There is always going to be one parent since its tmp dataset created directly from a physical dataset. datasetType = parentDataset.getType(); } else if(isUnsaved && vds.getDerivation() == Derivation.DERIVED_PHYSICAL) { displayFullPath = vds.getParentsList().get(0).getDatasetPathList(); } else { displayFullPath = fullPath;
public ParentDataset newMessage() { return new ParentDataset(); }
/** * @return true if pathList is an ancestor (parent or grand parent) of the virtual dataset */ private static boolean isAncestor(VirtualDatasetUI vds, List<String> pathList) { List<ParentDataset> parents = vds.getParentsList(); if (parents != null) { for (ParentDataset parent : parents) { if (pathList.equals(parent.getDatasetPathList())) { return true; } } } List<ParentDataset> grandParents = vds.getGrandParentsList(); if (grandParents != null) { for (ParentDataset parent : grandParents) { if (pathList.equals(parent.getDatasetPathList())) { return true; } } } return false; }
if(message.parents == null) message.parents = new ArrayList<com.dremio.service.namespace.dataset.proto.ParentDataset>(); message.parents.add(input.mergeObject(null, com.dremio.service.namespace.dataset.proto.ParentDataset.getSchema())); break; if(message.grandParents == null) message.grandParents = new ArrayList<com.dremio.service.namespace.dataset.proto.ParentDataset>(); message.grandParents.add(input.mergeObject(null, com.dremio.service.namespace.dataset.proto.ParentDataset.getSchema())); break;
private void addAllParents(DocumentWriter writer, List<ParentDataset> parents, List<ParentDataset> grandParents) { if (notEmpty(parents)) { grandParents = listNotNull(grandParents); int i = 0; final String[] allParents = new String[parents.size() + grandParents.size()]; for (ParentDataset parent : parents) { allParents[i++] = PathUtils.constructFullPath(parent.getDatasetPathList()); } for (ParentDataset grandParent : grandParents) { allParents[i++] = PathUtils.constructFullPath(grandParent.getDatasetPathList()); } writer.write(DATASET_ALLPARENTS, allParents); } } }
private void addParents(DocumentWriter writer, List<ParentDataset> parentDatasetList) { if (notEmpty(parentDatasetList)) { final String[] parents = new String[parentDatasetList.size()]; int i = 0; for (ParentDataset parent : parentDatasetList) { parents[i++] = PathUtils.constructFullPath(parent.getDatasetPathList()); } writer.write(DATASET_PARENTS, parents); } }
if(message.grandParents == null) message.grandParents = new ArrayList<com.dremio.service.namespace.dataset.proto.ParentDataset>(); message.grandParents.add(input.mergeObject(null, com.dremio.service.namespace.dataset.proto.ParentDataset.getSchema())); break;
@GET @Path("parents") @Produces(APPLICATION_JSON) public List<ParentDatasetUI> getParents() throws DatasetNotFoundException, NamespaceException { final VirtualDatasetUI virtualDatasetUI = datasetService.get(datasetPath, version); final List<ParentDatasetUI> parentDatasetUIs = Lists.newArrayList(); final List<NamespaceKey> parentDatasetPaths = Lists.newArrayList(); for (ParentDataset parentDataset : virtualDatasetUI.getParentsList()) { parentDatasetPaths.add(new NamespaceKey(parentDataset.getDatasetPathList())); } for (NameSpaceContainer nameSpaceContainer : datasetService.getNamespaceService().getEntities(parentDatasetPaths)) { if (nameSpaceContainer != null && nameSpaceContainer.getType() == Type.DATASET) { parentDatasetUIs.add(new ParentDatasetUI(nameSpaceContainer.getFullPathList(), nameSpaceContainer.getDataset().getType())); } } return parentDatasetUIs; }
if(message.grandParents == null) message.grandParents = new ArrayList<com.dremio.service.namespace.dataset.proto.ParentDataset>(); message.grandParents.add(input.mergeObject(null, com.dremio.service.namespace.dataset.proto.ParentDataset.getSchema())); break;
public static Integer computeDatasetHash(DatasetConfig dataset, NamespaceService namespaceService) throws NamespaceException { Queue<DatasetConfig> q = new LinkedList<>(); q.add(dataset); int hash = 1; while (!q.isEmpty()) { dataset = q.poll(); if (isPhysicalDataset(dataset.getType())) { hash = 31 * hash + (dataset.getRecordSchema() == null ? 1 : dataset.getRecordSchema().hashCode()); } else { hash = 31 * hash + dataset.getVirtualDataset().getSql().hashCode(); for (ParentDataset parent : dataset.getVirtualDataset().getParentsList()) { q.add(namespaceService.getDataset(new NamespaceKey(parent.getDatasetPathList()))); } } } return hash; }
private static void updateDerivationAfterLearningOriginsAndAncestors(VirtualDatasetUI newDataset){ // only resolve if we need to, otherwise we should leave the previous derivation alone. (e.g. during a transform) if(newDataset.getDerivation() != Derivation.DERIVED_UNKNOWN){ return; } // if we have don't have one parent, we must have had issues detecting parents of SQL we generated, fallback. if(newDataset.getParentsList() != null && newDataset.getParentsList().size() != 1){ newDataset.setDerivation(Derivation.UNKNOWN); return; } final Set<List<String>> origins = new HashSet<>(); for(FieldOrigin col : listNotNull(newDataset.getFieldOriginsList())) { for(Origin colOrigin : listNotNull(col.getOriginsList())){ origins.add(colOrigin.getTableList()); } } // logic: if we have a single parent and that parent is also the only // table listed in field origins, then we are derived from a physical // dataset. Otherwise, we are a virtual dataset. if(origins.size() == 1 && origins.iterator().next().equals(newDataset.getParentsList().get(0).getDatasetPathList())){ newDataset.setDerivation(Derivation.DERIVED_PHYSICAL); } else { newDataset.setDerivation(Derivation.DERIVED_VIRTUAL); } }
output.writeObject(19, grandParents, com.dremio.service.namespace.dataset.proto.ParentDataset.getSchema(), true);
allDatasets.add(new NamespaceKey(parentDataset.getDatasetPathList()));
output.writeObject(19, grandParents, com.dremio.service.namespace.dataset.proto.ParentDataset.getSchema(), true);