private void indexOrigins(final List<FieldOrigin> fieldOriginsList, Set<List<String>> parents, Map<Origin, String> refs) { for (FieldOrigin fieldOrigin : fieldOriginsList) { List<Origin> originsList = listNotNull(fieldOrigin.getOriginsList()); // if size != 1 then the field is derived. if (originsList.size() == 1) { Origin origin = originsList.get(0); // if the field was derived we can not use it // or we would need to be able to invert the expression // it has to be straight unchanged if (!origin.getDerived()) { parents.add(origin.getTableList()); // TODO(Julien): if the same column is referred more than once we could recommend all combinations by making refs a multimap refs.put(origin, fieldOrigin.getName()); } } } }
private void addSourcesAndOrigins(DocumentWriter writer, List<FieldOrigin> fieldOrigins) { if (notEmpty(fieldOrigins)) { Set<String> sources = Sets.newHashSet(); final List<String> empty = ImmutableList.of(); for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { // DX-3999: fix this in calcite final List<String> path = Optional.fromNullable(origin.getTableList()).or(empty); if (path.isEmpty()) { continue; } NamespaceKey dataset = new NamespaceKey(origin.getTableList()); sources.add(dataset.getRoot()); } } writer.write(DATASET_SOURCES, sources.toArray(new String[0])); } }
public static List<String> getSources(DatasetConfig datasetConfig) { final Set<String> sources = Sets.newHashSet(); if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { if (datasetConfig.getVirtualDataset().getFieldOriginsList() != null) { for (FieldOrigin fieldOrigin : datasetConfig.getVirtualDataset().getFieldOriginsList()) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { sources.add(origin.getTableList().get(0)); } } } } else { sources.add(datasetConfig.getFullPathList().get(0)); } return new ArrayList<>(sources); }
private static void updateDerivationAfterLearningOriginsAndAncestors(VirtualDatasetUI newDataset){ // only resolve if we need to, otherwise we should leave the previous derivation alone. (e.g. during a transform) if(newDataset.getDerivation() != Derivation.DERIVED_UNKNOWN){ return; } // if we have don't have one parent, we must have had issues detecting parents of SQL we generated, fallback. if(newDataset.getParentsList() != null && newDataset.getParentsList().size() != 1){ newDataset.setDerivation(Derivation.UNKNOWN); return; } final Set<List<String>> origins = new HashSet<>(); for(FieldOrigin col : listNotNull(newDataset.getFieldOriginsList())) { for(Origin colOrigin : listNotNull(col.getOriginsList())){ origins.add(colOrigin.getTableList()); } } // logic: if we have a single parent and that parent is also the only // table listed in field origins, then we are derived from a physical // dataset. Otherwise, we are a virtual dataset. if(origins.size() == 1 && origins.iterator().next().equals(newDataset.getParentsList().get(0).getDatasetPathList())){ newDataset.setDerivation(Derivation.DERIVED_PHYSICAL); } else { newDataset.setDerivation(Derivation.DERIVED_VIRTUAL); } }
private void addAllParents(DocumentWriter writer, List<ParentDataset> parents, List<ParentDataset> grandParents) { if (notEmpty(parents)) { grandParents = listNotNull(grandParents); int i = 0; final String[] allParents = new String[parents.size() + grandParents.size()]; for (ParentDataset parent : parents) { allParents[i++] = PathUtils.constructFullPath(parent.getDatasetPathList()); } for (ParentDataset grandParent : grandParents) { allParents[i++] = PathUtils.constructFullPath(grandParent.getDatasetPathList()); } writer.write(DATASET_ALLPARENTS, allParents); } } }
@Override public JoinRecommendations recommendJoins(Dataset dataset) { final List<FieldOrigin> fieldOriginsList = listNotNull(dataset.getDatasetConfig().getFieldOriginsList()); if (fieldOriginsList.isEmpty()) {
List<Column> newColumns = new ArrayList<>(); List<Column> groupBys = new ArrayList<>(); List<Dimension> dimensions = listNotNull(groupBy.getColumnsDimensionsList()); List<Measure> measures = listNotNull(groupBy.getColumnsMeasuresList());
if (fieldOrigins != null) { for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { List<String> tableList = listNotNull(origin.getTableList()); if (!tableList.isEmpty()) { final NamespaceKey parentDatasetPath = new NamespaceKey(tableList); for (ParentDataset parentDataset : listNotNull(jobInfo.getGrandParentsList())) { allDatasets.add(new NamespaceKey(parentDataset.getDatasetPathList()));