@Override public boolean apply(DatasetConfig input) { return !keys.contains(input.getFullPathList()); } };
@Override public NamespaceKey getName() { return new NamespaceKey(config.getFullPathList()); }
@Override public NamespaceKey getName() { return new NamespaceKey(datasetConfig.getFullPathList()); }
private DremioTable putTable(NamespaceKey requestKey, DremioTable table) { if (table == null) { return null; } final DatasetConfig dataset = table.getDatasetConfig(); if (dataset != null) { final NamespaceKey canonicalizedKey = new NamespaceKey(dataset.getFullPathList()); tablesByNamespaceKey.put(canonicalizedKey, table); } tablesByNamespaceKey.put(requestKey, table); return table; }
public static BatchSchema fromDataset(DatasetConfig config){ try{ ByteString bytes = DatasetHelper.getSchemaBytes(config); if(bytes == null){ throw new IllegalStateException(String.format("Schema is currently unavailable for dataset %s.", PathUtils.constructFullPath(config.getFullPathList()))); } return deserialize(bytes); }catch(Exception ex){ throw new IllegalStateException(String.format("Schema for dataset %s is corrupt.", PathUtils.constructFullPath(config.getFullPathList())), ex); } }
private void writeRelation(XMLStreamWriter xmlStreamWriter, DatasetConfig datasetConfig) throws XMLStreamException { DatasetPath dataset = new DatasetPath(datasetConfig.getFullPathList()); final String name = dataset.getLeaf().getName(); // If dataset schema contains a dot '.' in one of its component // Tableau won't be able to open it final String schema = Joiner.on('.').join(dataset.toParentPathList()); xmlStreamWriter.writeStartElement("relation"); xmlStreamWriter.writeAttribute("name", name); xmlStreamWriter.writeAttribute("type", "table"); xmlStreamWriter.writeAttribute("table", format("[%s].[%s]", tableauEncode(schema), tableauEncode(name))); xmlStreamWriter.writeEndElement(); }
public DatasetDetails( DatasetConfig datasetConfig, int jobCount, int descendants, DatasetContainer parentDatasetContainer ) { this.displayFullPath = datasetConfig.getFullPathList(); this.owner = datasetConfig.getOwner(); this.jobCount = jobCount; this.descendants = descendants; this.createdAt = datasetConfig.getCreatedAt(); this.parentDatasetContainer = parentDatasetContainer; }
public DremioTable getTable( String datasetId, MetadataRequestOptions options ) { final DatasetConfig config = getConfig(datasetId); if (config == null) { return null; } NamespaceKey key = new NamespaceKey(config.getFullPathList()); return getTable(key, options); }
DatasetConfig completeSave(DatasetConfig newConfig, List<DatasetSplit> splits, NamespaceAttribute... attributes) throws NamespaceException{ NamespaceKey key = new NamespaceKey(newConfig.getFullPathList()); systemUserNamespaceService.addOrUpdateDataset(key, newConfig, splits, attributes); updateListener.metadataUpdated(key); return newConfig; } }
public ResourceTreeEntity(DatasetConfig datasetConfig) throws UnsupportedEncodingException { // TODO File system folder datasets can further be explored. this.type = getResourceType(datasetConfig.getType()); this.name = datasetConfig.getName(); this.fullPath = datasetConfig.getFullPathList(); this.url = null; }
public void removeFormatFromDataset(DatasetConfig config, String version) { PhysicalDatasetPath datasetPath = new PhysicalDatasetPath(config.getFullPathList()); sourceService.deletePhysicalDataset(datasetPath.getSourceName(), datasetPath, version); }
public static List<String> getSources(DatasetConfig datasetConfig) { final Set<String> sources = Sets.newHashSet(); if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { if (datasetConfig.getVirtualDataset().getFieldOriginsList() != null) { for (FieldOrigin fieldOrigin : datasetConfig.getVirtualDataset().getFieldOriginsList()) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { sources.add(origin.getTableList().get(0)); } } } } else { sources.add(datasetConfig.getFullPathList().get(0)); } return new ArrayList<>(sources); }
private Optional<AccelerationSettings> getStoredReflectionSettingsForDataset(DatasetConfig datasetConfig) { return reflectionServiceHelper.getReflectionSettings().getStoredReflectionSettings(new NamespaceKey(datasetConfig.getFullPathList())); }
@Override public List<ReflectionGoal> getRecommendedReflections(String datasetId) { DatasetConfig datasetConfig = namespaceService.get().findDatasetByUUID(datasetId); if (datasetConfig == null) { throw new NotFoundException("Dataset not found"); } ReflectionAnalyzer analyzer = new ReflectionAnalyzer(jobsService.get(), catalogService.get()); TableStats tableStats = analyzer.analyze(new NamespaceKey(datasetConfig.getFullPathList())); ReflectionSuggester suggester = new ReflectionSuggester(datasetConfig, tableStats); return suggester.getReflectionGoals(); }
@Override public ReflectionGoal apply(Ord<ReflectionDetails> reflectionDetails) { return new ReflectionGoal() .setName(String.format("AUTO_%s_AGG_%d", SqlUtils.quotedCompound(datasetConfig.getFullPathList()), reflectionDetails.i)) .setDetails(reflectionDetails.e) .setType(ReflectionType.AGGREGATION); } });
@Override public ReflectionGoal apply(Ord<ReflectionDetails> reflectionDetails) { return new ReflectionGoal() .setName(String.format("AUTO_%s_RAW_%d", SqlUtils.quotedCompound(datasetConfig.getFullPathList()), reflectionDetails.i)) .setDetails(reflectionDetails.e) .setType(ReflectionType.RAW); } });
private AccelerationSettings getAccelerationSettings(DatasetConfig config) { final ReflectionSettings reflectionSettings = reflections.getReflectionSettings(); final NamespaceKey datasetKey = new NamespaceKey(config.getFullPathList()); // not all datasets have acceleration settings return isPhysicalDataset(config.getType()) ? reflectionSettings.getReflectionSettings(datasetKey) : null; }
public void deleteHomeDataset(DatasetConfig config, String version) throws IOException, NamespaceException { FileConfig formatSettings = config.getPhysicalDataset().getFormatSettings(); homeFileTool.deleteFile(formatSettings.getLocation()); namespaceService.deleteDataset(new NamespaceKey(config.getFullPathList()), version); }
public static PhysicalDatasetConfig toPhysicalDatasetConfig(DatasetConfig datasetConfig) { checkNotNull(datasetConfig.getPhysicalDataset()); final com.dremio.service.namespace.dataset.proto.PhysicalDataset physicalDataset = datasetConfig.getPhysicalDataset(); final PhysicalDatasetConfig physicalDatasetConfig = new PhysicalDatasetConfig(); physicalDatasetConfig.setFormatSettings(physicalDataset.getFormatSettings()); physicalDatasetConfig.setFullPathList(datasetConfig.getFullPathList()); physicalDatasetConfig.setType(datasetConfig.getType()); physicalDatasetConfig.setName(datasetConfig.getName()); physicalDatasetConfig.setTag(datasetConfig.getTag()); physicalDatasetConfig.setId(datasetConfig.getId().getId()); return physicalDatasetConfig; }
public static FileConfig toFileConfig(DatasetConfig datasetConfig) { final FileConfig fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); fileConfig.setCtime(datasetConfig.getCreatedAt()); fileConfig.setTag(datasetConfig.getTag()); fileConfig.setOwner(datasetConfig.getOwner()); fileConfig.setFullPathList(datasetConfig.getFullPathList()); fileConfig.setName(datasetConfig.getName()); return fileConfig; }