private DatasetConfig getConfig(final String datasetId) { return userNamespaceService.findDatasetByUUID(datasetId); }
/** * @return dataset hash, if dataset exists in the namespace (along with all its ancestors if its a VDS). null otherwise */ private Integer computeDatasetHash(String datasetId) { final DatasetConfig dataset = namespace.findDatasetByUUID(datasetId); if (dataset == null) { return null; } try { return ReflectionUtils.computeDatasetHash(dataset, namespace); } catch (NamespaceException e) { return null; } }
@Override public List<ReflectionGoal> getRecommendedReflections(String datasetId) { DatasetConfig datasetConfig = namespaceService.get().findDatasetByUUID(datasetId); if (datasetConfig == null) { throw new NotFoundException("Dataset not found"); } ReflectionAnalyzer analyzer = new ReflectionAnalyzer(jobsService.get(), catalogService.get()); TableStats tableStats = analyzer.analyze(new NamespaceKey(datasetConfig.getFullPathList())); ReflectionSuggester suggester = new ReflectionSuggester(datasetConfig, tableStats); return suggester.getReflectionGoals(); }
private ExternalReflectionStatus.STATUS computeStatus(ExternalReflection reflection) { final DatasetConfig queryDataset = namespaceService.get().findDatasetByUUID(reflection.getQueryDatasetId()); if (queryDataset == null) { return ExternalReflectionStatus.STATUS.INVALID; final DatasetConfig targetDataset = namespaceService.get().findDatasetByUUID(reflection.getTargetDatasetId()); if (targetDataset == null) { return ExternalReflectionStatus.STATUS.INVALID;
private void handleDatasetDeletion(String datasetId, ReflectionId rId) { // make sure the corresponding dataset was not deleted if (namespaceService.findDatasetByUUID(datasetId) == null) { // dataset not found, mark goal as deleted logger.debug("dataset deleted for reflection {}", rId.getId()); final ReflectionGoal goal = userStore.get(rId); if (goal != null) { try { userStore.save(goal.setState(ReflectionGoalState.DELETED)); return; } catch (ConcurrentModificationException cme) { // someone's changed the reflection goal, we'll delete it next time the manager wakes up logger.debug("concurrent modification when trying mark reflection goal {} as deleted", rId.getId()); } } final ExternalReflection externalReflection = externalReflectionStore.get(rId.getId()); if (externalReflection != null) { externalReflectionStore.deleteExternalReflection(rId.getId()); return; } // something wrong here throw new IllegalStateException("no reflection found for an existing reflection entry: " + rId.getId()); } }
public static MaterializationDescriptor getMaterializationDescriptor(final ExternalReflection externalReflection, final NamespaceService namespaceService) throws NamespaceException { DatasetConfig queryDataset = namespaceService.findDatasetByUUID(externalReflection.getQueryDatasetId()); DatasetConfig targetDataset = namespaceService.findDatasetByUUID(externalReflection.getTargetDatasetId());
final DatasetConfig config = Preconditions.checkNotNull(namespaceService.get().findDatasetByUUID(goal.getDatasetId()), "Dataset not present for reflection %s", id.getId()); final AccelerationSettings settings = reflectionSettings.getReflectionSettings(new NamespaceKey(config.getFullPathList()));
DatasetConfig datasetConfig = namespaceService.findDatasetByUUID(reflectionGoal.getDatasetId()); if (datasetConfig == null) { System.out.println(String.format(" skipping reflection [%s] for dataset [%s] because dataset could not be found", reflectionGoal.getName(), reflectionGoal.getDatasetId()));
final DatasetConfig dataset = namespaceService.findDatasetByUUID(goal.getDatasetId()); if (dataset == null) { throw new IllegalStateException(String.format("reflection %s has no corresponding dataset", ReflectionUtils.getId(goal)));
.setId(new EntityId(datasetId)) .setFullPathList(dataPath); when(namespaceService.findDatasetByUUID(datasetId)).thenReturn(dataset);
/** * @throws UserException if reflection goal is invalid */ public void validate(ReflectionGoal goal) { ReflectionUtils.validateReflectionGoalWithoutSchema(goal); // The dataset that the reflection refers to must exist. DatasetConfig dataset = namespaceService.get().findDatasetByUUID(goal.getDatasetId()); Preconditions.checkNotNull(dataset, "datasetId must reference an existing dataset"); Catalog catalog = catalogService.get().getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()); DremioTable table = catalog.getTable(new NamespaceKey(dataset.getFullPathList())); List<ViewFieldType> schemaFields = ViewFieldsHelper.getBatchSchemaFields(table.getSchema()); Map<String, ViewFieldType> schemaMap = Maps.newHashMap(); for (ViewFieldType type : schemaFields) { schemaMap.put(type.getName(), type); } // Array, List, Map, and Union types cannot be used as dimension or measure field. They also cannot be used for sorting, partitioning or distribution in raw reflections. validateFields(goal.getDetails().getDisplayFieldList(), schemaMap, "Display", false); validateMeasures(goal.getDetails().getMeasureFieldList(), schemaMap, "Measure"); validateFields(goal.getDetails().getDistributionFieldList(), schemaMap, "Distribution", true); validateFields(goal.getDetails().getSortFieldList(), schemaMap, "Sort", true); validateFields(goal.getDetails().getPartitionFieldList(), schemaMap, "Partition", true); validateDimensions(goal.getDetails().getDimensionFieldList(), schemaMap); }
DatasetConfig datasetConfig = namespaceService.findDatasetByUUID(accelerationDataset.getId().getId());
Integer queryHash = ReflectionUtils.computeDatasetHash(queryDatasetConfig, namespaceService); String queryDatasetId = UUID.randomUUID().toString(); when(namespaceService.findDatasetByUUID(queryDatasetId)).thenReturn(queryDatasetConfig); Integer targetHash = ReflectionUtils.computeDatasetHash(targetDatasetConfig, namespaceService); String targetDatasetId = UUID.randomUUID().toString(); when(namespaceService.findDatasetByUUID(targetDatasetId)).thenReturn(targetDatasetConfig);
config.setType(VIRTUAL_DATASET); config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config);
config.setType(PHYSICAL_DATASET); config.setPhysicalDataset(physicalDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config);
final DatasetConfig dataset = namespace.findDatasetByUUID(goal.getDatasetId()); if (dataset == null) { throw new IllegalStateException(String.format("reflection %s has no corresponding dataset", ReflectionUtils.getId(goal)));
private void updateDataset(Dataset dataset, NamespaceAttribute... attributes) throws NamespaceException, IOException { Preconditions.checkArgument(dataset.getId() != null, "Dataset Id is missing."); DatasetConfig currentDatasetConfig = namespaceService.findDatasetByUUID(dataset.getId()); if (currentDatasetConfig == null) { throw new IllegalArgumentException(String.format("Could not find dataset with id [%s]", dataset.getId()));
config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config);
public static Job submitRefreshJob(JobsService jobsService, NamespaceService namespaceService, ReflectionEntry entry, Materialization materialization, String sql, JobStatusListener jobStatusListener) { final SqlQuery query = new SqlQuery(sql, SYSTEM_USERNAME); NamespaceKey datasetPathList = new NamespaceKey(namespaceService.findDatasetByUUID(entry.getDatasetId()).getFullPathList()); MaterializationSummary materializationSummary = new MaterializationSummary() .setDatasetId(entry.getDatasetId()) .setReflectionId(entry.getId().getId()) .setLayoutVersion(entry.getTag()) .setMaterializationId(materialization.getId().getId()) .setReflectionName(entry.getName()) .setReflectionType(entry.getType().toString()); return jobsService.submitJob( JobRequest.newMaterializationJobBuilder(materializationSummary, new SubstitutionSettings(ImmutableList.of())) .setSqlQuery(query) .setQueryType(QueryType.ACCELERATOR_CREATE) .setDatasetPath(datasetPathList) .build(), jobStatusListener); }
@Test public void testUpdatePDSWithChangedTypeShouldFail() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.PHYSICAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, null, null, null, null ); DatasetConfig config = new DatasetConfig(); config.setId(new EntityId(dataset.getId())); config.setFullPathList(dataset.getPath()); config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(dataset.getSql()); virtualDataset.setContextList(dataset.getSqlContext()); config.setType(VIRTUAL_DATASET); config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config); thrown.expect(IllegalArgumentException.class); catalogServiceHelper.updateCatalogItem(dataset, dataset.getId()); }