@Override public boolean isScanAccessPathALeaf() { return dataset.getDatasetType() == DatasetType.EXTERNAL; }
public static int[] createFilterFields(Dataset dataset) { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return null; } List<String> filterField = getFilterField(dataset); if (filterField == null) { return null; } List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); int numKeys = partitioningKeys.size(); int[] filterFields = new int[1]; filterFields[0] = numKeys + 1; return filterFields; }
public DatasetDataSource(DataSourceId id, Dataset dataset, IAType itemType, IAType metaItemType, byte datasourceType, IDatasetDetails datasetDetails, INodeDomain datasetDomain) throws AlgebricksException { super(id, itemType, metaItemType, datasourceType, datasetDomain); this.dataset = dataset; switch (dataset.getDatasetType()) { case INTERNAL: initInternalDataset(itemType, metaItemType, datasetDetails); break; case EXTERNAL: initExternalDataset(itemType); break; } }
/** * Get the types of primary key (partitioning key) fields * * @param dataset, * the dataset to consider. * @param recordType, * the main record type. * @param metaRecordType * the auxiliary meta record type. * @return a list of IATypes, one for each corresponding primary key field. * @throws AlgebricksException */ public static List<IAType> getPartitoningKeyTypes(Dataset dataset, ARecordType recordType, ARecordType metaRecordType) throws AlgebricksException { if (dataset.getDatasetType() != DatasetType.INTERNAL) { return null; } InternalDatasetDetails datasetDetails = (InternalDatasetDetails) dataset.getDatasetDetails(); return getPartitioningKeyTypes(datasetDetails, recordType, metaRecordType); }
public static int[] createBTreeFieldsWhenThereisAFilter(Dataset dataset) { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return null; } List<String> filterField = getFilterField(dataset); if (filterField == null) { return null; } List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); int valueFields = dataset.hasMetaPart() ? 2 : 1; int[] btreeFields = new int[partitioningKeys.size() + valueFields]; for (int i = 0; i < btreeFields.length; ++i) { btreeFields[i] = i; } return btreeFields; }
public static ITypeTraits[] computeFilterTypeTraits(Dataset dataset, ARecordType itemType) throws AlgebricksException { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return null; } List<String> filterField = getFilterField(dataset); if (filterField == null) { return null; } ITypeTraits[] typeTraits = new ITypeTraits[1]; IAType type = itemType.getSubFieldType(filterField); typeTraits[0] = TypeTraitProvider.INSTANCE.getTypeTrait(type); return typeTraits; }
public Dataset addDatasetIfNotExists(Dataset dataset) { synchronized (datasets) { synchronized (indexes) { // Add the primary index associated with the dataset, if the dataset is an // internal dataset. if (dataset.getDatasetType() == DatasetType.INTERNAL) { Index index = IndexUtil.getPrimaryIndex(dataset); addIndexIfNotExistsInternal(index); } Map<String, Dataset> m = datasets.get(dataset.getDataverseName()); if (m == null) { m = new HashMap<>(); datasets.put(dataset.getDataverseName(), m); } if (!m.containsKey(dataset.getDatasetName())) { return m.put(dataset.getDatasetName(), dataset); } return null; } } }
public static IBinaryComparatorFactory[] computeFilterBinaryComparatorFactories(Dataset dataset, ARecordType itemType, IBinaryComparatorFactoryProvider comparatorFactoryProvider) throws AlgebricksException { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return null; } List<String> filterField = getFilterField(dataset); if (filterField == null) { return null; } IBinaryComparatorFactory[] bcfs = new IBinaryComparatorFactory[1]; IAType type = itemType.getSubFieldType(filterField); bcfs[0] = comparatorFactoryProvider.getBinaryComparatorFactory(type, true); return bcfs; }
public static Dataset validateIfDatasetExists(MetadataProvider metadataProvider, String dataverse, String datasetName) throws AlgebricksException { Dataset dataset = metadataProvider.findDataset(dataverse, datasetName); if (dataset == null) { throw new CompilationException("Unknown target dataset :" + datasetName); } if (!dataset.getDatasetType().equals(DatasetType.INTERNAL)) { throw new CompilationException("Statement not applicable. Dataset " + datasetName + " is not of required type " + DatasetType.INTERNAL); } return dataset; }
private static int[] getBloomFilterFields(Dataset dataset, Index index) throws AlgebricksException { if (index.isPrimaryIndex()) { return dataset.getPrimaryBloomFilterFields(); } else if (dataset.getDatasetType() == DatasetType.EXTERNAL) { if (index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) { return FilesIndexDescription.BLOOM_FILTER_FIELDS; } else { return new int[] { index.getKeyFieldNames().size() }; } } int numKeys = index.getKeyFieldNames().size(); int[] bloomFilterKeyFields = new int[numKeys]; for (int i = 0; i < numKeys; i++) { bloomFilterKeyFields[i] = i; } return bloomFilterKeyFields; } }
protected void writeDatasetDetailsRecordType(IARecordBuilder recordBuilder, Dataset dataset, DataOutput dataOutput) throws HyracksDataException { dataset.getDatasetDetails().writeDatasetDetailsRecordType(fieldValue.getDataOutput()); switch (dataset.getDatasetType()) { case INTERNAL: recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_INTERNALDETAILS_FIELD_INDEX, fieldValue); break; case EXTERNAL: recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_EXTERNALDETAILS_FIELD_INDEX, fieldValue); break; } }
@Override public List<List<String>> getPrimaryKeys() { if (getDatasetType() == DatasetType.EXTERNAL) { return IndexingConstants.getRIDKeys(((ExternalDatasetDetails) getDatasetDetails()).getProperties()); } return ((InternalDatasetDetails) getDatasetDetails()).getPartitioningKey(); }
/** * Creates output variables for the given unnest-map or left-outer-unnestmap operator * that does a secondary index lookup. * The order: SK, PK, [Optional: the result of a instantTryLock on PK] */ public static void appendSecondaryIndexOutputVars(Dataset dataset, ARecordType recordType, ARecordType metaRecordType, Index index, IOptimizationContext context, List<LogicalVariable> dest, boolean requireResultOfInstantTryLock) throws AlgebricksException { int numPrimaryKeys; if (dataset.getDatasetType() == DatasetType.EXTERNAL) { numPrimaryKeys = IndexingConstants .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); } else { numPrimaryKeys = dataset.getPrimaryKeys().size(); } int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recordType, metaRecordType); // In case of an inverted-index search, secondary keys will not be generated. int numVars = isInvertedIndex(index) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys; // If it's an index-only plan, add one more variable to put the result of instantTryLock on PK - // whether this lock can be granted on a primary key. // If it is granted, then we don't need to do a post verification (select). // If it is not granted, then we need to do a secondary index lookup, do a primary index lookup, and select. if (requireResultOfInstantTryLock) { numVars += 1; } for (int i = 0; i < numVars; i++) { dest.add(context.newVar()); } }
if (index.isPrimaryIndex()) { return primaryTypeTraits; } else if (dataset.getDatasetType() == DatasetType.EXTERNAL && index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) { return FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS;
public static JobSpecification dropDatasetJobSpec(Dataset dataset, MetadataProvider metadataProvider) throws AlgebricksException, ACIDException { LOGGER.info("DROP DATASET: " + dataset); if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext()); } JobSpecification specPrimary = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext()); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset); IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory( metadataProvider.getStorageComponentProvider().getStorageManager(), splitsAndConstraint.first); IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary, indexHelperFactory); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop, splitsAndConstraint.second); specPrimary.addRoot(primaryBtreeDrop); return specPrimary; }
private static int[] getBloomFilterFields(Dataset dataset, Index index) throws AlgebricksException { if (index.isPrimaryIndex()) { return dataset.getPrimaryBloomFilterFields(); } else if (dataset.getDatasetType() == DatasetType.EXTERNAL) { if (index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) { return FilesIndexDescription.BLOOM_FILTER_FIELDS; } else { return new int[] { index.getKeyFieldNames().size() }; } } int numKeys = index.getKeyFieldNames().size(); int[] bloomFilterKeyFields = new int[numKeys]; for (int i = 0; i < numKeys; i++) { bloomFilterKeyFields[i] = i; } return bloomFilterKeyFields; } }
public static DataSource lookupSourceInMetadata(IClusterStateManager clusterStateManager, MetadataTransactionContext mdTxnCtx, DataSourceId aqlId) throws AlgebricksException { Dataset dataset = findDataset(mdTxnCtx, aqlId.getDataverseName(), aqlId.getDatasourceName()); if (dataset == null) { throw new AlgebricksException("Datasource with id " + aqlId + " was not found."); } IAType itemType = findType(mdTxnCtx, dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); IAType metaItemType = findType(mdTxnCtx, dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); INodeDomain domain = findNodeDomain(clusterStateManager, mdTxnCtx, dataset.getNodeGroupName()); byte datasourceType = dataset.getDatasetType().equals(DatasetType.EXTERNAL) ? DataSource.Type.EXTERNAL_DATASET : DataSource.Type.INTERNAL_DATASET; return new DatasetDataSource(aqlId, dataset, itemType, metaItemType, datasourceType, dataset.getDatasetDetails(), domain); } }
private DatasetDataSource validateDatasetInfo(MetadataProvider metadataProvider, String dataverseName, String datasetName, SourceLocation sourceLoc) throws AlgebricksException { Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); if (dataset == null) { throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc, datasetName, dataverseName); } if (dataset.getDatasetType() == DatasetType.EXTERNAL) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Cannot write output to an external dataset."); } DataSourceId sourceId = new DataSourceId(dataverseName, datasetName); IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); INodeDomain domain = metadataProvider.findNodeDomain(dataset.getNodeGroupName()); return new DatasetDataSource(sourceId, dataset, itemType, metaItemType, DataSource.Type.INTERNAL_DATASET, dataset.getDatasetDetails(), domain); }
@Override public void addDataset(TxnId txnId, Dataset dataset) throws AlgebricksException, RemoteException { try { // Insert into the 'dataset' dataset. DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true); ITupleReference datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset); insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple); if (dataset.getDatasetType() == DatasetType.INTERNAL) { // Add the primary index for the dataset. InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails(); Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp()); addIndex(txnId, primaryIndex); } } catch (HyracksDataException e) { if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() == ErrorCode.DUPLICATE_KEY) { throw new AlgebricksException("A dataset with this name " + dataset.getDatasetName() + " already exists in dataverse '" + dataset.getDataverseName() + "'.", e); } else { throw new AlgebricksException(e); } } }
private void addDataset(ICcApplicationContext appCtx, Dataset source, int datasetPostfix, boolean abort) throws Exception { Dataset dataset = new Dataset(source.getDataverseName(), "ds_" + datasetPostfix, source.getDataverseName(), source.getDatasetType().name(), source.getNodeGroupName(), NoMergePolicyFactory.NAME, null, source.getDatasetDetails(), source.getHints(), DatasetConfig.DatasetType.INTERNAL, datasetPostfix, 0); MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext writeTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(writeTxn); try { MetadataManager.INSTANCE.addDataset(writeTxn, dataset); if (abort) { MetadataManager.INSTANCE.abortTransaction(writeTxn); } else { MetadataManager.INSTANCE.commitTransaction(writeTxn); } } finally { metadataProvider.getLocks().unlock(); } } }