public static Dataset createTransactionDataset(Dataset dataset) { ExternalDatasetDetails originalDsd = (ExternalDatasetDetails) dataset.getDatasetDetails(); ExternalDatasetDetails dsd = new ExternalDatasetDetails(originalDsd.getAdapter(), originalDsd.getProperties(), originalDsd.getTimestamp(), TransactionState.BEGIN); return new Dataset(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getItemTypeDataverseName(), dataset.getItemTypeName(), dataset.getNodeGroupName(), dataset.getCompactionPolicy(), dataset.getCompactionPolicyProperties(), dsd, dataset.getHints(), DatasetType.EXTERNAL, dataset.getDatasetId(), dataset.getPendingOp()); }
public static DataSource lookupSourceInMetadata(IClusterStateManager clusterStateManager, MetadataTransactionContext mdTxnCtx, DataSourceId aqlId) throws AlgebricksException { Dataset dataset = findDataset(mdTxnCtx, aqlId.getDataverseName(), aqlId.getDatasourceName()); if (dataset == null) { throw new AlgebricksException("Datasource with id " + aqlId + " was not found."); } IAType itemType = findType(mdTxnCtx, dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); IAType metaItemType = findType(mdTxnCtx, dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); INodeDomain domain = findNodeDomain(clusterStateManager, mdTxnCtx, dataset.getNodeGroupName()); byte datasourceType = dataset.getDatasetType().equals(DatasetType.EXTERNAL) ? DataSource.Type.EXTERNAL_DATASET : DataSource.Type.INTERNAL_DATASET; return new DatasetDataSource(aqlId, dataset, itemType, metaItemType, datasourceType, dataset.getDatasetDetails(), domain); } }
public static int[] createBTreeFieldsWhenThereisAFilter(Dataset dataset) { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return null; } List<String> filterField = getFilterField(dataset); if (filterField == null) { return null; } List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); int valueFields = dataset.hasMetaPart() ? 2 : 1; int[] btreeFields = new int[partitioningKeys.size() + valueFields]; for (int i = 0; i < btreeFields.length; ++i) { btreeFields[i] = i; } return btreeFields; }
public CompiledIndexCompactStatement(Dataset dataset, Index index) { super(dataset.getDataverseName(), dataset.getDatasetName()); this.dataset = dataset; this.index = index; }
public static ARecordType getMetaType(MetadataProvider metadataProvider, Dataset dataset) throws AlgebricksException { if (dataset.hasMetaPart()) { return (ARecordType) metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); } return null; }
throws Exception { Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>(); if (getDatasetType() == DatasetType.INTERNAL) { new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(), getMetaItemTypeDataverseName(), getMetaItemTypeName(), getNodeGroupName(), getCompactionPolicy(), getCompactionPolicyProperties(), getDatasetDetails(), getHints(), getDatasetType(), getDatasetId(), MetadataUtil.PENDING_DROP_OP, getCompressionScheme())); new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(), getNodeGroupName(), getCompactionPolicy(), getCompactionPolicyProperties(), getDatasetDetails(), getHints(), getDatasetType(), getDatasetId(), MetadataUtil.PENDING_DROP_OP));
private void confirmDatatypeIsUnusedByDatasets(TxnId txnId, String dataverseName, String datatypeName) throws AlgebricksException, RemoteException { // If any dataset uses this type, throw an error List<Dataset> datasets = getAllDatasets(txnId); for (Dataset set : datasets) { if (set.getItemTypeName().equals(datatypeName) && set.getItemTypeDataverseName().equals(dataverseName)) { throw new AlgebricksException("Cannot drop type " + dataverseName + "." + datatypeName + " being used by dataset " + set.getDataverseName() + "." + set.getDatasetName()); } } }
aString.setValue(dataset.getDataverseName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(dataset.getDatasetName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(dataset.getDataverseName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATAVERSENAME_FIELD_INDEX, fieldValue); aString.setValue(dataset.getDatasetName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATASETNAME_FIELD_INDEX, fieldValue); aString.setValue(dataset.getItemTypeDataverseName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATATYPEDATAVERSENAME_FIELD_INDEX, fieldValue); aString.setValue(dataset.getItemTypeName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATATYPENAME_FIELD_INDEX, fieldValue); aString.setValue(dataset.getDatasetType().toString()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATASETTYPE_FIELD_INDEX, fieldValue); aString.setValue(dataset.getNodeGroupName()); stringSerde.serialize(aString, fieldValue.getDataOutput());
int[] keyFields, boolean propagateFilter, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean isIndexOnlyPlan) throws AlgebricksException { int numPrimaryKeys = dataset.getPrimaryKeys().size(); Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), indexName); if (secondaryIndex == null) { throw new AlgebricksException( "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName()); byte[] failValueForIndexOnlyPlan = null; if (isIndexOnlyPlan) { ARecordType recType = (ARecordType) findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); List<List<String>> secondaryKeyFields = secondaryIndex.getKeyFieldNames(); List<IAType> secondaryKeyTypes = secondaryIndex.getKeyFieldTypes(); dataset.getSearchCallbackFactory(storageComponentProvider, secondaryIndex, IndexOperation.SEARCH, primaryKeyFields, primaryKeyFieldsInSecondaryIndex, isIndexOnlyPlan); RTreeSearchOperatorDescriptor rtreeSearchOp; IIndexDataflowHelperFactory indexDataflowHelperFactory = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), spPc.first); if (dataset.getDatasetType() == DatasetType.INTERNAL) { rtreeSearchOp = new RTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, keyFields, true, true, indexDataflowHelperFactory, retainInput, retainMissing, context.getMissingWriterFactory(),
public static Index getPrimaryIndex(Dataset dataset) { InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails(); return new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), DatasetConfig.IndexType.BTREE, id.getPartitioningKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp()); }
public ITypeTraits[] getPrimaryTypeTraits(MetadataProvider metadataProvider, ARecordType recordType, ARecordType metaType) throws AlgebricksException { IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider(); ITypeTraitProvider ttProvider = storageComponentProvider.getTypeTraitProvider(); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); ITypeTraits[] typeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = datasetType == DatasetType.EXTERNAL ? IndexingConstants.getFieldType(i) : (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); typeTraits[i] = ttProvider.getTypeTrait(keyType); } typeTraits[numPrimaryKeys] = ttProvider.getTypeTrait(recordType); if (hasMetaPart()) { typeTraits[numPrimaryKeys + 1] = ttProvider.getTypeTrait(metaType); } return typeTraits; }
private void addDataset(ICcApplicationContext appCtx, Dataset source, int datasetPostfix, boolean abort) throws Exception { Dataset dataset = new Dataset(source.getDataverseName(), "ds_" + datasetPostfix, source.getDataverseName(), source.getDatasetType().name(), source.getNodeGroupName(), NoMergePolicyFactory.NAME, null, source.getDatasetDetails(), source.getHints(), DatasetConfig.DatasetType.INTERNAL, datasetPostfix, 0); MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext writeTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(writeTxn); try { MetadataManager.INSTANCE.addDataset(writeTxn, dataset); if (abort) { MetadataManager.INSTANCE.abortTransaction(writeTxn); } else { MetadataManager.INSTANCE.commitTransaction(writeTxn); } } finally { metadataProvider.getLocks().unlock(); } } }
dataset.getDataverseName(), dataset.getItemTypeName()).getDatatype(); ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails(); LookupAdapterFactory<?> adapterFactory = AdapterFactoryProvider.getLookupAdapterFactory( getApplicationContext().getServiceContext(), datasetDetails.getProperties(), itemType, ridIndexes, retainInput, retainMissing, context.getMissingWriterFactory()); String fileIndexName = IndexingConstants.getFilesIndexName(dataset.getDatasetName()); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc = metadataProvider.getSplitProviderAndConstraints(dataset, fileIndexName); Index fileIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), fileIndexName); dataset.getSearchCallbackFactory(storageComponentProvider, fileIndex, IndexOperation.SEARCH, null);
Set<String> sourceNodes = new HashSet<>(metadataProvider.findNodes(sourceDataset.getNodeGroupName())); String nodeGroupName = DatasetUtil.createNodeGroupForNewDataset(sourceDataset.getDataverseName(), sourceDataset.getDatasetName(), sourceDataset.getRebalanceCount() + 1, targetNcNames, metadataProvider); targetDataset = sourceDataset.getTargetDatasetForRebalance(nodeGroupName); sourceDataset.getDatasetName(), sourceDataset.getNodeGroupName(), sourceNodes, targetDataset.getNodeGroupName(), targetNcNames);
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException { switch (dataset.getDatasetType()) { case EXTERNAL: if (tupleFilterFactory != null || outputLimit >= 0) { String itemTypeName = externalDataset.getItemTypeName(); IAType itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype(); ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails(); IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset, edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null);
public IAType findType(Dataset dataset) throws AlgebricksException { return findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); }
private static int[] getBloomFilterFields(Dataset dataset, Index index) throws AlgebricksException { if (index.isPrimaryIndex()) { return dataset.getPrimaryBloomFilterFields(); } else if (dataset.getDatasetType() == DatasetType.EXTERNAL) { if (index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) { return FilesIndexDescription.BLOOM_FILTER_FIELDS; } else { return new int[] { index.getKeyFieldNames().size() }; } } int numKeys = index.getKeyFieldNames().size(); int[] bloomFilterKeyFields = new int[numKeys]; for (int i = 0; i < numKeys; i++) { bloomFilterKeyFields[i] = i; } return bloomFilterKeyFields; } }
/** * Creates output variables for the given unnest-map or left-outer-unnestmap operator * that does a secondary index lookup. * The order: SK, PK, [Optional: the result of a instantTryLock on PK] */ public static void appendSecondaryIndexOutputVars(Dataset dataset, ARecordType recordType, ARecordType metaRecordType, Index index, IOptimizationContext context, List<LogicalVariable> dest, boolean requireResultOfInstantTryLock) throws AlgebricksException { int numPrimaryKeys; if (dataset.getDatasetType() == DatasetType.EXTERNAL) { numPrimaryKeys = IndexingConstants .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); } else { numPrimaryKeys = dataset.getPrimaryKeys().size(); } int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recordType, metaRecordType); // In case of an inverted-index search, secondary keys will not be generated. int numVars = isInvertedIndex(index) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys; // If it's an index-only plan, add one more variable to put the result of instantTryLock on PK - // whether this lock can be granted on a primary key. // If it is granted, then we don't need to do a post verification (select). // If it is not granted, then we need to do a secondary index lookup, do a primary index lookup, and select. if (requireResultOfInstantTryLock) { numVars += 1; } for (int i = 0; i < numVars; i++) { dest.add(context.newVar()); } }
/** * Get the types of primary key (partitioning key) fields * * @param dataset, * the dataset to consider. * @param recordType, * the main record type. * @param metaRecordType * the auxiliary meta record type. * @return a list of IATypes, one for each corresponding primary key field. * @throws AlgebricksException */ public static List<IAType> getPartitoningKeyTypes(Dataset dataset, ARecordType recordType, ARecordType metaRecordType) throws AlgebricksException { if (dataset.getDatasetType() != DatasetType.INTERNAL) { return null; } InternalDatasetDetails datasetDetails = (InternalDatasetDetails) dataset.getDatasetDetails(); return getPartitioningKeyTypes(datasetDetails, recordType, metaRecordType); }
private void fillPKVars(DataSourceScanOperator dataScan, List<LogicalVariable> pkVars) { pkVars.clear(); DatasetDataSource datasetDataSource = (DatasetDataSource) dataScan.getDataSource(); pkVars.clear(); if (datasetDataSource.getDataset().getDatasetDetails() instanceof InternalDatasetDetails) { int numPKs = datasetDataSource.getDataset().getPrimaryKeys().size(); for (int i = 0; i < numPKs; i++) { pkVars.add(dataScan.getVariables().get(i)); } } }