/** * Get the types of primary key (partitioning key) fields * * @param datasetDetails, * contains specific data structures for an internal dataset. * @param recordType, * the main record type. * @param metaRecordType * the auxiliary meta record type. * @return a list of IATypes, one for each corresponding primary key field. * @throws AlgebricksException */ public static List<IAType> getPartitioningKeyTypes(InternalDatasetDetails datasetDetails, ARecordType recordType, ARecordType metaRecordType) throws AlgebricksException { List<Integer> keySourceIndicators = datasetDetails.getKeySourceIndicator(); List<List<String>> partitioningKeys = datasetDetails.getPartitioningKey(); return getKeyTypes(recordType, metaRecordType, partitioningKeys, keySourceIndicators); }
public static int getPositionOfPartitioningKeyField(Dataset dataset, List<String> fieldExpr, boolean fieldFromMeta) { List<Integer> keySourceIndicator = null; IDatasetDetails datasetDetails = dataset.getDatasetDetails(); if (datasetDetails.getDatasetType() == DatasetType.INTERNAL) { keySourceIndicator = ((InternalDatasetDetails) datasetDetails).getKeySourceIndicator(); } List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); for (int i = 0; i < partitioningKeys.size(); i++) { List<String> partitioningKey = partitioningKeys.get(i); if (partitioningKey.equals(fieldExpr) && keySourceMatches(keySourceIndicator, i, fieldFromMeta)) { return i; } } return -1; }
/** * Gets the comparator factories for the primary key fields of this dataset. * * @param metadataProvider, * the metadata provider. * @return the comparator factories for the primary key fields of this dataset. * @throws AlgebricksException */ public IBinaryComparatorFactory[] getPrimaryComparatorFactories(MetadataProvider metadataProvider, ARecordType recordType, ARecordType metaType) throws AlgebricksException { IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider(); IBinaryComparatorFactoryProvider cmpFactoryProvider = storageComponentProvider.getComparatorFactoryProvider(); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[numPrimaryKeys]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); cmpFactories[i] = cmpFactoryProvider.getBinaryComparatorFactory(keyType, true); } return cmpFactories; }
/** * Gets the hash function factories for the primary key fields of this dataset. * * @param metadataProvider, * the metadata provider. * @return the hash function factories for the primary key fields of this dataset. * @throws AlgebricksException */ public IBinaryHashFunctionFactory[] getPrimaryHashFunctionFactories(MetadataProvider metadataProvider) throws AlgebricksException { ARecordType recordType = (ARecordType) metadataProvider.findType(this); ARecordType metaType = (ARecordType) metadataProvider.findMetaType(this); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); IBinaryHashFunctionFactory[] hashFuncFactories = new IBinaryHashFunctionFactory[numPrimaryKeys]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); hashFuncFactories[i] = BinaryHashFunctionFactoryProvider.INSTANCE.getBinaryHashFunctionFactory(keyType); } return hashFuncFactories; }
public ITypeTraits[] getPrimaryTypeTraits(MetadataProvider metadataProvider, ARecordType recordType, ARecordType metaType) throws AlgebricksException { IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider(); ITypeTraitProvider ttProvider = storageComponentProvider.getTypeTraitProvider(); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); ITypeTraits[] typeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = datasetType == DatasetType.EXTERNAL ? IndexingConstants.getFieldType(i) : (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); typeTraits[i] = ttProvider.getTypeTrait(keyType); } typeTraits[numPrimaryKeys] = ttProvider.getTypeTrait(recordType); if (hasMetaPart()) { typeTraits[numPrimaryKeys + 1] = ttProvider.getTypeTrait(metaType); } return typeTraits; }
List<Integer> keySourceIndicators = datasetDetails.getKeySourceIndicator(); LogicalVariable recordVar = hasMeta ? indexSearchVars.get(indexSearchVars.size() - 2) : indexSearchVars.get(indexSearchVars.size() - 1);
public static Index getPrimaryIndex(Dataset dataset) { InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails(); return new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), DatasetConfig.IndexType.BTREE, id.getPartitioningKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp()); }
partitioningKeys = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey(); if (dataset.hasMetaPart()) { keySourceIndicator = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
private void setPrimaryRecDescAndComparators() throws AlgebricksException { List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys]; primaryBloomFilterKeyFields = new int[numPrimaryKeys]; ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); List<Integer> indicators = null; if (dataset.hasMetaPart()) { indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType); primaryComparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true); primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType); primaryBloomFilterKeyFields[i] = i; } primaryRecFields[numPrimaryKeys] = payloadSerde; primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); if (dataset.hasMetaPart()) { primaryRecFields[numPrimaryKeys + 1] = payloadSerde; primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(metaType); } primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits); }
List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator();
List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails()) .getKeySourceIndicator(); ArrayList<LogicalVariable> vars = new ArrayList<>(); ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
keySourceIndicators = ((InternalDatasetDetails) ds.getDatasetDetails()).getKeySourceIndicator();
@Override public void addDataset(TxnId txnId, Dataset dataset) throws AlgebricksException, RemoteException { try { // Insert into the 'dataset' dataset. DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true); ITupleReference datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset); insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple); if (dataset.getDatasetType() == DatasetType.INTERNAL) { // Add the primary index for the dataset. InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails(); Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp()); addIndex(txnId, primaryIndex); } } catch (HyracksDataException e) { if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() == ErrorCode.DUPLICATE_KEY) { throw new AlgebricksException("A dataset with this name " + dataset.getDatasetName() + " already exists in dataverse '" + dataset.getDataverseName() + "'.", e); } else { throw new AlgebricksException(e); } } }
List<Integer> indicators = null; if (dataset.hasMetaPart()) { indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
List<Integer> keySourceIndicator = getKeySourceIndicator(); boolean needSerialization = false; if (keySourceIndicator != null) {