public static Index getPrimaryIndex(Dataset dataset) { InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails(); return new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), DatasetConfig.IndexType.BTREE, id.getPartitioningKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp()); }
public static List<String> getFilterField(Dataset dataset) { return ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField(); }
public LoadableDataSource(Dataset targetDataset, IAType itemType, IAType metaItemType, String adapter, Map<String, String> properties) throws AlgebricksException, IOException { super(new DataSourceId("loadable_dv", "loadable_ds"), itemType, metaItemType, Type.LOADABLE, null); this.targetDataset = targetDataset; this.adapter = adapter; this.adapterProperties = properties; partitioningKeys = targetDataset.getPrimaryKeys(); ARecordType recType = (ARecordType) itemType; isPKAutoGenerated = ((InternalDatasetDetails) targetDataset.getDatasetDetails()).isAutogenerated(); if (isPKAutoGenerated) { // Since the key is auto-generated, we need to use another // record type (possibly nested) which has all fields except the PK recType = getStrippedPKType(new LinkedList<>(partitioningKeys.get(0)), recType); } schemaTypes = new IAType[] { recType }; }
/** * Get the types of primary key (partitioning key) fields * * @param datasetDetails, * contains specific data structures for an internal dataset. * @param recordType, * the main record type. * @param metaRecordType * the auxiliary meta record type. * @return a list of IATypes, one for each corresponding primary key field. * @throws AlgebricksException */ public static List<IAType> getPartitioningKeyTypes(InternalDatasetDetails datasetDetails, ARecordType recordType, ARecordType metaRecordType) throws AlgebricksException { List<Integer> keySourceIndicators = datasetDetails.getKeySourceIndicator(); List<List<String>> partitioningKeys = datasetDetails.getPartitioningKey(); return getKeyTypes(recordType, metaRecordType, partitioningKeys, keySourceIndicators); }
aString.setValue(getFileStructure().toString()); stringSerde.serialize(aString, fieldValue.getDataOutput()); internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_FILESTRUCTURE_FIELD_INDEX, aString.setValue(getPartitioningStrategy().toString()); stringSerde.serialize(aString, fieldValue.getDataOutput()); internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_PARTITIONSTRATEGY_FIELD_INDEX, ABoolean b = isAutogenerated() ? ABoolean.TRUE : ABoolean.FALSE; booleanSerde.serialize(b, fieldValue.getDataOutput()); internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX, List<String> filterField = getFilterField(); if (filterField != null) { listBuilder.reset(heterogeneousList); List<Integer> keySourceIndicator = getKeySourceIndicator(); boolean needSerialization = false; if (keySourceIndicator != null) {
public static int getPositionOfPartitioningKeyField(Dataset dataset, List<String> fieldExpr, boolean fieldFromMeta) { List<Integer> keySourceIndicator = null; IDatasetDetails datasetDetails = dataset.getDatasetDetails(); if (datasetDetails.getDatasetType() == DatasetType.INTERNAL) { keySourceIndicator = ((InternalDatasetDetails) datasetDetails).getKeySourceIndicator(); } List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); for (int i = 0; i < partitioningKeys.size(); i++) { List<String> partitioningKey = partitioningKeys.get(i); if (partitioningKey.equals(fieldExpr) && keySourceMatches(keySourceIndicator, i, fieldFromMeta)) { return i; } } return -1; }
/** * Inserts a metadata dataset to the physical dataset index Should be performed * on a bootstrap of a new universe * * @param mdTxnCtx * @param indexes * @throws AlgebricksException */ public static void insertMetadataDatasets(MetadataTransactionContext mdTxnCtx, IMetadataIndex[] indexes) throws AlgebricksException { for (int i = 0; i < indexes.length; i++) { IDatasetDetails id = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH, indexes[i].getPartitioningExpr(), indexes[i].getPartitioningExpr(), null, indexes[i].getPartitioningExprType(), false, null); MetadataManager.INSTANCE.addDataset(mdTxnCtx, new Dataset(indexes[i].getDataverseName(), indexes[i].getIndexedDatasetName(), indexes[i].getDataverseName(), indexes[i].getPayloadRecordType().getTypeName(), indexes[i].getNodeGroupName(), GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME, GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, id, new HashMap<String, String>(), DatasetType.INTERNAL, indexes[i].getDatasetId().getId(), MetadataUtil.PENDING_NO_OP)); } if (LOGGER.isInfoEnabled()) { LOGGER.info("Finished inserting initial datasets."); } }
@Override public void addDataset(TxnId txnId, Dataset dataset) throws AlgebricksException, RemoteException { try { // Insert into the 'dataset' dataset. DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true); ITupleReference datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset); insertTupleIntoIndex(txnId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple); if (dataset.getDatasetType() == DatasetType.INTERNAL) { // Add the primary index for the dataset. InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails(); Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, false, true, dataset.getPendingOp()); addIndex(txnId, primaryIndex); } } catch (HyracksDataException e) { if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() == ErrorCode.DUPLICATE_KEY) { throw new AlgebricksException("A dataset with this name " + dataset.getDatasetName() + " already exists in dataverse '" + dataset.getDataverseName() + "'.", e); } else { throw new AlgebricksException(e); } } }
List<List<String>> primaryKey = datasetDetails.getPrimaryKey(); Map<String, Integer> fieldNameToIndexMap = new HashMap<String, Integer>(); String[] fieldNames = recordType.getFieldNames(); List<Integer> keySourceIndicators = datasetDetails.getKeySourceIndicator(); LogicalVariable recordVar = hasMeta ? indexSearchVars.get(indexSearchVars.size() - 2) : indexSearchVars.get(indexSearchVars.size() - 1);
boolean autogenerated = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).isAutogenerated(); if (!autogenerated) { return false; ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).getPrimaryKey().get(0); VariableReferenceExpression rec0 = new VariableReferenceExpression(inputRecord); rec0.setSourceLocation(inputRecordSourceLoc);
@Override public List<List<String>> getPrimaryKeys() { if (getDatasetType() == DatasetType.EXTERNAL) { return IndexingConstants.getRIDKeys(((ExternalDatasetDetails) getDatasetDetails()).getProperties()); } return ((InternalDatasetDetails) getDatasetDetails()).getPartitioningKey(); }
/** * Gets the comparator factories for the primary key fields of this dataset. * * @param metadataProvider, * the metadata provider. * @return the comparator factories for the primary key fields of this dataset. * @throws AlgebricksException */ public IBinaryComparatorFactory[] getPrimaryComparatorFactories(MetadataProvider metadataProvider, ARecordType recordType, ARecordType metaType) throws AlgebricksException { IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider(); IBinaryComparatorFactoryProvider cmpFactoryProvider = storageComponentProvider.getComparatorFactoryProvider(); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[numPrimaryKeys]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); cmpFactories[i] = cmpFactoryProvider.getBinaryComparatorFactory(keyType, true); } return cmpFactories; }
@Before public void createIndex() throws Exception { List<List<String>> partitioningKeys = new ArrayList<>(); partitioningKeys.add(Collections.singletonList("key")); dataset = new TestDataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null), null, DatasetType.INTERNAL, DATASET_ID, 0); PrimaryIndexInfo primaryIndexInfo = nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST, 0); IndexDataflowHelperFactory iHelperFactory = new IndexDataflowHelperFactory(nc.getStorageManager(), primaryIndexInfo.getFileSplitProvider()); JobId jobId = nc.newJobId(); ctx = nc.createTestContext(jobId, 0, false); indexDataflowHelper = iHelperFactory.create(ctx.getJobletContext().getServiceContext(), 0); indexDataflowHelper.open(); lsmBtree = (TestLsmBtree) indexDataflowHelper.getIndexInstance(); indexDataflowHelper.close(); txnCtx = nc.getTransactionManager().beginTransaction(nc.getTxnJobId(ctx), new TransactionOptions(ITransactionManager.AtomicityLevel.ENTITY_LEVEL)); insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager, null).getLeft(); }
List<Mutable<ILogicalExpression>> keyAccessExpression = new ArrayList<>(); keyAccessScalarFunctionCallExpression = new ArrayList<>(); pkTypes = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPrimaryKeyType(); partitioningKeys = ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey(); if (dataset.hasMetaPart()) { keySourceIndicator = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
/** * Gets the hash function factories for the primary key fields of this dataset. * * @param metadataProvider, * the metadata provider. * @return the hash function factories for the primary key fields of this dataset. * @throws AlgebricksException */ public IBinaryHashFunctionFactory[] getPrimaryHashFunctionFactories(MetadataProvider metadataProvider) throws AlgebricksException { ARecordType recordType = (ARecordType) metadataProvider.findType(this); ARecordType metaType = (ARecordType) metadataProvider.findMetaType(this); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); IBinaryHashFunctionFactory[] hashFuncFactories = new IBinaryHashFunctionFactory[numPrimaryKeys]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); hashFuncFactories[i] = BinaryHashFunctionFactoryProvider.INSTANCE.getBinaryHashFunctionFactory(keyType); } return hashFuncFactories; }
Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null, new InternalDatasetDetails(null, InternalDatasetDetails.PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null), null, DatasetConfig.DatasetType.INTERNAL, DATASET_ID, 0);
List<String> filteringFields = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField(); List<LogicalVariable> filteringVars; List<Mutable<ILogicalExpression>> filteringExpressions = null;
public ITypeTraits[] getPrimaryTypeTraits(MetadataProvider metadataProvider, ARecordType recordType, ARecordType metaType) throws AlgebricksException { IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider(); ITypeTraitProvider ttProvider = storageComponentProvider.getTypeTraitProvider(); List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); ITypeTraits[] typeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = datasetType == DatasetType.EXTERNAL ? IndexingConstants.getFieldType(i) : (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); typeTraits[i] = ttProvider.getTypeTrait(keyType); } typeTraits[numPrimaryKeys] = ttProvider.getTypeTrait(recordType); if (hasMetaPart()) { typeTraits[numPrimaryKeys + 1] = ttProvider.getTypeTrait(metaType); } return typeTraits; }
partitioningKeys.add(Collections.singletonList("key")); dataset = new TestDataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null), null, DatasetType.INTERNAL, DATASET_ID, 0);
private void setPrimaryRecDescAndComparators() throws AlgebricksException { List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys]; primaryBloomFilterKeyFields = new int[numPrimaryKeys]; ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); List<Integer> indicators = null; if (dataset.hasMetaPart()) { indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { IAType keyType = (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i)); primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType); primaryComparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true); primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType); primaryBloomFilterKeyFields[i] = i; } primaryRecFields[numPrimaryKeys] = payloadSerde; primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); if (dataset.hasMetaPart()) { primaryRecFields[numPrimaryKeys + 1] = payloadSerde; primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(metaType); } primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits); }