public static JobSpecification buildIndexUpdateOp(Dataset ds, Index index, List<ExternalFile> metadataFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { // Create files list ArrayList<ExternalFile> files = new ArrayList<>(); for (ExternalFile metadataFile : metadataFiles) { if (metadataFile.getPendingOp() != ExternalFilePendingOp.APPEND_OP) { files.add(metadataFile); } else { metadataFile.setPendingOp(ExternalFilePendingOp.NO_OP); files.add(metadataFile); } } // add new files for (ExternalFile file : addedFiles) { files.add(file); } // add appended files for (ExternalFile file : appendedFiles) { files.add(file); } return IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider, files, sourceLoc); }
private static void dropDatasetFiles(Dataset dataset, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { List<JobSpecification> jobs = new ArrayList<>(); List<Index> indexes = metadataProvider.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName()); for (Index index : indexes) { jobs.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, dataset, EnumSet.of(DropOption.IF_EXISTS, DropOption.WAIT_ON_IN_USE), null)); } for (JobSpecification jobSpec : jobs) { JobUtils.runJob(hcc, jobSpec, true); } }
protected void prepareCompactJobsForExternalDataset(List<Index> indexes, Dataset ds, List<JobSpecification> jobsToExecute, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { for (int j = 0; j < indexes.size(); j++) { jobsToExecute .add(IndexUtil.buildSecondaryIndexCompactJobSpec(ds, indexes.get(j), metadataProvider, sourceLoc)); } }
private static void createAndLoadSecondaryIndexesForTarget(Dataset source, Dataset target, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { for (Index index : metadataProvider.getDatasetIndexes(source.getDataverseName(), source.getDatasetName())) { if (!index.isSecondaryIndex()) { continue; } // Creates the secondary index. JobSpecification indexCreationJobSpec = IndexUtil.buildSecondaryIndexCreationJobSpec(target, index, metadataProvider, null); JobUtils.runJob(hcc, indexCreationJobSpec, true); // Loads the secondary index. JobSpecification indexLoadingJobSpec = IndexUtil.buildSecondaryIndexLoadingJobSpec(target, index, metadataProvider, null); JobUtils.runJob(hcc, indexLoadingJobSpec, true); } }
spec = IndexUtil.buildSecondaryIndexCreationJobSpec(ds, index, metadataProvider, sourceLoc); if (spec == null) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, spec = IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider, sourceLoc); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; metadataProvider.setMetadataTxnContext(mdTxnCtx); try { JobSpecification jobSpec = IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds, sourceLoc); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false;
Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException { int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits); int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index); IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider(); ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType);
IndexUtil.bindJobEventListener(spec, metadataProvider);
public Dataset addDatasetIfNotExists(Dataset dataset) { synchronized (datasets) { synchronized (indexes) { // Add the primary index associated with the dataset, if the dataset is an // internal dataset. if (dataset.getDatasetType() == DatasetType.INTERNAL) { Index index = IndexUtil.getPrimaryIndex(dataset); addIndexIfNotExistsInternal(index); } Map<String, Dataset> m = datasets.get(dataset.getDataverseName()); if (m == null) { m = new HashMap<>(); datasets.put(dataset.getDataverseName(), m); } if (!m.containsKey(dataset.getDatasetName())) { return m.put(dataset.getDatasetName(), dataset); } return null; } } }
public static int[] getFilterFields(Dataset dataset, Index index, ITypeTraits[] filterTypeTraits) throws AlgebricksException { if (index.isPrimaryIndex()) { return DatasetUtil.createFilterFields(dataset); } return secondaryFilterFields(dataset, index, filterTypeTraits); }
@Override public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException { int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits); int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index); IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider(); ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType); IBinaryComparatorFactory[] cmpFactories = getCmpFactories(mdProvider, dataset, index, recordType, metaType); int[] bloomFilterFields = getBloomFilterFields(dataset, index); double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(); ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index); if (opTrackerFactory instanceof PrimaryIndexOperationTrackerFactory) { opTrackerFactory = new TestPrimaryIndexOperationTrackerFactory(dataset.getDatasetId()); } ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index); IStorageManager storageManager = storageComponentProvider.getStorageManager(); IMetadataPageManagerFactory metadataPageManagerFactory = storageComponentProvider.getMetadataPageManagerFactory(); ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider(); AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()); return new TestLsmBtreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, true, bloomFilterFields, bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields); }
@Override public JobSpecification buildLoadingJobSpec() throws AlgebricksException { JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext()); IndexUtil.bindJobEventListener(spec, metadataProvider);
public static JobSpecification createDatasetJobSpec(Dataset dataset, MetadataProvider metadataProvider) throws AlgebricksException { Index index = IndexUtil.getPrimaryIndex(dataset); ARecordType itemType = (ARecordType) metadataProvider.findType(dataset);
public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider, List<ExternalFile> files, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper; if (dataset.isCorrelated()) { secondaryIndexHelper = SecondaryCorrelatedTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); } else { secondaryIndexHelper = SecondaryTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); } if (files != null) { secondaryIndexHelper.setExternalFiles(files); } return secondaryIndexHelper.buildLoadingJobSpec(); }
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName); for (Index index : indexes) { jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, dataset, sourceLoc)); ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider, dataset)); } else { jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k), metadataProvider, dataset, sourceLoc));
IndexUtil.bindJobEventListener(spec, metadataProvider);
if (index.isSecondaryIndex()) { jobsToExecute.add( IndexUtil.buildSecondaryIndexCompactJobSpec(ds, index, metadataProvider, sourceLoc));
primaryIndexName); Assert.assertNotNull(index); jobSpecification = IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null); jobSpecification.getOperatorMap().values().forEach(iOperatorDescriptor -> { Assert.assertFalse(iOperatorDescriptor instanceof AbstractSorterOperatorDescriptor); secondaryIndexName); Assert.assertNotNull(index); jobSpecification = IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null); final long numOfSortOperators = jobSpecification.getOperatorMap().values().stream() .filter(op -> op instanceof AbstractSorterOperatorDescriptor).count();
if (indexes.get(j).isSecondaryIndex()) { jobsToExecute .add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this, sourceLoc)); if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) { jobsToExecute .add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this, sourceLoc)); } else { jobsToExecute.add(DatasetUtil.buildDropFilesIndexJobSpec(metadataProvider, this));
@Override public JobSpecification buildLoadingJobSpec() throws AlgebricksException { JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext()); IndexUtil.bindJobEventListener(spec, metadataProvider);
validateDatasetState(metadataProvider, ds, sourceLoc); jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds, sourceLoc)); jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds, sourceLoc)); List<Index> datasetIndexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);