private void loadDiskComponents() throws HyracksDataException { diskComponents.clear(); List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles(); for (LSMComponentFileReferences lsmComponentFileReferences : validFileReferences) { ILSMDiskComponent component = createDiskComponent(componentFactory, lsmComponentFileReferences.getInsertIndexFileReference(), lsmComponentFileReferences.getDeleteIndexFileReference(), lsmComponentFileReferences.getBloomFilterFileReference(), false); diskComponents.add(component); } }
@Override protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) { boolean returnDeletedTuples = false; ILSMIndexAccessor accessor = createAccessor(opCtx); List<ILSMComponent> mergingComponents = opCtx.getComponentHolder(); if (mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1)) { returnDeletedTuples = true; } LSMBTreeRangeSearchCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples); return new LSMBTreeMergeOperation(accessor, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier()); } }
@Override public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException { LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); }
@Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) { ILSMIndexAccessor accessor = createAccessor(opCtx); return new LSMBTreeFlushOperation(accessor, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier()); }
@Override public LSMBTreeWithBloomFilterDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) throws HyracksDataException { return new LSMBTreeWithBloomFilterDiskComponent(lsmIndex, btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), filterHelper == null ? null : filterHelper.createFilter()); }
@Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileReferences = fileManager.getTransactionFileReferenceForCommit(); ILSMDiskComponent component = null; if (componentFileReferences != null) { component = createDiskComponent(componentFactory, componentFileReferences.getInsertIndexFileReference(), null, componentFileReferences.getBloomFilterFileReference(), false); } getHarness().addTransactionComponents(component); }
@Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException { LSMRTreeAccessor accessor = new LSMRTreeAccessor(getHarness(), opCtx, buddyBTreeFields); return new LSMRTreeFlushOperation(accessor, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier()); }
@Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit(); ILSMDiskComponent component = null; if (componentFileRefrences != null) { component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false); } ((ExternalIndexHarness) getHarness()).addTransactionComponents(component); }
@Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit(); ILSMDiskComponent component = null; if (componentFileRefrences != null) { component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false); } getHarness().addTransactionComponents(component); }
@Override public LSMInvertedIndexDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) throws HyracksDataException { return new LSMInvertedIndexDiskComponent(lsmIndex, diskInvIndexFactory.createIndexInstance(cfr.getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), filterHelper == null ? null : filterHelper.createFilter()); } }
@Override protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException { ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(getHarness(), opCtx); IIndexCursor cursor = new LSMInvertedIndexMergeCursor(opCtx); return new LSMInvertedIndexMergeOperation(accessor, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier()); } }
@Override public LSMBTreeWithBuddyDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) throws HyracksDataException { return new LSMBTreeWithBuddyDiskComponent(lsmIndex, btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), buddyBtreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), filterHelper == null ? null : filterHelper.createFilter()); } }
@Override public LSMRTreeDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr) throws HyracksDataException { return new LSMRTreeDiskComponent(lsmIndex, rtreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()), btreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()), bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), filterHelper == null ? null : filterHelper.createFilter()); } }
@Override protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException { LSMRTreeSortedCursor cursor = new LSMRTreeSortedCursor(opCtx, linearizer, buddyBTreeFields); ILSMIndexAccessor accessor = new LSMRTreeAccessor(getHarness(), opCtx, buddyBTreeFields); return new LSMRTreeMergeOperation(accessor, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier()); } }
@Override protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx, LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException { return new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(getHarness(), opCtx), componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), callback, getIndexIdentifier()); }
@Override public ILSMIOOperation createMergeOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, -1); rctx.setOperation(IndexOperation.MERGE); List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); LSMRTreeSortedCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1), (ILSMDiskComponent) mergingComponents.get(0)); ILSMIndexAccessor accessor = new LSMRTreeAccessor(getHarness(), rctx, buddyBTreeFields); // create the merge operation. LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), ioOpCallback, fileManager.getBaseDir().getAbsolutePath()); ioOpCallback.scheduled(mergeOp); return mergeOp; }
public LSMTwoPCBTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, boolean isTransaction, Map<String, Object> parameters) throws HyracksDataException { this.isTransaction = isTransaction; // Create the appropriate target LSMComponentFileReferences componentFileRefs; if (isTransaction) { try { componentFileRefs = fileManager.getNewTransactionFileReference(); } catch (IOException e) { throw HyracksDataException.create(e); } component = createDiskComponent(transactionComponentFactory, componentFileRefs.getInsertIndexFileReference(), null, componentFileRefs.getBloomFilterFileReference(), true); } else { componentFileRefs = fileManager.getRelFlushFileReference(); component = createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); } loadOp = new LoadOperation(componentFileRefs, ioOpCallback, getIndexIdentifier(), parameters); loadOp.setNewComponent(component); ioOpCallback.scheduled(loadOp); ioOpCallback.beforeOperation(loadOp); componentBulkLoader = component.createBulkLoader(loadOp, fillFactor, verifyInput, numElementsHint, false, true, true); }
public LSMTwoPCRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, boolean isTransaction, Map<String, Object> parameters) throws HyracksDataException { this.isTransaction = isTransaction; // Create the appropriate target LSMComponentFileReferences componentFileRefs; if (isTransaction) { try { componentFileRefs = fileManager.getNewTransactionFileReference(); } catch (IOException e) { throw HyracksDataException.create(e); } component = createDiskComponent(componentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); } else { componentFileRefs = fileManager.getRelFlushFileReference(); component = createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); } loadOp = new LoadOperation(componentFileRefs, ioOpCallback, getIndexIdentifier(), parameters); loadOp.setNewComponent(component); ioOpCallback.scheduled(loadOp); ioOpCallback.beforeOperation(loadOp); componentBulkLoader = component.createBulkLoader(loadOp, fillFactor, verifyInput, numElementsHint, false, true, false); }
public LSMTwoPCBTreeWithBuddyBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint, boolean isTransaction, Map<String, Object> parameters) throws HyracksDataException { this.isTransaction = isTransaction; // Create the appropriate target LSMComponentFileReferences componentFileRefs; if (isTransaction) { try { componentFileRefs = fileManager.getNewTransactionFileReference(); } catch (IOException e) { throw HyracksDataException.create(e); } component = createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); } else { componentFileRefs = fileManager.getRelFlushFileReference(); component = createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true); } loadOp = new LoadOperation(componentFileRefs, ioOpCallback, getIndexIdentifier(), parameters); loadOp.setNewComponent(component); ioOpCallback.scheduled(loadOp); ioOpCallback.beforeOperation(loadOp); componentBulkLoader = component.createBulkLoader(loadOp, fillFactor, verifyInput, numElementsHint, false, true, false); }
public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint, Map<String, Object> parameters) throws HyracksDataException { AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE); opCtx.setParameters(parameters); LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference(); LoadOperation loadOp = new LoadOperation(componentFileRefs, ioOpCallback, getIndexIdentifier(), parameters); loadOp.setNewComponent(createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true)); ioOpCallback.scheduled(loadOp); opCtx.setIoOperation(loadOp); return new LSMIndexDiskComponentBulkLoader(this, opCtx, fillLevel, verifyInput, numElementsHint); }