@Override public void clear() throws HyracksDataException { if (!isActive) { throw new HyracksDataException("Failed to clear the index since it is not activated."); } ((ExternalIndexHarness) getHarness()).indexClear(); for (ILSMDiskComponent c : diskComponents) { c.deactivateAndDestroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } for (ILSMDiskComponent c : secondDiskComponents) { c.deactivateAndDestroy(); } diskComponents.clear(); secondDiskComponents.clear(); version = 0; }
public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMDiskComponentFactory componentFactory, ILSMDiskComponentFactory bulkLoadComponentFactory, boolean durable, ITracer tracer) throws HyracksDataException { this.ioManager = ioManager; this.diskBufferCache = diskBufferCache; this.fileManager = fileManager; this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate; this.ioOpCallback = ioOpCallbackFactory.createIoOpCallback(this); this.componentFactory = componentFactory; this.bulkLoadComponentFactory = bulkLoadComponentFactory; this.durable = durable; this.tracer = tracer; lsmHarness = new ExternalIndexHarness(this, ioScheduler, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled()); isActive = false; diskComponents = new LinkedList<>(); this.inactiveDiskComponents = new LinkedList<>(); // Memory related objects are nulled virtualBufferCaches = null; memoryComponents = null; currentMutableComponentId = null; flushRequests = null; filterHelper = null; filterFrameFactory = null; filterManager = null; treeFields = null; filterFields = null; }
public void beforeSubsumeMergedComponents(ILSMComponent newComponent, List<ILSMComponent> mergedComponents) throws HyracksDataException { ITwoPCIndex index = (ITwoPCIndex) lsmIndex; // check if merge will affect the first list if (index.getFirstComponentList().containsAll(mergedComponents)) { // exit un-needed components for (ILSMComponent c : mergedComponents) { exitComponent((ILSMDiskComponent) c); } // enter new component enterComponent(newComponent); } // check if merge will affect the second list if (index.getSecondComponentList().containsAll(mergedComponents)) { // exit un-needed components for (ILSMComponent c : mergedComponents) { exitComponent((ILSMDiskComponent) c); } // enter new component enterComponent(newComponent); } }
@Override public void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException { LSMOperationType opType = LSMOperationType.SEARCH; getAndEnterComponents(ctx, opType, false); try { lsmIndex.search(ctx, cursor, pred); } catch (Exception e) { exitComponents(ctx, opType, null, true); throw e; } }
@Override protected boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean isTryOperation) throws HyracksDataException { validateOperationEnterComponentsState(ctx); synchronized (opTracker) { while (true) { lsmIndex.getOperationalComponents(ctx); // Before entering the components, prune those corner cases that indeed should not proceed. switch (opType) { case MERGE: if (ctx.getComponentHolder().size() < 2) { // There is only a single component. There is nothing to merge. return false; } default: break; } if (enterComponents(ctx, opType)) { return true; } else if (isTryOperation) { return false; } } } }
@Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileReferences = fileManager.getTransactionFileReferenceForCommit(); ILSMDiskComponent component = null; if (componentFileReferences != null) { component = createDiskComponent(componentFactory, componentFileReferences.getInsertIndexFileReference(), null, componentFileReferences.getBloomFilterFileReference(), false); } getHarness().addTransactionComponents(component); }
secondDiskComponents.add(component); getHarness().indexFirstTimeActivated(); } else {
@Override public void end() throws HyracksDataException { try { ioOpCallback.afterOperation(loadOp); componentBulkLoader.end(); if (component.getComponentSize() > 0) { if (isTransaction) { // Since this is a transaction component, validate and // deactivate. it could later be added or deleted try { component.markAsValid(durable, loadOp); } finally { ioOpCallback.afterFinalize(loadOp); } component.deactivate(); } else { ioOpCallback.afterFinalize(loadOp); getHarness().addBulkLoadedComponent(loadOp); } } } finally { ioOpCallback.completed(loadOp); } }
@SuppressWarnings("squid:S1181") @Override public void addBulkLoadedComponent(ILSMIOOperation ioOperation) throws HyracksDataException { ILSMDiskComponent c = ioOperation.getNewComponent(); try { c.markAsValid(lsmIndex.isDurable(), ioOperation); } catch (Throwable th) { ioOperation.setFailure(th); } if (ioOperation.hasFailed()) { throw HyracksDataException.create(ioOperation.getFailure()); } synchronized (opTracker) { lsmIndex.addDiskComponent(c); if (replicationEnabled) { componentsToBeReplicated.clear(); componentsToBeReplicated.add(c); triggerReplication(componentsToBeReplicated, LSMOperationType.LOAD); } // Enter the component enterComponent(c); mergePolicy.diskComponentAdded(lsmIndex, false); } }
beforeSubsumeMergedComponents(newComponent, ctx.getComponentHolder()); lsmIndex.subsumeMergedComponents(newComponent, ctx.getComponentHolder()); if (replicationEnabled) { componentsToBeReplicated.clear(); componentsToBeReplicated.add(newComponent); triggerReplication(componentsToBeReplicated, opType);
public void indexFirstTimeActivated() throws HyracksDataException { ITwoPCIndex index = (ITwoPCIndex) lsmIndex; // Enter disk components <-- To avoid deleting them when they are // still needed--> for (ILSMComponent c : index.getFirstComponentList()) { enterComponent(c); } for (ILSMComponent c : index.getSecondComponentList()) { enterComponent(c); } }
public void indexClear() throws HyracksDataException { ITwoPCIndex index = (ITwoPCIndex) lsmIndex; for (ILSMDiskComponent c : index.getFirstComponentList()) { exitComponent(c); } for (ILSMDiskComponent c : index.getSecondComponentList()) { exitComponent(c); } }
@Override public void endSearch(ILSMIndexOperationContext ctx) throws HyracksDataException { if (ctx.getOperation() == IndexOperation.SEARCH) { try { exitComponents(ctx, LSMOperationType.SEARCH, null, false); } catch (Exception e) { throw HyracksDataException.create(e); } } }
@Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit(); ILSMDiskComponent component = null; if (componentFileRefrences != null) { component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false); } getHarness().addTransactionComponents(component); }
secondDiskComponents.add(component); ((ExternalIndexHarness) getHarness()).indexFirstTimeActivated(); } else {
@Override public void end() throws HyracksDataException { try { ioOpCallback.afterOperation(loadOp); componentBulkLoader.end(); if (component.getComponentSize() > 0) { if (isTransaction) { // Since this is a transaction component, validate and // deactivate. it could later be added or deleted try { component.markAsValid(durable, loadOp); } finally { ioOpCallback.afterFinalize(loadOp); } component.deactivate(); } else { ioOpCallback.afterFinalize(loadOp); getHarness().addBulkLoadedComponent(loadOp); } } } finally { ioOpCallback.completed(loadOp); } }
@Override public void commitTransaction() throws HyracksDataException { LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit(); ILSMDiskComponent component = null; if (componentFileRefrences != null) { component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(), componentFileRefrences.getDeleteIndexFileReference(), componentFileRefrences.getBloomFilterFileReference(), false); } ((ExternalIndexHarness) getHarness()).addTransactionComponents(component); }
@Override public void clear() throws HyracksDataException { if (!isActive) { throw new HyracksDataException("Failed to clear the index since it is not activated."); } getHarness().indexClear(); for (ILSMDiskComponent c : diskComponents) { c.deactivateAndDestroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } for (ILSMDiskComponent c : secondDiskComponents) { c.deactivateAndDestroy(); } diskComponents.clear(); secondDiskComponents.clear(); version = 0; }
secondDiskComponents.add(component); getHarness().indexFirstTimeActivated(); } else {
public void addTransactionComponents(ILSMDiskComponent newComponent) throws HyracksDataException { ITwoPCIndex index = (ITwoPCIndex) lsmIndex; synchronized (opTracker) { List<ILSMDiskComponent> newerList; List<ILSMDiskComponent> olderList; if (index.getCurrentVersion() == 0) { newerList = index.getFirstComponentList(); olderList = index.getSecondComponentList(); } else { newerList = index.getSecondComponentList(); olderList = index.getFirstComponentList(); } // Exit components in old version of the index so they are ready to be // deleted if they are not needed anymore for (ILSMDiskComponent c : olderList) { exitComponent(c); } // Enter components in the newer list for (ILSMDiskComponent c : newerList) { enterComponent(c); } if (newComponent != null) { // Enter new component enterComponent(newComponent); } index.commitTransactionDiskComponent(newComponent); mergePolicy.diskComponentAdded(lsmIndex, fullMergeIsRequested.get()); } }