public static void checkAndSetFirstLSN(AbstractLSMIndex lsmIndex, ILogManager logManager) throws HyracksDataException { // If the index has an empty memory component, we need to set its first LSN (For soft checkpoint) if (lsmIndex.isCurrentMutableComponentEmpty()) { //prevent transactions from incorrectly setting the first LSN on a modified component by checking the index is still empty synchronized (lsmIndex.getOperationTracker()) { if (lsmIndex.isCurrentMutableComponentEmpty()) { LSMIOOperationCallback ioOpCallback = (LSMIOOperationCallback) lsmIndex.getIOOperationCallback(); ioOpCallback.setFirstLsnForCurrentMemoryComponent(logManager.getAppendLSN()); } } } }
private void addOperationalMemoryComponents(List<ILSMComponent> operationalComponents, boolean modification) { // add current memory component first if needed if (numScheduledFlushes < memoryComponents.size()) { ILSMMemoryComponent c = memoryComponents.get(currentMutableComponentId.get()); // The current mutable component is added if modification or readable // This ensures that activation of new component only happens in case of modifications // and allow for controlling that without stopping search operations if (modification || c.isReadable()) { operationalComponents.add(c); } } if (modification && numScheduledFlushes >= memoryComponents.size()) { // will fail the enterComponent call and retry operationalComponents.add(memoryComponents.get(0)); return; } addImmutableMemoryComponents(operationalComponents); }
@Override public boolean isCurrentMutableComponentEmpty() throws HyracksDataException { synchronized (getOperationTracker()) { ILSMMemoryComponent cmc = getCurrentMemoryComponent(); ComponentState state = cmc.getState(); return state == ComponentState.READABLE_UNWRITABLE_FLUSHING || state == ComponentState.INACTIVE || state == ComponentState.UNREADABLE_UNWRITABLE || !cmc.isModified(); } }
private ILSMMemoryComponent getOldestReadableMemoryComponent() { synchronized (getOperationTracker()) { int cmc = currentMutableComponentId.get(); int numImmutableMemoryComponents = Integer.min(numScheduledFlushes, memoryComponents.size()); int next = numScheduledFlushes < memoryComponents.size() ? cmc : getNextToBeFlushed(); for (int i = 0; i < numImmutableMemoryComponents; i++) { next--; if (next < 0) { next = memoryComponents.size() - 1; } } // start going forward for (int i = 0; i < numImmutableMemoryComponents; i++) { if (memoryComponents.get(next).isReadable()) { return memoryComponents.get(next); } next++; if (next == memoryComponents.size()) { next = 0; } } throw new IllegalStateException("Couldn't find any readable component"); } }
@Override public ILSMIOOperation createFlushOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMMemoryComponent flushingComponent = getCurrentMemoryComponent(); if (flushingComponent.getWriterCount() > 0) { throw new IllegalStateException( "createFlushOperation is called on a component with writers: " + flushingComponent); } // take care of the flush cycling ILSMIOOperation flushOp = TracedIOOperation.wrap(createFlushOperation(createOpContext(NoOpIndexAccessParameters.INSTANCE), fileManager.getRelFlushFileReference(), ioOpCallback), tracer); // Changing the flush status should *always* precede changing the mutable component. flushingComponent.schedule(LSMIOOperationType.FLUSH); numScheduledFlushes++; changeFlushStatusForCurrentMutableCompoent(false); changeMutableComponent(); ILSMIndexAccessor accessor = flushOp.getAccessor(); ILSMIndexOperationContext flushCtx = accessor.getOpContext(); flushCtx.setOperation(ctx.getOperation()); // Could be component delete flushCtx.getComponentHolder().add(flushingComponent); flushCtx.setIoOperation(flushOp); propagateMap(ctx, flushCtx); ioOpCallback.scheduled(flushOp); return flushOp; }
AbstractLSMIndex index = (AbstractLSMIndex) appCtx.getDatasetLifecycleManager() .getIndex(idx.getDatasetId().getId(), idx.getResourceId()); PrimaryIndexOperationTracker opTracker = (PrimaryIndexOperationTracker) index.getOperationTracker(); final MetadataTransactionContext mdTxn2 = MetadataManager.INSTANCE.beginTransaction(); int mutableComponentBeforeFlush = index.getCurrentMemoryComponentIndex(); int diskComponentsBeforeFlush = index.getDiskComponents().size(); Assert.assertEquals(mutableComponentBeforeFlush, index.getCurrentMemoryComponentIndex()); Assert.assertEquals(diskComponentsBeforeFlush, index.getDiskComponents().size()); Assert.assertNotEquals(mutableComponentBeforeFlush, index.getCurrentMemoryComponentIndex()); Assert.assertEquals(diskComponentsBeforeFlush + 1, index.getDiskComponents().size());
if (flush && memoryComponentsAllocated) { try { createAccessor(NoOpIndexAccessParameters.INSTANCE).scheduleFlush().sync(); } catch (InterruptedException e) { throw HyracksDataException.create(e); deactivateDiskComponents(); LOGGER.log(Level.INFO, "Deallocating memory components of: {}", this); deallocateMemoryComponents(); isActive = false; LOGGER.log(Level.INFO, "Deactivating the index: {}. COMPLETED", this);
private void activeate() throws HyracksDataException { if (state == ComponentState.INACTIVE) { state = ComponentState.READABLE_WRITABLE; lsmIndex.getIOOperationCallback().recycled(this); } }
private void presistComponentToDisk() throws HyracksDataException { try { lsmIndex.getIOOperationCallback().afterOperation(opCtx.getIoOperation()); componentBulkLoader.end(); } catch (Throwable th) { // NOSONAR Must not call afterFinalize without setting failure fail(th); throw th; } finally { lsmIndex.getIOOperationCallback().afterFinalize(opCtx.getIoOperation()); } if (opCtx.getIoOperation().getStatus() == LSMIOOperationStatus.SUCCESS && opCtx.getIoOperation().getNewComponent().getComponentSize() > 0) { lsmIndex.getHarness().addBulkLoadedComponent(opCtx.getIoOperation()); } }
@Override public long getLocalMinFirstLSN() throws HyracksDataException { final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager(); List<IIndex> openIndexList = datasetLifecycleManager.getOpenResources(); long firstLSN; //the min first lsn can only be the current append or smaller long minFirstLSN = logMgr.getAppendLSN(); if (!openIndexList.isEmpty()) { for (IIndex index : openIndexList) { LSMIOOperationCallback ioCallback = (LSMIOOperationCallback) ((ILSMIndex) index).getIOOperationCallback(); if (!((AbstractLSMIndex) index).isCurrentMutableComponentEmpty() || ioCallback.hasPendingFlush()) { firstLSN = ioCallback.getPersistenceLsn(); minFirstLSN = Math.min(minFirstLSN, firstLSN); } } } return minFirstLSN; }
modOpCallbackFactory.createModificationOperationCallback(indexHelper.getResource(), ctx, this); IIndexAccessParameters iap = new IndexAccessParameters(modCallback, NoOpOperationCallback.INSTANCE); indexAccessor = lsmIndex.createAccessor(iap); if (tupleFilterFactory != null) { tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
break; case INSERT: addOperationalMemoryComponents(operationalComponents, true); operationalComponents.addAll(diskComponents); break; case SEARCH: if (memoryComponentsAllocated) { addOperationalMemoryComponents(operationalComponents, false);
public IChainedComponentBulkLoader createBloomFilterBulkLoader(long numElementsHint) throws HyracksDataException { BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec( BloomCalculations.maxBucketsPerElement(numElementsHint), getLsmIndex().bloomFilterFalsePositiveRate()); return new BloomFilterBulkLoader(getBloomFilter().createBuilder(numElementsHint, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements())); }
@Override public void abort() throws HyracksDataException { opCtx.getIoOperation().setStatus(LSMIOOperationStatus.FAILURE); try { try { componentBulkLoader.abort(); } finally { lsmIndex.getIOOperationCallback().afterFinalize(opCtx.getIoOperation()); } } finally { lsmIndex.getIOOperationCallback().completed(opCtx.getIoOperation()); } }
@Override public void end() throws HyracksDataException { try { presistComponentToDisk(); } catch (Throwable th) { // NOSONAR must cleanup in case of any failure fail(th); throw th; } finally { lsmIndex.getIOOperationCallback().completed(opCtx.getIoOperation()); } }