@Override public Map<String, Object> getParameters() { return accessor.getOpContext().getParameters(); }
public List<ILSMComponent> getMergingComponents() { return accessor.getOpContext().getComponentHolder(); }
public ILSMComponent getFlushingComponent() { return accessor.getOpContext().getComponentHolder().get(0); }
@Override public synchronized void scheduled(ILSMIOOperation operation) throws HyracksDataException { dsInfo.declareActiveIOOperation(); if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) { pendingFlushes++; FlushOperation flush = (FlushOperation) operation; Map<String, Object> map = operation.getAccessor().getOpContext().getParameters(); Long flushLsn = (Long) map.get(KEY_FLUSH_LOG_LSN); map.put(KEY_FIRST_LSN, firstLsnForCurrentMemoryComponent); map.put(KEY_FLUSHED_COMPONENT_ID, flush.getFlushingComponent().getId()); componentIds.add((ILSMComponentId) map.get(KEY_NEXT_COMPONENT_ID)); firstLsnForCurrentMemoryComponent = flushLsn; // Advance the first lsn for new component } }
@Override public final ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException { ILSMIndexAccessor accessor = operation.getAccessor(); ILSMIndexOperationContext opCtx = accessor.getOpContext(); return opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS ? EmptyComponent.INSTANCE : doMerge(operation); }
@Override public void merge(ILSMIOOperation operation) throws HyracksDataException { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Started a merge operation for index: {}", lsmIndex); } synchronized (opTracker) { enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE); } try { doIo(operation); } finally { exitComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE, operation.getNewComponent(), operation.getStatus() == LSMIOOperationStatus.FAILURE); opTracker.completeOperation(lsmIndex, LSMOperationType.MERGE, operation.getAccessor().getOpContext().getSearchOperationCallback(), operation.getAccessor().getOpContext().getModificationCallback()); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Finished the merge operation for index: {}. Result: {}", lsmIndex, operation.getStatus()); } }
@Override public synchronized void completed(ILSMIOOperation operation) { if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) { pendingFlushes--; if (operation.getStatus() == LSMIOOperationStatus.SUCCESS) { Map<String, Object> map = operation.getAccessor().getOpContext().getParameters(); persistenceLsn = pendingFlushes == 0 ? firstLsnForCurrentMemoryComponent : (Long) map.get(KEY_FLUSH_LOG_LSN); } } dsInfo.undeclareActiveIOOperation(); }
@SuppressWarnings("squid:S2142") @Override public void flush(ILSMIOOperation operation) throws HyracksDataException { LOGGER.debug("Started a flush operation for index: {}", lsmIndex); synchronized (opTracker) { while (!enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.FLUSH)) { try { opTracker.wait(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw HyracksDataException.create(e); } } } try { doIo(operation); } finally { exitComponents(operation.getAccessor().getOpContext(), LSMOperationType.FLUSH, operation.getNewComponent(), operation.getStatus() == LSMIOOperationStatus.FAILURE); opTracker.completeOperation(lsmIndex, LSMOperationType.FLUSH, operation.getAccessor().getOpContext().getSearchOperationCallback(), operation.getAccessor().getOpContext().getModificationCallback()); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Finished the flush operation for index: {}. Result: {}", lsmIndex, operation.getStatus()); } }
public TestFlushOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback, String indexIdentifier, LSMComponentFileReferences files, LSMComponentId componentId) throws HyracksDataException { super(accessor, target, callback, indexIdentifier); this.files = files; flushingComponent = accessor.getOpContext().getIndex().getCurrentMemoryComponent(); Mockito.when(flushingComponent.getId()).thenReturn(componentId); }
for (ILSMIndex lsmIndex : dsInfo.getDatasetPartitionOpenIndexes(partition)) { ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.getOpContext().setParameters(flushMap); ILSMIOOperation flush = accessor.scheduleFlush(); lastFlushTime = System.nanoTime();
@Override public final ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { ILSMIndexAccessor accessor = operation.getAccessor(); ILSMIndexOperationContext opCtx = accessor.getOpContext(); ILSMMemoryComponent memoryComponent = (ILSMMemoryComponent) opCtx.getComponentHolder().get(0); if (memoryComponent != getOldestReadableMemoryComponent()) { throw new IllegalStateException("An attempt to flush a memory component that is not the oldest"); } if (!memoryComponent.isModified() || opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS) { return EmptyComponent.INSTANCE; } if (LOGGER.isInfoEnabled()) { FlushOperation flushOp = (FlushOperation) operation; LOGGER.log(Level.INFO, "Flushing component with id: " + flushOp.getFlushingComponent().getId() + " in the index " + this); } return doFlush(operation); }
flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.getOpContext().setParameters(flushMap); long minId = logRecord.getFlushingComponentMinId(); long maxId = logRecord.getFlushingComponentMaxId();
@Override public void afterFinalize(ILSMIOOperation operation) throws HyracksDataException { if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { return; } if (operation.getIOOpertionType() != LSMIOOperationType.LOAD && operation.getAccessor().getOpContext().getOperation() == IndexOperation.DELETE_COMPONENTS) { deleteComponentsFromCheckpoint(operation); } else if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH || operation.getIOOpertionType() == LSMIOOperationType.LOAD) { addComponentToCheckpoint(operation); } }
private void deleteComponentsFromCheckpoint(ILSMIOOperation operation) throws HyracksDataException { // component was deleted... if a flush, do nothing.. if a merge, must update the checkpoint file if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) { // Get component id of the last disk component LSMComponentId mostRecentComponentId = getMostRecentComponentId(operation.getAccessor().getOpContext().getComponentsToBeMerged()); // Update the checkpoint file FileReference target = operation.getTarget(); final ResourceReference ref = ResourceReference.of(target.getAbsolutePath()); indexCheckpointManagerProvider.get(ref).setLastComponentId(mostRecentComponentId.getMaxId()); } else if (operation.getIOOpertionType() != LSMIOOperationType.FLUSH) { throw new IllegalStateException("Unexpected IO operation: " + operation.getIOOpertionType()); } }
(ILSMIndex) datasetLifecycleManager.getIndex(logRecord.getDatasetId(), logRecord.getResourceId()); ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); ILSMIndexOperationContext opCtx = indexAccessor.getOpContext(); opCtx.setFilterSkip(true); try {
@Override public ILSMIOOperation createFlushOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMMemoryComponent flushingComponent = getCurrentMemoryComponent(); if (flushingComponent.getWriterCount() > 0) { throw new IllegalStateException( "createFlushOperation is called on a component with writers: " + flushingComponent); } // take care of the flush cycling ILSMIOOperation flushOp = TracedIOOperation.wrap(createFlushOperation(createOpContext(NoOpIndexAccessParameters.INSTANCE), fileManager.getRelFlushFileReference(), ioOpCallback), tracer); // Changing the flush status should *always* precede changing the mutable component. flushingComponent.schedule(LSMIOOperationType.FLUSH); numScheduledFlushes++; changeFlushStatusForCurrentMutableCompoent(false); changeMutableComponent(); ILSMIndexAccessor accessor = flushOp.getAccessor(); ILSMIndexOperationContext flushCtx = accessor.getOpContext(); flushCtx.setOperation(ctx.getOperation()); // Could be component delete flushCtx.getComponentHolder().add(flushingComponent); flushCtx.setIoOperation(flushOp); propagateMap(ctx, flushCtx); ioOpCallback.scheduled(flushOp); return flushOp; }
@Override public void run() { ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE); try { dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION, indexPath) .refresh(); ILSMComponentId next = dsLifecycleMgr .getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION, indexPath).getId(); long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN(); Map<String, Object> flushMap = new HashMap<>(); flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn); flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next); lsmAccessor.getOpContext().setParameters(flushMap); lsmAccessor.deleteComponents(predicate); } catch (HyracksDataException e) { failure = e; } } };
@Override public void afterOperation(ILSMIOOperation operation) throws HyracksDataException { if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { return; } if (operation.getIOOpertionType() == LSMIOOperationType.LOAD) { Map<String, Object> map = operation.getParameters(); putComponentIdIntoMetadata(operation.getNewComponent(), (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID)); } else if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) { Map<String, Object> map = operation.getParameters(); putLSNIntoMetadata(operation.getNewComponent(), (Long) map.get(KEY_FLUSH_LOG_LSN)); putComponentIdIntoMetadata(operation.getNewComponent(), ((FlushOperation) operation).getFlushingComponent().getId()); } else if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) { List<ILSMDiskComponent> mergedComponents = operation.getAccessor().getOpContext().getComponentsToBeMerged(); putLSNIntoMetadata(operation.getNewComponent(), mergedComponents); putComponentIdIntoMetadata(operation.getNewComponent(), mergedComponents); LongPointable markerLsn = LongPointable.FACTORY.createPointable(ComponentUtils.getLong(mergedComponents.get(0).getMetadata(), ComponentUtils.MARKER_LSN_KEY, ComponentUtils.NOT_FOUND, buffer)); operation.getNewComponent().getMetadata().put(ComponentUtils.MARKER_LSN_KEY, markerLsn); } }
private static void redo(ILogRecord logRecord, IDatasetLifecycleManager datasetLifecycleManager) { try { int datasetId = logRecord.getDatasetId(); long resourceId = logRecord.getResourceId(); ILSMIndex index = (ILSMIndex) datasetLifecycleManager.getIndex(datasetId, resourceId); ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); ILSMIndexOperationContext opCtx = indexAccessor.getOpContext(); opCtx.setFilterSkip(true); opCtx.setRecovery(true); if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.INSERT_BYTE) { indexAccessor.forceInsert(logRecord.getNewValue()); } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.DELETE_BYTE) { indexAccessor.forceDelete(logRecord.getNewValue()); } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.UPSERT_BYTE) { // redo, upsert the new value indexAccessor.forceUpsert(logRecord.getNewValue()); } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.FILTER_BYTE) { opCtx.setFilterSkip(false); indexAccessor.updateFilter(logRecord.getNewValue()); } else { throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp()); } } catch (Exception e) { throw new IllegalStateException("Failed to redo", e); } }
flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, expectedId); ILSMIndexAccessor accessor = new TestLSMIndexAccessor(new TestLSMIndexOperationContext(mockIndex)); accessor.getOpContext().setParameters(flushMap); FileReference target = new FileReference(Mockito.mock(IODeviceHandle.class), getComponentFileName()); LSMComponentFileReferences files = new LSMComponentFileReferences(target, target, target);