public CommitLogSegmentManagerCDC(final CommitLog commitLog, String storageDirectory) { super(commitLog, storageDirectory); cdcSizeTracker = new CDCSizeTracker(this, new File(DatabaseDescriptor.getCDCLogLocation())); }
public void discard(CommitLogSegment segment, boolean delete) { segment.close(); addSize(-segment.onDiskSize()); cdcSizeTracker.processDiscardedSegment(segment); if (segment.getCDCState() == CDCState.CONTAINS) FileUtils.renameWithConfirm(segment.logFile.getAbsolutePath(), DatabaseDescriptor.getCDCLogLocation() + File.separator + segment.logFile.getName()); else { if (delete) FileUtils.deleteWithConfirm(segment.logFile); } }
public void submitOverflowSizeRecalculation() { try { cdcSizeCalculationExecutor.submit(() -> recalculateOverflowSize()); } catch (RejectedExecutionException e) { // Do nothing. Means we have one in flight so this req. should be satisfied when it completes. } }
/** * Synchronous size recalculation on each segment creation/deletion call could lead to very long delays in new * segment allocation, thus long delays in thread signaling to wake waiting allocation / writer threads. * * This can be reached either from the segment management thread in ABstractCommitLogSegmentManager or from the * size recalculation executor, so we synchronize on this object to reduce the race overlap window available for * size to get off. * * Reference DirectorySizerBench for more information about performance of the directory size recalc. */ void processNewSegment(CommitLogSegment segment) { // See synchronization in CommitLogSegment.setCDCState synchronized(segment.cdcStateLock) { segment.setCDCState(defaultSegmentSize() + totalCDCSizeOnDisk() > allowableCDCBytes() ? CDCState.FORBIDDEN : CDCState.PERMITTED); if (segment.getCDCState() == CDCState.PERMITTED) unflushedCDCSize += defaultSegmentSize(); } // Take this opportunity to kick off a recalc to pick up any consumer file deletion. submitOverflowSizeRecalculation(); }
/** * Synchronous size recalculation on each segment creation/deletion call could lead to very long delays in new * segment allocation, thus long delays in thread signaling to wake waiting allocation / writer threads. * * This can be reached either from the segment management thread in ABstractCommitLogSegmentManager or from the * size recalculation executor, so we synchronize on this object to reduce the race overlap window available for * size to get off. * * Reference DirectorySizerBench for more information about performance of the directory size recalc. */ void processNewSegment(CommitLogSegment segment) { // See synchronization in CommitLogSegment.setCDCState synchronized(segment.cdcStateLock) { segment.setCDCState(defaultSegmentSize() + totalCDCSizeOnDisk() > allowableCDCBytes() ? CDCState.FORBIDDEN : CDCState.PERMITTED); if (segment.getCDCState() == CDCState.PERMITTED) unflushedCDCSize += defaultSegmentSize(); } // Take this opportunity to kick off a recalc to pick up any consumer file deletion. submitOverflowSizeRecalculation(); }
/** * Synchronous size recalculation on each segment creation/deletion call could lead to very long delays in new * segment allocation, thus long delays in thread signaling to wake waiting allocation / writer threads. * * This can be reached either from the segment management thread in ABstractCommitLogSegmentManager or from the * size recalculation executor, so we synchronize on this object to reduce the race overlap window available for * size to get off. * * Reference DirectorySizerBench for more information about performance of the directory size recalc. */ void processNewSegment(CommitLogSegment segment) { // See synchronization in CommitLogSegment.setCDCState synchronized(segment.cdcStateLock) { segment.setCDCState(defaultSegmentSize() + totalCDCSizeOnDisk() > allowableCDCBytes() ? CDCState.FORBIDDEN : CDCState.PERMITTED); if (segment.getCDCState() == CDCState.PERMITTED) unflushedCDCSize += defaultSegmentSize(); } // Take this opportunity to kick off a recalc to pick up any consumer file deletion. submitOverflowSizeRecalculation(); }
/** * Only use for testing / validation that size tracker is working. Not for production use. */ @VisibleForTesting public long updateCDCTotalSize() { cdcSizeTracker.submitOverflowSizeRecalculation(); // Give the update time to run try { Thread.sleep(DatabaseDescriptor.getCDCDiskCheckInterval() + 10); } catch (InterruptedException e) {} return cdcSizeTracker.totalCDCSizeOnDisk(); } }
/** * Only use for testing / validation that size tracker is working. Not for production use. */ @VisibleForTesting public long updateCDCTotalSize() { cdcSizeTracker.submitOverflowSizeRecalculation(); // Give the update time to run try { Thread.sleep(DatabaseDescriptor.getCDCDiskCheckInterval() + 10); } catch (InterruptedException e) {} return cdcSizeTracker.totalCDCSizeOnDisk(); } }
/** * Only use for testing / validation that size tracker is working. Not for production use. */ @VisibleForTesting public long updateCDCTotalSize() { cdcSizeTracker.submitOverflowSizeRecalculation(); // Give the update time to run try { Thread.sleep(DatabaseDescriptor.getCDCDiskCheckInterval() + 10); } catch (InterruptedException e) {} return cdcSizeTracker.totalCDCSizeOnDisk(); } }
private void throwIfForbidden(Mutation mutation, CommitLogSegment segment) throws WriteTimeoutException { if (mutation.trackedByCDC() && segment.getCDCState() == CDCState.FORBIDDEN) { cdcSizeTracker.submitOverflowSizeRecalculation(); NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 10, TimeUnit.SECONDS, "Rejecting Mutation containing CDC-enabled table. Free up space in {}.", DatabaseDescriptor.getCDCLogLocation()); throw new WriteTimeoutException(WriteType.CDC, ConsistencyLevel.LOCAL_ONE, 0, 1); } }
private void throwIfForbidden(Mutation mutation, CommitLogSegment segment) throws WriteTimeoutException { if (mutation.trackedByCDC() && segment.getCDCState() == CDCState.FORBIDDEN) { cdcSizeTracker.submitOverflowSizeRecalculation(); NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 10, TimeUnit.SECONDS, "Rejecting Mutation containing CDC-enabled table. Free up space in {}.", DatabaseDescriptor.getCDCLogLocation()); throw new WriteTimeoutException(WriteType.CDC, ConsistencyLevel.LOCAL_ONE, 0, 1); } }
private void throwIfForbidden(Mutation mutation, CommitLogSegment segment) throws WriteTimeoutException { if (mutation.trackedByCDC() && segment.getCDCState() == CDCState.FORBIDDEN) { cdcSizeTracker.submitOverflowSizeRecalculation(); NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 10, TimeUnit.SECONDS, "Rejecting Mutation containing CDC-enabled table. Free up space in {}.", DatabaseDescriptor.getCDCLogLocation()); throw new WriteTimeoutException(WriteType.CDC, ConsistencyLevel.LOCAL_ONE, 0, 1); } }
void processDiscardedSegment(CommitLogSegment segment) { // See synchronization in CommitLogSegment.setCDCState synchronized(segment.cdcStateLock) { // Add to flushed size before decrementing unflushed so we don't have a window of false generosity if (segment.getCDCState() == CDCState.CONTAINS) size += segment.onDiskSize(); if (segment.getCDCState() != CDCState.FORBIDDEN) unflushedCDCSize -= defaultSegmentSize(); } // Take this opportunity to kick off a recalc to pick up any consumer file deletion. submitOverflowSizeRecalculation(); }
void processDiscardedSegment(CommitLogSegment segment) { // See synchronization in CommitLogSegment.setCDCState synchronized(segment.cdcStateLock) { // Add to flushed size before decrementing unflushed so we don't have a window of false generosity if (segment.getCDCState() == CDCState.CONTAINS) size += segment.onDiskSize(); if (segment.getCDCState() != CDCState.FORBIDDEN) unflushedCDCSize -= defaultSegmentSize(); } // Take this opportunity to kick off a recalc to pick up any consumer file deletion. submitOverflowSizeRecalculation(); }
void processDiscardedSegment(CommitLogSegment segment) { // See synchronization in CommitLogSegment.setCDCState synchronized(segment.cdcStateLock) { // Add to flushed size before decrementing unflushed so we don't have a window of false generosity if (segment.getCDCState() == CDCState.CONTAINS) size += segment.onDiskSize(); if (segment.getCDCState() != CDCState.FORBIDDEN) unflushedCDCSize -= defaultSegmentSize(); } // Take this opportunity to kick off a recalc to pick up any consumer file deletion. submitOverflowSizeRecalculation(); }
@Override void start() { cdcSizeTracker.start(); super.start(); }
public void discard(CommitLogSegment segment, boolean delete) { segment.close(); addSize(-segment.onDiskSize()); cdcSizeTracker.processDiscardedSegment(segment); if (segment.getCDCState() == CDCState.CONTAINS) FileUtils.renameWithConfirm(segment.logFile.getAbsolutePath(), DatabaseDescriptor.getCDCLogLocation() + File.separator + segment.logFile.getName()); else { if (delete) FileUtils.deleteWithConfirm(segment.logFile); } }
/** * Initiates the shutdown process for the management thread. Also stops the cdc on-disk size calculator executor. */ public void shutdown() { cdcSizeTracker.shutdown(); super.shutdown(); }
private void recalculateOverflowSize() { rateLimiter.acquire(); calculateSize(); CommitLogSegment allocatingFrom = segmentManager.allocatingFrom(); if (allocatingFrom.getCDCState() == CDCState.FORBIDDEN) processNewSegment(allocatingFrom); }
/** * On segment creation, flag whether the segment should accept CDC mutations or not based on the total currently * allocated unflushed CDC segments and the contents of cdc_raw */ public CommitLogSegment createSegment() { CommitLogSegment segment = CommitLogSegment.createSegment(commitLog, this); cdcSizeTracker.processNewSegment(segment); return segment; }