"CompactionManager", CompactionManager.instance.getActiveCompactions(), CompactionManager.instance.getPendingTasks())); int pendingLargeMessages = 0; for (int n : MessagingService.instance().getLargeMessagePendingTasks().values())
public void performMaximal(final ColumnFamilyStore cfStore) throws InterruptedException, ExecutionException { FBUtilities.waitOnFutures(submitMaximal(cfStore, getDefaultGcBefore(cfStore))); }
public LeveledScanner(Collection<SSTableReader> sstables, Range<Token> range) { this.range = range; // add only sstables that intersect our range, and estimate how much data that involves this.sstables = new ArrayList<SSTableReader>(sstables.size()); long length = 0; for (SSTableReader sstable : sstables) { this.sstables.add(sstable); long estimatedKeys = sstable.estimatedKeys(); double estKeysInRangeRatio = 1.0; if (estimatedKeys > 0 && range != null) estKeysInRangeRatio = ((double) sstable.estimatedKeysForRanges(Collections.singleton(range))) / estimatedKeys; length += sstable.uncompressedLength() * estKeysInRangeRatio; } totalLength = length; Collections.sort(this.sstables, SSTableReader.sstableComparator); sstableIterator = this.sstables.iterator(); assert sstableIterator.hasNext(); // caller should check intersecting first currentScanner = sstableIterator.next().getScanner(range, CompactionManager.instance.getRateLimiter()); }
private void invalidate() { // interrupt in-progress compactions Collection<ColumnFamilyStore> cfss = Collections.singleton(indexCfs); CompactionManager.instance.interruptCompactionForCFs(cfss, true); CompactionManager.instance.waitForCessation(cfss); Keyspace.writeOrder.awaitNewBarrier(); indexCfs.forceBlockingFlush(); indexCfs.readOrdering.awaitNewBarrier(); indexCfs.invalidate(); }
public void forceUserDefinedCompaction(String dataFiles) { String[] filenames = dataFiles.split(","); Multimap<ColumnFamilyStore, Descriptor> descriptors = ArrayListMultimap.create(); for (String filename : filenames) { // extract keyspace and columnfamily name from filename Descriptor desc = Descriptor.fromFilename(filename.trim()); if (Schema.instance.getCFMetaData(desc) == null) { logger.warn("Schema does not exist for file {}. Skipping.", filename); continue; } // group by keyspace/columnfamily ColumnFamilyStore cfs = Keyspace.open(desc.ksname).getColumnFamilyStore(desc.cfname); descriptors.put(cfs, cfs.getDirectories().find(new File(filename.trim()).getName())); } List<Future<?>> futures = new ArrayList<>(); int nowInSec = FBUtilities.nowInSeconds(); for (ColumnFamilyStore cfs : descriptors.keySet()) futures.add(submitUserDefined(cfs, descriptors.get(cfs), getDefaultGcBefore(cfs, nowInSec))); FBUtilities.waitOnFutures(futures); }
return; if (!needsCleanup(sstable, ranges)) RateLimiter limiter = getRateLimiter(); double compressionRatio = sstable.getCompressionRatio(); if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO) try (SSTableRewriter writer = SSTableRewriter.construct(cfs, txn, false, sstable.maxDataAge); ISSTableScanner scanner = cleanupStrategy.getScanner(sstable, null); CompactionController controller = new CompactionController(cfs, txn.originals(), getDefaultGcBefore(cfs, nowInSec)); Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)); CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, txn)); long lastBytesScanned = 0; compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
sstables = getSSTablesToValidate(cfs, validator); if (sstables == null) return; // this means the parent repair session was removed - the repair session failed on another node and we removed it gcBefore = validator.gcBefore; else gcBefore = getDefaultGcBefore(cfs, nowInSec); MerkleTrees tree = createMerkleTrees(sstables, validator.desc.ranges, cfs); long start = System.nanoTime(); try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables, validator.desc.ranges);
public void run() { try { logger.debug("Checking {}.{}", cfs.keyspace.getName(), cfs.name); if (!cfs.isValid()) { logger.debug("Aborting compaction for dropped CF"); return; } AbstractCompactionStrategy strategy = cfs.getCompactionStrategy(); AbstractCompactionTask task = strategy.getNextBackgroundTask(getDefaultGcBefore(cfs)); if (task == null) { logger.debug("No tasks available"); return; } task.execute(metrics); } finally { compactingCF.remove(cfs); } submitBackground(cfs); } }
public void interruptCompactionForCFs(Iterable<ColumnFamilyStore> cfss, boolean interruptValidation) { List<CFMetaData> metadata = new ArrayList<>(); for (ColumnFamilyStore cfs : cfss) metadata.add(cfs.metadata); interruptCompactionFor(metadata, interruptValidation); }
CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES); LifecycleTransaction.waitForDeletions(); System.exit(0); // We need that to stop non daemonized threads
public void forceCompactionForTokenRange(ColumnFamilyStore cfStore, Collection<Range<Token>> ranges) { final Collection<AbstractCompactionTask> tasks = cfStore.runWithCompactionsDisabled(() -> { Collection<SSTableReader> sstables = sstablesInBounds(cfStore, ranges); if (sstables == null || sstables.isEmpty()) { logger.debug("No sstables found for the provided token range"); return null; } return cfStore.getCompactionStrategyManager().getUserDefinedTasks(sstables, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds())); }, false, false); if (tasks == null) return; Runnable runnable = new WrappedRunnable() { protected void runMayThrow() { for (AbstractCompactionTask task : tasks) if (task != null) task.execute(metrics); } }; if (executor.isShutdown()) { logger.info("Compaction executor has shut down, not submitting task"); return; } FBUtilities.waitOnFuture(executor.submit(runnable)); }
SSTableRewriter unRepairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(anticompactionGroup, false, groupMaxDataAge); AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(anticompactionGroup.originals()); CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec)); CompactionIterator ci = new CompactionIterator(OperationType.ANTICOMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, sstableAsSet, anticompactionGroup)); unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstableAsSet, anticompactionGroup)); Range.OrderedRangeContainmentChecker containmentChecker = new Range.OrderedRangeContainmentChecker(ranges); while (ci.hasNext())
RateLimiter limiter = CompactionManager.instance.getRateLimiter(); long start = System.nanoTime(); long startTime = System.currentTimeMillis(); CompactionManager.compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
: new CompactionController(cfs, Collections.singleton(sstable), CompactionManager.getDefaultGcBefore(cfs)); this.isCommutative = cfs.metadata.isCounter(); : sstable.openDataReader(CompactionManager.instance.getRateLimiter());
CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs))) repairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable)); unRepairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstable));
public void dropCf(UUID cfId) { assert columnFamilyStores.containsKey(cfId); ColumnFamilyStore cfs = columnFamilyStores.remove(cfId); if (cfs == null) return; cfs.getCompactionStrategyManager().shutdown(); CompactionManager.instance.interruptCompactionForCFs(cfs.concatWithIndexes(), true); // wait for any outstanding reads/writes that might affect the CFS cfs.keyspace.writeOrder.awaitNewBarrier(); cfs.readOrdering.awaitNewBarrier(); unloadCf(cfs); }
CompactionManager.instance.forceShutdown(); CompactionManager.instance.forceShutdown();
assert txn.originals().equals(sstables); if (!sstables.isEmpty()) doAntiCompaction(cfs, ranges, txn, repairedAt); txn.finish();
writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable, transaction)); try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable, transaction))
@Override public void execute(LifecycleTransaction txn) throws IOException { logger.debug("Garbage collecting {}", txn.originals()); CompactionTask task = new CompactionTask(cfStore, txn, getDefaultGcBefore(cfStore, FBUtilities.nowInSeconds())) { @Override protected CompactionController getCompactionController(Set<SSTableReader> toCompact) { return new CompactionController(cfStore, toCompact, gcBefore, null, tombstoneOption); } }; task.setUserDefined(true); task.setCompactionType(OperationType.GARBAGE_COLLECT); task.execute(metrics); } }, jobs, OperationType.GARBAGE_COLLECT);