private int compare(Clustering c1, Clustering c2, ColumnFamilyStore cfs) { if (c1 == Clustering.STATIC_CLUSTERING) return c2 == Clustering.STATIC_CLUSTERING ? 0 : -1; if (c2 == Clustering.STATIC_CLUSTERING) return 1; return cfs.getComparator().compare(c1, c2); }
public long[] getValue() { return combineHistograms(cfs.getSSTables(SSTableSet.CANONICAL), new GetHistogram() { public EstimatedHistogram getHistogram(SSTableReader reader) { return reader.getEstimatedPartitionSize(); } }); } });
public String getCompactionParametersJson() { return FBUtilities.json(getCompactionParameters()); }
public String toString() { return String.format("TimeWindowCompactionStrategy[%s/%s]", cfs.getMinimumCompactionThreshold(), cfs.getMaximumCompactionThreshold()); } }
private void updateEstimatedCompactionsByTasks(List<List<SSTableReader>> tasks) { int n = 0; for (List<SSTableReader> bucket : tasks) { for (List<SSTableReader> stcsBucket : getSTCSBuckets(bucket, stcsOptions)) if (stcsBucket.size() >= cfs.getMinimumCompactionThreshold()) n += Math.ceil((double)stcsBucket.size() / cfs.getMaximumCompactionThreshold()); } estimatedRemainingTasks = n; cfs.getCompactionStrategyManager().compactionLogger.pending(this, n); }
public void migrate() { // nothing to migrate if (legacyHintsTable.isEmpty()) return; logger.info("Migrating legacy hints to new storage"); // major-compact all of the existing sstables to get rid of the tombstones + expired hints logger.info("Forcing a major compaction of {}.{} table", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS); compactLegacyHints(); // paginate over legacy hints and write them to the new storage logger.info("Writing legacy hints to the new storage"); migrateLegacyHints(); // truncate the legacy hints table logger.info("Truncating {}.{} table", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS); legacyHintsTable.truncateBlocking(); }
ColumnFamilyStore.scrubDataDirectories(cfm); for (ColumnFamilyStore store : cfs.concatWithIndexes()) store.disableAutoCompaction(); for (final ColumnFamilyStore store : cfs.concatWithIndexes()) store.reload(); //reload CFs in case there was a change of disk boundaries if (store.getCompactionStrategyManager().shouldBeEnabled()) store.enableAutoCompaction(); ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(ColumnFamilyStore.getBackgroundCompactionTaskSubmitter(), 5, 1, TimeUnit.MINUTES);
private synchronized List<SSTableReader> getNextBackgroundSSTables(final int gcBefore) { // make local copies so they can't be changed out from under us mid-method int minThreshold = cfs.getMinimumCompactionThreshold(); int maxThreshold = cfs.getMaximumCompactionThreshold(); Iterable<SSTableReader> candidates = filterSuspectSSTables(filter(cfs.getUncompactingSSTables(), sstables::contains)); List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(candidates), sizeTieredOptions.bucketHigh, sizeTieredOptions.bucketLow, sizeTieredOptions.minSSTableSize); logger.trace("Compaction buckets are {}", buckets); estimatedRemainingTasks = getEstimatedCompactionsByTasks(cfs, buckets); cfs.getCompactionStrategyManager().compactionLogger.pending(this, estimatedRemainingTasks); List<SSTableReader> mostInteresting = mostInterestingBucket(buckets, minThreshold, maxThreshold); if (!mostInteresting.isEmpty()) return mostInteresting; // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone // ratio is greater than threshold. List<SSTableReader> sstablesWithTombstones = new ArrayList<>(); for (SSTableReader sstable : candidates) { if (worthDroppingTombstones(sstable, gcBefore)) sstablesWithTombstones.add(sstable); } if (sstablesWithTombstones.isEmpty()) return Collections.emptyList(); return Collections.singletonList(Collections.max(sstablesWithTombstones, SSTableReader.sizeComparator)); }
ColumnFamilyStore.scrubDataDirectories(cfm); try ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfm, unfinishedCompactions.get(kscf)); ColumnFamilyStore.scrubDataDirectories(cfm); for (ColumnFamilyStore store : cfs.concatWithIndexes()) store.disableAutoCompaction(); for (final ColumnFamilyStore store : cfs.concatWithIndexes()) if (store.getCompactionStrategy().shouldBeEnabled()) store.enableAutoCompaction(); ScheduledExecutors.optionalTasks.scheduleWithFixedDelay(ColumnFamilyStore.getBackgroundCompactionTaskSubmitter(), 5, 1, TimeUnit.MINUTES);
if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE))) return Collections.emptyList(); Set<SSTableReader> uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains)); expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, options.ignoreOverlaps ? Collections.emptySet() : cfs.getOverlappingLiveSSTables(uncompacting), gcBefore, options.ignoreOverlaps); lastExpiredCheck = System.currentTimeMillis();
private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore) { // make local copies so they can't be changed out from under us mid-method int minThreshold = cfs.getMinimumCompactionThreshold(); int maxThreshold = cfs.getMaximumCompactionThreshold(); Iterable<SSTableReader> candidates = filterSuspectSSTables(filter(cfs.getUncompactingSSTables(), sstables::contains)); List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(candidates), sizeTieredOptions.bucketHigh, sizeTieredOptions.bucketLow, sizeTieredOptions.minSSTableSize); logger.trace("Compaction buckets are {}", buckets); updateEstimatedCompactionsByTasks(buckets); List<SSTableReader> mostInteresting = mostInterestingBucket(buckets, minThreshold, maxThreshold); if (!mostInteresting.isEmpty()) return mostInteresting; // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone // ratio is greater than threshold. List<SSTableReader> sstablesWithTombstones = new ArrayList<>(); for (SSTableReader sstable : candidates) { if (worthDroppingTombstones(sstable, gcBefore)) sstablesWithTombstones.add(sstable); } if (sstablesWithTombstones.isEmpty()) return Collections.emptyList(); Collections.sort(sstablesWithTombstones, new SSTableReader.SizeComparator()); return Collections.singletonList(sstablesWithTombstones.get(0)); }
if (Iterables.isEmpty(cfs.getSSTables(SSTableSet.LIVE))) return Collections.emptyList(); Set<SSTableReader> uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains)); expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(SSTableSet.CANONICAL, uncompacting), gcBefore); lastExpiredCheck = System.currentTimeMillis();
clearEphemeralSnapshots(directories); scrubDataDirectories(indexMetadata);
public Throwable dropSSTablesIfInvalid(Throwable accumulate) { if (!isDummy() && !cfstore.isValid()) accumulate = dropSSTables(accumulate); return accumulate; }
public void run() { try { logger.trace("Checking {}.{}", cfs.keyspace.getName(), cfs.name); if (!cfs.isValid()) { logger.trace("Aborting compaction for dropped CF"); return; } CompactionStrategyManager strategy = cfs.getCompactionStrategyManager(); AbstractCompactionTask task = strategy.getNextBackgroundTask(getDefaultGcBefore(cfs, FBUtilities.nowInSeconds())); if (task == null) { logger.trace("No tasks available"); return; } task.execute(metrics); } finally { compactingCF.remove(cfs); } submitBackground(cfs); } }
/** * * @param gcBefore * @return */ private synchronized List<SSTableReader> getNextBackgroundSSTables(final int gcBefore) { if (sstables.isEmpty()) return Collections.emptyList(); Set<SSTableReader> uncompacting = ImmutableSet.copyOf(filter(cfs.getUncompactingSSTables(), sstables::contains)); Set<SSTableReader> expired = Collections.emptySet(); // we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency) { // Find fully expired SSTables. Those will be included no matter what. expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingLiveSSTables(uncompacting), gcBefore); lastExpiredCheck = System.currentTimeMillis(); } Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting)); List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore)); if (!expired.isEmpty()) { logger.trace("Including expired sstables: {}", expired); compactionCandidates.addAll(expired); } return compactionCandidates; }
public Long getValue() { long memtablePartitions = 0; for (Memtable memtable : cfs.getTracker().getView().getAllMemtables()) memtablePartitions += memtable.partitionCount(); return SSTableReader.getApproximateKeyCount(cfs.getSSTables(SSTableSet.CANONICAL)) + memtablePartitions; } });
public static boolean isIndexBuilt(String keyspaceName, String indexName) { ColumnFamilyStore cfs = Keyspace.open(Keyspace.SYSTEM_KS).getColumnFamilyStore(INDEX_CF); QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes(keyspaceName)), INDEX_CF, FBUtilities.singleton(cfs.getComparator().makeCellName(indexName), cfs.getComparator()), System.currentTimeMillis()); return ColumnFamilyStore.removeDeleted(cfs.getColumnFamily(filter), Integer.MAX_VALUE) != null; }
private SSTableWriter createCompactionWriter(long repairedAt) { MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator()); sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel()); return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(directory)), estimatedRows, repairedAt, cfs.metadata, sstableMetadataCollector, SerializationHeader.make(cfs.metadata, Sets.newHashSet(sstable)), cfs.indexManager.listIndexes(), transaction); }
public Callable<?> getInitializationTask() { // if we're just linking in the index on an already-built index post-restart or if the base // table is empty we've nothing to do. Otherwise, submit for building via SecondaryIndexBuilder return isBuilt() || baseCfs.isEmpty() ? null : getBuildIndexTask(); }