public static PartitionIterator mergeAndFilter(List<UnfilteredPartitionIterator> iterators, int nowInSec, MergeListener listener) { // TODO: we could have a somewhat faster version if we were to merge the UnfilteredRowIterators directly as RowIterators return filter(merge(iterators, nowInSec, listener), nowInSec); }
private UnfilteredPartitionIterator mergeWithShortReadProtection(List<UnfilteredPartitionIterator> results, InetAddress[] sources, DataLimits.Counter mergedResultCounter) { // If we have only one results, there is no read repair to do and we can't get short reads if (results.size() == 1) return results.get(0); /* * So-called short reads stems from nodes returning only a subset of the results they have due to the limit, * but that subset not being enough post-reconciliation. So if we don't have a limit, don't bother. */ if (!command.limits().isUnlimited()) for (int i = 0; i < results.size(); i++) results.set(i, extendWithShortReadProtection(results.get(i), sources[i], mergedResultCounter)); return UnfilteredPartitionIterators.merge(results, command.nowInSec(), new RepairMergeListener(sources)); }
private UnfilteredPartitionIterator mergeWithShortReadProtection(List<UnfilteredPartitionIterator> results, InetAddress[] sources, DataLimits.Counter mergedResultCounter) { // If we have only one results, there is no read repair to do and we can't get short reads if (results.size() == 1) return results.get(0); /* * So-called short reads stems from nodes returning only a subset of the results they have due to the limit, * but that subset not being enough post-reconciliation. So if we don't have a limit, don't bother. */ if (!command.limits().isUnlimited()) for (int i = 0; i < results.size(); i++) results.set(i, extendWithShortReadProtection(results.get(i), sources[i], mergedResultCounter)); return UnfilteredPartitionIterators.merge(results, command.nowInSec(), new RepairMergeListener(sources)); }
private UnfilteredPartitionIterator mergeWithShortReadProtection(List<UnfilteredPartitionIterator> results, InetAddress[] sources, DataLimits.Counter mergedResultCounter) { // If we have only one results, there is no read repair to do and we can't get short reads if (results.size() == 1) return results.get(0); /* * So-called short reads stems from nodes returning only a subset of the results they have due to the limit, * but that subset not being enough post-reconciliation. So if we don't have a limit, don't bother. */ if (!command.limits().isUnlimited()) for (int i = 0; i < results.size(); i++) results.set(i, extendWithShortReadProtection(results.get(i), sources[i], mergedResultCounter)); return UnfilteredPartitionIterators.merge(results, command.nowInSec(), new RepairMergeListener(sources)); }
@SuppressWarnings("resource") // We make sure to close mergedIterator in close() and CompactionIterator is itself an AutoCloseable public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, CompactionController controller, int nowInSec, UUID compactionId, CompactionMetrics metrics) { this.controller = controller; this.type = type; this.scanners = scanners; this.nowInSec = nowInSec; this.compactionId = compactionId; this.bytesRead = 0; long bytes = 0; for (ISSTableScanner scanner : scanners) bytes += scanner.getLengthInBytes(); this.totalBytes = bytes; this.mergeCounters = new long[scanners.size()]; this.metrics = metrics; if (metrics != null) metrics.beginCompaction(this); UnfilteredPartitionIterator merged = scanners.isEmpty() ? EmptyIterators.unfilteredPartition(controller.cfs.metadata, false) : UnfilteredPartitionIterators.merge(scanners, nowInSec, listener()); boolean isForThrift = merged.isForThrift(); // to stop capture of iterator in Purger, which is confusing for debug merged = Transformation.apply(merged, new GarbageSkipper(controller, nowInSec)); this.compacted = Transformation.apply(merged, new Purger(isForThrift, controller, nowInSec)); }
@SuppressWarnings("resource") // We make sure to close mergedIterator in close() and CompactionIterator is itself an AutoCloseable public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, CompactionController controller, int nowInSec, UUID compactionId, CompactionMetrics metrics) { this.controller = controller; this.type = type; this.scanners = scanners; this.nowInSec = nowInSec; this.compactionId = compactionId; this.bytesRead = 0; long bytes = 0; for (ISSTableScanner scanner : scanners) bytes += scanner.getLengthInBytes(); this.totalBytes = bytes; this.mergeCounters = new long[scanners.size()]; this.metrics = metrics; if (metrics != null) metrics.beginCompaction(this); UnfilteredPartitionIterator merged = scanners.isEmpty() ? EmptyIterators.unfilteredPartition(controller.cfs.metadata, false) : UnfilteredPartitionIterators.merge(scanners, nowInSec, listener()); boolean isForThrift = merged.isForThrift(); // to stop capture of iterator in Purger, which is confusing for debug merged = Transformation.apply(merged, new GarbageSkipper(controller, nowInSec)); this.compacted = Transformation.apply(merged, new Purger(isForThrift, controller, nowInSec)); }
@SuppressWarnings("resource") // We make sure to close mergedIterator in close() and CompactionIterator is itself an AutoCloseable public CompactionIterator(OperationType type, List<ISSTableScanner> scanners, CompactionController controller, int nowInSec, UUID compactionId, CompactionMetrics metrics) { this.controller = controller; this.type = type; this.scanners = scanners; this.nowInSec = nowInSec; this.compactionId = compactionId; this.bytesRead = 0; long bytes = 0; for (ISSTableScanner scanner : scanners) bytes += scanner.getLengthInBytes(); this.totalBytes = bytes; this.mergeCounters = new long[scanners.size()]; this.metrics = metrics; if (metrics != null) metrics.beginCompaction(this); UnfilteredPartitionIterator merged = scanners.isEmpty() ? EmptyIterators.unfilteredPartition(controller.cfs.metadata, false) : UnfilteredPartitionIterators.merge(scanners, nowInSec, listener()); boolean isForThrift = merged.isForThrift(); // to stop capture of iterator in Purger, which is confusing for debug merged = Transformation.apply(merged, new GarbageSkipper(controller, nowInSec)); this.compacted = Transformation.apply(merged, new Purger(isForThrift, controller, nowInSec)); }