SegmentsToMerge segmentsToMerge = new SegmentsToMerge(); if (!segmentsToMerge.add(timelineObjects.get(i)) || segmentsToMerge.getByteCount() > params.getCoordinatorDynamicConfig().getMergeBytesLimit() || segmentsToMerge.getSegmentCount() >= params.getCoordinatorDynamicConfig().getMergeSegmentsLimit()) { i -= segmentsToMerge.backtrack(params.getCoordinatorDynamicConfig().getMergeBytesLimit()); if (segmentsToMerge.getSegmentCount() > 1) { stats.addToGlobalStat("mergedCount", mergeSegments(segmentsToMerge, entry.getKey())); if (segmentsToMerge.getSegmentCount() == 0) { segmentsToMerge = new SegmentsToMerge(); segmentsToMerge.backtrack(params.getCoordinatorDynamicConfig().getMergeBytesLimit()); if (segmentsToMerge.getSegmentCount() > 1) { stats.addToGlobalStat("mergedCount", mergeSegments(segmentsToMerge, entry.getKey()));
/** * Does this set of segments fully cover union(all segment intervals)? * * @return true if this set is complete */ public boolean isComplete() { return timelineObjects.size() == 0 || getMergedTimelineInterval().equals(getMergedUnderlyingInterval()); }
/** * Issue merge request for some list of segments. * * @return number of segments merged */ private int mergeSegments(SegmentsToMerge segmentsToMerge, String dataSource) { final List<DataSegment> segments = segmentsToMerge.getSegments(); final List<SegmentId> segmentNames = Lists.transform(segments, DataSegment::getId); log.info("[%s] Found %d segments to merge %s", dataSource, segments.size(), segmentNames); try { indexingServiceClient.mergeSegments(segments); } catch (Exception e) { log.error(e, "[%s] Merging error for segments %s", dataSource, segmentNames); } return segments.size(); }
SegmentsToMerge segmentsToMerge = new SegmentsToMerge(); if (!segmentsToMerge.add(timelineObjects.get(i)) || segmentsToMerge.getByteCount() > params.getCoordinatorDynamicConfig().getMergeBytesLimit() || segmentsToMerge.getSegmentCount() >= params.getCoordinatorDynamicConfig().getMergeSegmentsLimit()) { i -= segmentsToMerge.backtrack(params.getCoordinatorDynamicConfig().getMergeBytesLimit()); if (segmentsToMerge.getSegmentCount() > 1) { stats.addToGlobalStat("mergedCount", mergeSegments(segmentsToMerge, entry.getKey())); if (segmentsToMerge.getSegmentCount() == 0) { segmentsToMerge = new SegmentsToMerge(); segmentsToMerge.backtrack(params.getCoordinatorDynamicConfig().getMergeBytesLimit()); if (segmentsToMerge.getSegmentCount() > 1) { stats.addToGlobalStat("mergedCount", mergeSegments(segmentsToMerge, entry.getKey()));
final Interval mergedUnderlyingInterval = getMergedUnderlyingInterval(); if (mergedUnderlyingInterval == null) { timelineObjects.add(Pair.of(timelineObject, underlyingInterval));
final List<DataSegment> segments = segmentsToMerge.getSegments(); final List<String> segmentNames = Lists.transform( segments,
/** * Remove timelineObjects from this holder until we have a complete set with total size <= maxSize. * * @return number of timeline object holders removed */ public int backtrack(long maxSize) { Preconditions.checkArgument(maxSize >= 0, "maxSize >= 0"); int removed = 0; while (!isComplete() || byteCount > maxSize) { removed++; final TimelineObjectHolder<String, DataSegment> removedHolder = timelineObjects.remove( timelineObjects.size() - 1 ).lhs; for (final PartitionChunk<DataSegment> segment : removedHolder.getObject()) { segments.remove(segment.getObject()); if (segments.count(segment.getObject()) == 0) { byteCount -= segment.getObject().getSize(); } } } return removed; } }
/** * Does this set of segments fully cover union(all segment intervals)? * * @return true if this set is complete */ public boolean isComplete() { return timelineObjects.size() == 0 || getMergedTimelineInterval().equals(getMergedUnderlyingInterval()); }
final Interval mergedUnderlyingInterval = getMergedUnderlyingInterval(); if (mergedUnderlyingInterval == null) { timelineObjects.add(Pair.of(timelineObject, underlyingInterval));
/** * Remove timelineObjects from this holder until we have a complete set with total size <= maxSize. * * @return number of timeline object holders removed */ public int backtrack(long maxSize) { Preconditions.checkArgument(maxSize >= 0, "maxSize >= 0"); int removed = 0; while (!isComplete() || byteCount > maxSize) { removed++; final TimelineObjectHolder<String, DataSegment> removedHolder = timelineObjects.remove( timelineObjects.size() - 1 ).lhs; for (final PartitionChunk<DataSegment> segment : removedHolder.getObject()) { segments.remove(segment.getObject()); if (segments.count(segment.getObject()) == 0) { byteCount -= segment.getObject().getSize(); } } } return removed; } }