@Override public String apply(DataSegment input) { return input.getDataSource(); } }
private String getDataSource() { return segments.get(0).getDataSource(); } }
@Override public boolean apply(@Nullable DataSegment segment) { return segment == null || !segment.getDataSource().equalsIgnoreCase(dataSource); } }
private void setSegmentSignature(final DataSegment segment, final SegmentMetadataHolder segmentMetadataHolder) { synchronized (lock) { TreeMap<DataSegment, SegmentMetadataHolder> dataSourceSegments = segmentMetadataInfo.computeIfAbsent( segment.getDataSource(), x -> new TreeMap<>(SEGMENT_ORDER) ); if (dataSourceSegments.put(segment, segmentMetadataHolder) == null) { totalSegments++; } } }
private double computeDataSourceCost(DataSegment segment) { SegmentsCostCache costCache = segmentsPerDataSource.get(segment.getDataSource()); return (costCache == null) ? 0.0 : costCache.cost(segment); }
SegmentProvider(List<DataSegment> segments) { Preconditions.checkArgument(segments != null && !segments.isEmpty()); final String dataSource = segments.get(0).getDataSource(); Preconditions.checkArgument( segments.stream().allMatch(segment -> segment.getDataSource().equals(dataSource)), "segments should have the same dataSource" ); this.segments = segments; this.dataSource = dataSource; this.interval = JodaUtils.umbrellaInterval( segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()) ); }
public Builder addSegment(DataSegment dataSegment) { allSegmentsCostCache.addSegment(dataSegment); segmentsPerDataSource .computeIfAbsent(dataSegment.getDataSource(), d -> SegmentsCostCache.builder()) .addSegment(dataSegment); return this; }
public static SegmentIdWithShardSpec fromDataSegment(final DataSegment segment) { return new SegmentIdWithShardSpec( segment.getDataSource(), segment.getInterval(), segment.getVersion(), segment.getShardSpec() ); } }
public Builder removeSegment(DataSegment dataSegment) { allSegmentsCostCache.removeSegment(dataSegment); segmentsPerDataSource.computeIfPresent( dataSegment.getDataSource(), (ds, builder) -> builder.removeSegment(dataSegment).isEmpty() ? null : builder ); return this; }
static String getDefaultStorageDir(DataSegment segment, boolean useUniquePath) { return JOINER.join( segment.getDataSource(), StringUtils.format("%s_%s", segment.getInterval().getStart(), segment.getInterval().getEnd()), segment.getVersion(), segment.getShardSpec().getPartitionNum(), useUniquePath ? generateUniquePath() : null ); }
public Builder(DataSegment segment) { this.dataSource = segment.getDataSource(); this.interval = segment.getInterval(); this.version = segment.getVersion(); this.loadSpec = segment.getLoadSpec(); this.dimensions = segment.getDimensions(); this.metrics = segment.getMetrics(); this.shardSpec = segment.getShardSpec(); this.binaryVersion = segment.getBinaryVersion(); this.size = segment.getSize(); }
@Test public void testUsedOverlapHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), Intervals.of("2015-1-1T23:59:59.999Z/2015-02-01T00Z") ) ) ); }
@Test public void testSimpleUsedList() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval() ) ) ); }
@Test public void testUsedOutOfBoundsHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertTrue( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getEnd(), defaultSegment.getInterval().getEnd().plusDays(10)) ).isEmpty() ); }
@Test public void testUnUsedUnderlapLow() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getStart().plus(1), defaultSegment.getInterval().getEnd()) ).isEmpty() ); }
@Test public void testUsedOutOfBoundsLow() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertTrue( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getStart().minus(1), defaultSegment.getInterval().getStart()) ).isEmpty() ); }
@Test public void testUsedWithinBoundsEnd() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval().withEnd(defaultSegment.getInterval().getEnd().minusMillis(1)) ) ) ); }
@Test public void testUsedOverlapEnd() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); Assert.assertEquals( SEGMENTS, ImmutableSet.copyOf( coordinator.getUsedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval().withEnd(defaultSegment.getInterval().getEnd().plusMillis(1)) ) ) ); }
@Test public void testUnUsedOverlapHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), defaultSegment.getInterval().withStart(defaultSegment.getInterval().getEnd().minus(1)) ).isEmpty() ); }
@Test public void testUnUsedUnderlapHigh() throws IOException { coordinator.announceHistoricalSegments(SEGMENTS); unUseSegment(); Assert.assertTrue( coordinator.getUnusedSegmentsForInterval( defaultSegment.getDataSource(), new Interval(defaultSegment.getInterval().getStart(), defaultSegment.getInterval().getEnd().minus(1)) ).isEmpty() ); }