private SegmentIdWithShardSpec tryAllocateFirstSegment(TaskActionToolbox toolbox, Task task, Interval rowInterval) { // No existing segments for this row, but there might still be nearby ones that conflict with our preferred // segment granularity. Try that first, and then progressively smaller ones if it fails. final List<Interval> tryIntervals = Granularity.granularitiesFinerThan(preferredSegmentGranularity) .stream() .map(granularity -> granularity.bucket(timestamp)) .collect(Collectors.toList()); for (Interval tryInterval : tryIntervals) { if (tryInterval.contains(rowInterval)) { final SegmentIdWithShardSpec identifier = tryAllocate(toolbox, task, tryInterval, rowInterval, false); if (identifier != null) { return identifier; } } } return null; }
interval = config.getGranularitySpec() .getSegmentGranularity() .bucket(DateTimes.utc(inputRow.getTimestampFromEpoch()));
public static List<Granularity> granularitiesFinerThan(final Granularity gran0) { final List<Granularity> retVal = new ArrayList<>(); final DateTime origin = (gran0 instanceof PeriodGranularity) ? ((PeriodGranularity) gran0).getOrigin() : null; final DateTimeZone tz = (gran0 instanceof PeriodGranularity) ? ((PeriodGranularity) gran0).getTimeZone() : null; for (GranularityType gran : GranularityType.values()) { /** * All and None are excluded b/c when asked to give all granularities finer * than "TEN_MINUTE", you want the answer to be "FIVE_MINUTE, MINUTE and SECOND" * it doesn't make sense to include ALL or None to be part of this. */ if (gran == GranularityType.ALL || gran == GranularityType.NONE) { continue; } final Granularity segmentGranularity = gran.create(origin, tz); final long segmentGranularityDurationMillis = segmentGranularity.bucket(DateTimes.EPOCH).toDurationMillis(); final long gran0DurationMillis = gran0.bucket(DateTimes.EPOCH).toDurationMillis(); if (segmentGranularityDurationMillis <= gran0DurationMillis) { retVal.add(segmentGranularity); } } retVal.sort((g1, g2) -> { long duration1 = g2.bucket(DateTimes.EPOCH).toDurationMillis(); long duration2 = g1.bucket(DateTimes.EPOCH).toDurationMillis(); return Longs.compare(duration1, duration2); }); return retVal; }
@Override public SegmentIdWithShardSpec allocate( final InputRow row, final String sequenceName, final String previousSegmentId, final boolean skipSegmentLineageCheck ) { synchronized (counters) { DateTime dateTimeTruncated = granularity.bucketStart(row.getTimestamp()); final long timestampTruncated = dateTimeTruncated.getMillis(); if (!counters.containsKey(timestampTruncated)) { counters.put(timestampTruncated, new AtomicInteger()); } final int partitionNum = counters.get(timestampTruncated).getAndIncrement(); return new SegmentIdWithShardSpec( dataSource, granularity.bucket(dateTimeTruncated), VERSION, new NumberedShardSpec(partitionNum, 0) ); } } }
/** * Build a filter for an expression like FLOOR(column TO granularity) [operator] rhsMillis */ private static DimFilter buildTimeFloorFilter( final String column, final Granularity granularity, final SqlKind operatorKind, final long rhsMillis ) { final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC); final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis)); // Is rhs aligned on granularity boundaries? final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis; return getBoundTimeDimFilter(operatorKind, boundRefKey, rhsInterval, rhsAligned); }
interval = granularitySpec.getSegmentGranularity().bucket(inputRow.getTimestamp()); } else { if (!Intervals.ETERNITY.contains(inputRow.getTimestamp())) {
final Interval rowInterval = queryGranularity.bucket(timestamp);
interval = config.getGranularitySpec().getSegmentGranularity().bucket(DateTimes.utc(key.get())); } else { Optional<Interval> intervalOptional = config.getGranularitySpec().bucketInterval(DateTimes.utc(key.get()));
@Test public void testBucket() { DateTime dt = DateTimes.of("2011-02-03T04:05:06.100"); Assert.assertEquals(Intervals.of("2011-01-01/2012-01-01"), YEAR.bucket(dt)); Assert.assertEquals(Intervals.of("2011-02-01/2011-03-01"), MONTH.bucket(dt)); Assert.assertEquals(Intervals.of("2011-01-31/2011-02-07"), WEEK.bucket(dt)); Assert.assertEquals(Intervals.of("2011-02-03/2011-02-04"), DAY.bucket(dt)); Assert.assertEquals(Intervals.of("2011-02-03T04/2011-02-03T05"), HOUR.bucket(dt)); Assert.assertEquals(Intervals.of("2011-02-03T04:05:00/2011-02-03T04:06:00"), MINUTE.bucket(dt)); Assert.assertEquals(Intervals.of("2011-02-03T04:05:06/2011-02-03T04:05:07"), SECOND.bucket(dt)); // Test with aligned DateTime Assert.assertEquals(Intervals.of("2011-01-01/2011-01-02"), DAY.bucket(DateTimes.of("2011-01-01"))); }
final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis));
DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(0, 2)) DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(1, 2)) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2)
DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(0, 2)) DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(1, 2)) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2)
@Test public void testCannotAddToExistingSingleDimensionShardSpecs() throws Exception { final Task task = new NoopTask(null, null, 0, 0, null, null, null); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new SingleDimensionShardSpec("foo", null, "bar", 0)) .build(), DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new SingleDimensionShardSpec("foo", "bar", null, 1)) .build() ) ); taskActionTestKit.getTaskLockbox().add(task); final SegmentIdWithShardSpec id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null); Assert.assertNull(id1); }
@Test public void testCannotAddToExistingNumberedShardSpecsWithCoarserQueryGranularity() throws Exception { final Task task = new NoopTask(null, null, 0, 0, null, null, null); taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments( ImmutableSet.of( DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(0, 2)) .build(), DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(1, 2)) .build() ) ); taskActionTestKit.getTaskLockbox().add(task); final SegmentIdWithShardSpec id1 = allocate(task, PARTY_TIME, Granularities.DAY, Granularities.DAY, "s1", null); Assert.assertNull(id1); }
DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new LinearShardSpec(0)) DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new LinearShardSpec(1)) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new LinearShardSpec(2) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new LinearShardSpec(3)
DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(0, 2)) DataSegment.builder() .dataSource(DATA_SOURCE) .interval(Granularities.HOUR.bucket(PARTY_TIME)) .version(PARTY_TIME.toString()) .shardSpec(new NumberedShardSpec(1, 2)) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(3, 2)
Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp()); ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0); return (RetType) new SegmentIdWithShardSpec(action.getDataSource(), interval, "latestVersion", shardSpec);
new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(2, 0)
new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(2, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(0, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(1, 0)
new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(0, 0) new SegmentIdWithShardSpec( DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0)