@Test public void testV1SerializationNullMetrics() throws Exception { final DataSegment segment = DataSegment.builder() .dataSource("foo") .interval(Intervals.of("2012-01-01/2012-01-02")) .version(DateTimes.of("2012-01-01T11:22:33.444Z").toString()) .build(); final DataSegment segment2 = mapper.readValue(mapper.writeValueAsString(segment), DataSegment.class); Assert.assertEquals("empty dimensions", ImmutableList.of(), segment2.getDimensions()); Assert.assertEquals("empty metrics", ImmutableList.of(), segment2.getMetrics()); }
private DataSegment makeSegment(int offset) { return DataSegment.builder() .dataSource("foo") .interval( new Interval( DateTimes.of("2013-01-01").plusDays(offset), DateTimes.of("2013-01-02").plusDays(offset) ) ) .version(DateTimes.nowUtc().toString()) .dimensions(ImmutableList.of("dim1", "dim2")) .metrics(ImmutableList.of("met1", "met2")) .loadSpec(ImmutableMap.of("type", "local")) .build(); }
private DataSegment makeSegment(int offset) { return DataSegment.builder() .dataSource("foo") .interval( new Interval( SEGMENT_INTERVAL_START.plusDays(offset), SEGMENT_INTERVAL_START.plusDays(offset + 1) ) ) .version(DateTimes.nowUtc().toString()) .build(); }
@Test public void testOverlappingMerge1() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P4D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("3").size(25).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("1").size(25).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-05/P1D")).version("3").size(25).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-06/P1D")).version("2").size(80).build() ); Assert.assertEquals( ImmutableList.of(), merge(segments) ); }
@Test public void testOverlappingMergeWithGapsNonalignedStart() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P8D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P1D")).version("3").size(8).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("3").size(8).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-09/P1D")).version("3").size(8).build() ); Assert.assertEquals( ImmutableList.of( ImmutableList.of(segments.get(0), segments.get(1), segments.get(2)) ), merge(segments) ); }
@Test public void testOverlappingMergeWithGapsAlignedStart() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P8D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("3").size(8).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("3").size(8).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-09/P1D")).version("3").size(8).build() ); Assert.assertEquals( ImmutableList.of( ImmutableList.of(segments.get(1), segments.get(0), segments.get(2)) ), merge(segments) ); }
@Test public void testMergeAtStart() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(20).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("2").size(20).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("2").size(90).build() ); Assert.assertEquals( ImmutableList.of( ImmutableList.of(segments.get(0), segments.get(1)) ), merge(segments) ); }
private DataSegment dataSegmentWithInterval(String intervalStr) { return DataSegment.builder() .dataSource("test_load_queue_peon") .interval(Intervals.of(intervalStr)) .loadSpec(ImmutableMap.of()) .version("2015-05-27T03:38:35.683Z") .dimensions(ImmutableList.of()) .metrics(ImmutableList.of()) .shardSpec(NoneShardSpec.instance()) .binaryVersion(9) .size(1200) .build(); }
@Test public void testIdentifierWithNonzeroPartition() { final DataSegment segment = DataSegment.builder() .dataSource("foo") .interval(Intervals.of("2012-01-01/2012-01-02")) .version(DateTimes.of("2012-01-01T11:22:33.444Z").toString()) .shardSpec(getShardSpec(7)) .build(); Assert.assertEquals( "foo_2012-01-01T00:00:00.000Z_2012-01-02T00:00:00.000Z_2012-01-01T11:22:33.444Z_7", segment.getId().toString() ); }
@Test public void testIdentifier() { final DataSegment segment = DataSegment.builder() .dataSource("foo") .interval(Intervals.of("2012-01-01/2012-01-02")) .version(DateTimes.of("2012-01-01T11:22:33.444Z").toString()) .shardSpec(NoneShardSpec.instance()) .build(); Assert.assertEquals( "foo_2012-01-01T00:00:00.000Z_2012-01-02T00:00:00.000Z_2012-01-01T11:22:33.444Z", segment.getId().toString() ); }
@Test public void testMergeAtEnd() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("2").size(20).build() ); Assert.assertEquals( ImmutableList.of( ImmutableList.of(segments.get(2), segments.get(3)) ), merge(segments) ); }
@Test public void testMergeInMiddle() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("2").size(10).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("2").size(20).build() ); Assert.assertEquals( ImmutableList.of( ImmutableList.of(segments.get(1), segments.get(2)) ), merge(segments) ); }
@Test public void testMergeNoncontiguous() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(10).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("2").size(10).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("2").size(10).build() ); Assert.assertEquals( ImmutableList.of( ImmutableList.of(segments.get(0), segments.get(1), segments.get(2)) ), merge(segments) ); }
@Test public void testIdentifierWithZeroPartition() { final DataSegment segment = DataSegment.builder() .dataSource("foo") .interval(Intervals.of("2012-01-01/2012-01-02")) .version(DateTimes.of("2012-01-01T11:22:33.444Z").toString()) .shardSpec(getShardSpec(0)) .build(); Assert.assertEquals( "foo_2012-01-01T00:00:00.000Z_2012-01-02T00:00:00.000Z_2012-01-01T11:22:33.444Z", segment.getId().toString() ); }
@Test public void testNoMerges() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("2").size(80).build() ); Assert.assertEquals( ImmutableList.of(), merge(segments) ); }
@Override public TaskStatus run(TaskToolbox toolbox) throws Exception { final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction())); final DataSegment segment = DataSegment.builder() .dataSource("ds") .interval(Intervals.of("2012-01-01/P2D")) .version(myLock.getVersion()) .build(); toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment))); return TaskStatus.success(getId()); } };
public DataSegment withVersion(String version) { return builder(this).version(version).build(); }
@Override public TaskStatus run(TaskToolbox toolbox) throws Exception { final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction())); final DataSegment segment = DataSegment.builder() .dataSource("ds") .interval(Intervals.of("2012-01-01/P1D")) .version(myLock.getVersion() + "1!!!1!!") .build(); toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment))); return TaskStatus.success(getId()); } };
@Test public void testOverlappingMerge8() { final List<DataSegment> segments = ImmutableList.of( DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-01/P1D")).version("2").size(80).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-02/P4D")).version("2").size(120).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-03/P1D")).version("3").size(1).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-04/P1D")).version("1").size(1).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-05/P1D")).version("3").size(1).build(), DataSegment.builder().dataSource("foo").interval(Intervals.of("2012-01-06/P1D")).version("2").size(80).build() ); Assert.assertEquals(ImmutableList.of(ImmutableList.of(segments.get(4), segments.get(5))), merge(segments)); }
private DataSegment makeDataSegment(String dataSource, String interval, String version) { return DataSegment.builder() .dataSource(dataSource) .interval(Intervals.of(interval)) .version(version) .size(1) .build(); } }