List<TimelineObjectHolder<String, DataSegment>> existingChunks = timeline.lookup(segment.getInterval()); if (existingChunks.size() > 1) { + "Not possible to append new segment.", dataSource, segment.getInterval(), existingChunks.size())); dataSegmentPusher); finalSegmentsToPublish.add(publishedSegment); timeline.add(publishedSegment.getInterval(), publishedSegment.getVersion(), publishedSegment.getShardSpec().createChunk(publishedSegment)); .put("dataSource", segment.getDataSource()) .put("created_date", new DateTime().toString()) .put("start", segment.getInterval().getStart().toString()) .put("end", segment.getInterval().getEnd().toString()) .put("partitioned", !(segment.getShardSpec() instanceof NoneShardSpec)) .put("version", segment.getVersion())
private static VersionedIntervalTimeline<String, DataSegment> getTimelineForIntervalWithHandle(final Handle handle, final String dataSource, final Interval interval, final MetadataStorageTablesConfig dbTables) throws IOException { Query<Map<String, Object>> sql = handle.createQuery(String.format( "SELECT payload FROM %s WHERE used = true AND dataSource = ? AND start <= ? AND \"end\" >= ?", dbTables.getSegmentsTable())) .bind(0, dataSource) .bind(1, interval.getEnd().toString()) .bind(2, interval.getStart().toString()); final VersionedIntervalTimeline<String, DataSegment> timeline = new VersionedIntervalTimeline<>(Ordering.natural()); try (ResultIterator<byte[]> dbSegments = sql.map(ByteArrayMapper.FIRST).iterator()) { while (dbSegments.hasNext()) { final byte[] payload = dbSegments.next(); DataSegment segment = JSON_MAPPER.readValue(payload, DataSegment.class); timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment)); } } return timeline; }
@Override public Interval apply(DataSegment segment) { return segment.getInterval(); } }
@Override public Interval apply(DataSegment segment) { return segment.getInterval(); } }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { return appliesTo(segment.getInterval(), referenceTimestamp); }
@Override public int compareTo(SegmentAndSum o) { int c = Comparators.intervalsByStartThenEnd().compare(dataSegment.getInterval(), o.dataSegment.getInterval()); return (c != 0) ? c : dataSegment.compareTo(o.dataSegment); }
public static WindowedDataSegment of(final DataSegment segment) { return new WindowedDataSegment(segment, segment.getInterval()); } }
@Override public String apply(DataSegment x) { return String.format( "%s_%s_%s_%s", x.getInterval().getStart(), x.getInterval().getEnd(), x.getVersion(), x.getShardSpec().getPartitionNum() ); } }
public static VersionConverterTask create(DataSegment segment) { final Interval interval = segment.getInterval(); final String dataSource = segment.getDataSource(); final String id = makeId(dataSource, interval); return new VersionConverterTask(id, id, dataSource, interval, segment); }
@Override public boolean apply(TaskLock taskLock) { final boolean versionOk = allowOlderVersions ? taskLock.getVersion().compareTo(segment.getVersion()) >= 0 : taskLock.getVersion().equals(segment.getVersion()); return versionOk && taskLock.getDataSource().equals(segment.getDataSource()) && taskLock.getInterval().contains(segment.getInterval()); } }
@Override public void unannounceSegment(final DataSegment segment) throws IOException { try { toolbox.getSegmentAnnouncer().unannounceSegment(segment); } finally { toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval())); } }
@Override public void announceSegment(final DataSegment segment) throws IOException { // Side effect: Calling announceSegment causes a lock to be acquired toolbox.getTaskActionClient().submit(new LockAcquireAction(segment.getInterval())); toolbox.getSegmentAnnouncer().announceSegment(segment); }
@Override public void announceSegments(Iterable<DataSegment> segments) throws IOException { // Side effect: Calling announceSegments causes locks to be acquired for (DataSegment segment : segments) { toolbox.getTaskActionClient().submit(new LockAcquireAction(segment.getInterval())); } toolbox.getSegmentAnnouncer().announceSegments(segments); }
@Override public void unannounceSegment(final DataSegment segment) throws IOException { try { toolbox.getSegmentAnnouncer().unannounceSegment(segment); } finally { toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval())); } }