@Override public String apply(DataSegment input) { return input.getVersion(); } }
private Pair<Map<Interval, String>, Map<Interval, List<DataSegment>>> getVersionAndBaseSegments( List<DataSegment> snapshot ) { Map<Interval, String> versions = new HashMap<>(); Map<Interval, List<DataSegment>> segments = new HashMap<>(); for (DataSegment segment : snapshot) { Interval interval = segment.getInterval(); versions.put(interval, segment.getVersion()); segments.putIfAbsent(interval, new ArrayList<>()); segments.get(interval).add(segment); } return new Pair<>(versions, segments); }
@Override public String toString() { return "DataSegment{" + "size=" + size + ", shardSpec=" + shardSpec + ", metrics=" + metrics + ", dimensions=" + dimensions + ", version='" + getVersion() + '\'' + ", loadSpec=" + loadSpec + ", interval=" + getInterval() + ", dataSource='" + getDataSource() + '\'' + ", binaryVersion='" + binaryVersion + '\'' + '}'; }
public SegmentDescriptor toDescriptor() { return new SegmentDescriptor(getInterval(), getVersion(), shardSpec.getPartitionNum()); }
public static SegmentIdWithShardSpec fromDataSegment(final DataSegment segment) { return new SegmentIdWithShardSpec( segment.getDataSource(), segment.getInterval(), segment.getVersion(), segment.getShardSpec() ); } }
@Override public Void withHandle(Handle handle) throws Exception { final PreparedBatch batch = handle.prepareBatch( StringUtils.format( "INSERT INTO %1$s (id, dataSource, created_date, start, %2$send%2$s, partitioned, version, used, payload) " + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)", tableName, connector.getQuoteString() ) ); for (final DataSegment segment : segments) { batch.add( new ImmutableMap.Builder<String, Object>() .put("id", segment.getId().toString()) .put("dataSource", segment.getDataSource()) .put("created_date", DateTimes.nowUtc().toString()) .put("start", segment.getInterval().getStart().toString()) .put("end", segment.getInterval().getEnd().toString()) .put("partitioned", (segment.getShardSpec() instanceof NoneShardSpec) ? false : true) .put("version", segment.getVersion()) .put("used", true) .put("payload", mapper.writeValueAsBytes(segment)) .build() ); log.info("Published %s", segment.getId()); } batch.execute(); return null; } }
public Builder withDataSources(Collection<ImmutableDruidDataSource> dataSourcesCollection) { dataSourcesCollection.forEach( dataSource -> { VersionedIntervalTimeline<String, DataSegment> timeline = dataSources.computeIfAbsent( dataSource.getName(), k -> new VersionedIntervalTimeline<>(String.CASE_INSENSITIVE_ORDER) ); dataSource.getSegments().forEach( segment -> timeline.add( segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment) ) ); } ); return this; }
/** * Due to https://issues.apache.org/jira/browse/HDFS-13 ":" are not allowed in * path names. So we format paths differently for HDFS. */ @Override public String getStorageDir(DataSegment segment, boolean useUniquePath) { // This is only called by HdfsDataSegmentPusher.push(), which will always set useUniquePath to false since any // 'uniqueness' will be applied not to the directory but to the filename along with the shard number. This is done // to avoid performance issues due to excessive HDFS directories. Hence useUniquePath is ignored here and we // expect it to be false. Preconditions.checkArgument( !useUniquePath, "useUniquePath must be false for HdfsDataSegmentPusher.getStorageDir()" ); return JOINER.join( segment.getDataSource(), StringUtils.format( "%s_%s", segment.getInterval().getStart().toString(ISODateTimeFormat.basicDateTime()), segment.getInterval().getEnd().toString(ISODateTimeFormat.basicDateTime()) ), segment.getVersion().replace(':', '_') ); }
@VisibleForTesting static boolean isLockCoversSegments( final Task task, final TaskLockbox taskLockbox, final Collection<DataSegment> segments ) { // Verify that each of these segments falls under some lock // NOTE: It is possible for our lock to be revoked (if the task has failed and given up its locks) after we check // NOTE: it and before we perform the segment insert, but, that should be OK since the worst that happens is we // NOTE: insert some segments from the task but not others. final NavigableMap<DateTime, TaskLock> taskLockMap = getTaskLockMap(taskLockbox, task); if (taskLockMap.isEmpty()) { return false; } return segments.stream().allMatch( segment -> { final Entry<DateTime, TaskLock> entry = taskLockMap.floorEntry(segment.getInterval().getStart()); if (entry == null) { return false; } final TaskLock taskLock = entry.getValue(); return taskLock.getInterval().contains(segment.getInterval()) && taskLock.getDataSource().equals(segment.getDataSource()) && taskLock.getVersion().compareTo(segment.getVersion()) >= 0; } ); }
@Override public void publishSegment(final DataSegment segment) throws IOException { publishSegment( segment.getId().toString(), segment.getDataSource(), DateTimes.nowUtc().toString(), segment.getInterval().getStart().toString(), segment.getInterval().getEnd().toString(), (segment.getShardSpec() instanceof NoneShardSpec) ? false : true, segment.getVersion(), true, jsonMapper.writeValueAsBytes(segment) ); }
private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) { SegmentId segmentId = segment.getId(); synchronized (lock) { log.debug("Adding segment[%s] for server[%s]", segment, server); SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); if (segmentLoadInfo == null) { // servers escape the scope of this object so use ConcurrentSet segmentLoadInfo = new SegmentLoadInfo(segment); VersionedIntervalTimeline<String, SegmentLoadInfo> timeline = timelines.get(segment.getDataSource()); if (timeline == null) { timeline = new VersionedIntervalTimeline<>(Ordering.natural()); timelines.put(segment.getDataSource(), timeline); } timeline.add( segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segmentLoadInfo) ); segmentLoadInfos.put(segmentId, segmentLoadInfo); } segmentLoadInfo.addServer(server); } }
@Override public String getStorageDir(DataSegment dataSegment, boolean useUniquePath) { String seg = JOINER.join( dataSegment.getDataSource(), StringUtils.format( "%s_%s", // Use ISODateTimeFormat.basicDateTime() format, to avoid using colons in file path. dataSegment.getInterval().getStart().toString(ISODateTimeFormat.basicDateTime()), dataSegment.getInterval().getEnd().toString(ISODateTimeFormat.basicDateTime()) ), dataSegment.getVersion().replace(':', '_'), dataSegment.getShardSpec().getPartitionNum(), useUniquePath ? DataSegmentPusher.generateUniquePath() : null ); log.info("DataSegment: [%s]", seg); // Replace colons with underscores, since they are not supported through wasb:// prefix return seg; }
static String getDefaultStorageDir(DataSegment segment, boolean useUniquePath) { return JOINER.join( segment.getDataSource(), StringUtils.format("%s_%s", segment.getInterval().getStart(), segment.getInterval().getEnd()), segment.getVersion(), segment.getShardSpec().getPartitionNum(), useUniquePath ? generateUniquePath() : null ); }
static boolean isSegmentLoaded(Iterable<ImmutableSegmentLoadInfo> serverView, SegmentDescriptor descriptor) { for (ImmutableSegmentLoadInfo segmentLoadInfo : serverView) { if (segmentLoadInfo.getSegment().getInterval().contains(descriptor.getInterval()) && segmentLoadInfo.getSegment().getShardSpec().getPartitionNum() == descriptor.getPartitionNumber() && segmentLoadInfo.getSegment().getVersion().compareTo(descriptor.getVersion()) >= 0 && Iterables.any( segmentLoadInfo.getServers(), DruidServerMetadata::segmentReplicatable )) { return true; } } return false; } }
@Override public String apply(DataSegment x) { return StringUtils.format( "%s_%s_%s_%s", x.getInterval().getStart(), x.getInterval().getEnd(), x.getVersion(), x.getShardSpec().getPartitionNum() ); } }
private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) { SegmentId segmentId = segment.getId(); synchronized (lock) { log.debug("Adding segment[%s] for server[%s]", segment, server); ServerSelector selector = selectors.get(segmentId); if (selector == null) { selector = new ServerSelector(segment, tierSelectorStrategy); VersionedIntervalTimeline<String, ServerSelector> timeline = timelines.get(segment.getDataSource()); if (timeline == null) { timeline = new VersionedIntervalTimeline<>(Ordering.natural()); timelines.put(segment.getDataSource(), timeline); } timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); selectors.put(segmentId, selector); } QueryableDruidServer queryableDruidServer = clients.get(server.getName()); if (queryableDruidServer == null) { queryableDruidServer = addServer(baseView.getInventoryValue(server.getName())); } selector.addServerAndUpdateSegment(queryableDruidServer, segment); runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); } }
public Builder(DataSegment segment) { this.dataSource = segment.getDataSource(); this.interval = segment.getInterval(); this.version = segment.getVersion(); this.loadSpec = segment.getLoadSpec(); this.dimensions = segment.getDimensions(); this.metrics = segment.getMetrics(); this.shardSpec = segment.getShardSpec(); this.binaryVersion = segment.getBinaryVersion(); this.size = segment.getSize(); }
public SpecificSegmentsQuerySegmentWalker add( final DataSegment descriptor, final QueryableIndex index ) { final Segment segment = new QueryableIndexSegment(index, descriptor.getId()); if (!timelines.containsKey(descriptor.getDataSource())) { timelines.put(descriptor.getDataSource(), new VersionedIntervalTimeline<>(Ordering.natural())); } final VersionedIntervalTimeline<String, Segment> timeline = timelines.get(descriptor.getDataSource()); timeline.add(descriptor.getInterval(), descriptor.getVersion(), descriptor.getShardSpec().createChunk(segment)); segments.add(descriptor); closeables.add(index); return this; }
private void addMoreData(String dataSource, int day) { for (int i = 0; i < 2; i++) { DataSegment newSegment = createSegment(dataSource, day, true, i); dataSources.get(dataSource).add( newSegment.getInterval(), newSegment.getVersion(), newSegment.getShardSpec().createChunk(newSegment) ); newSegment = createSegment(dataSource, day, false, i); dataSources.get(dataSource).add( newSegment.getInterval(), newSegment.getVersion(), newSegment.getShardSpec().createChunk(newSegment) ); } }
@Override public Set<DataSegment> findUsedSegments(Set<SegmentIdWithShardSpec> identifiers) { final VersionedIntervalTimeline<String, DataSegment> timeline = new VersionedIntervalTimeline<>(Ordering.natural()); for (DataSegment dataSegment : appenderatorTester.getPushedSegments()) { timeline.add( dataSegment.getInterval(), dataSegment.getVersion(), dataSegment.getShardSpec().createChunk(dataSegment) ); } final Set<DataSegment> retVal = new HashSet<>(); for (SegmentIdWithShardSpec identifier : identifiers) { for (TimelineObjectHolder<String, DataSegment> holder : timeline.lookup(identifier.getInterval())) { for (PartitionChunk<DataSegment> chunk : holder.getObject()) { if (identifiers.contains(SegmentIdWithShardSpec.fromDataSegment(chunk.getObject()))) { retVal.add(chunk.getObject()); } } } } return retVal; } }