@Override public boolean apply(@Nullable DataSegment segment) { return segment == null || !(segment.getShardSpec() instanceof NoneShardSpec); } }
public static void addSegments( VersionedIntervalTimeline<String, DataSegment> timeline, Iterator<DataSegment> segments ) { timeline.addAll( Iterators.transform(segments, segment -> segment.getShardSpec().createChunk(segment)), DataSegment::getInterval, DataSegment::getVersion ); }
@Override public File getSegmentFiles(DataSegment segment) { // dummy file to represent the downloaded segment's dir return new File("" + segment.getShardSpec().getPartitionNum()); }
@Override public String makeIndexPathName(DataSegment dataSegment, String indexName) { // This is only called from Hadoop batch which doesn't require unique segment paths so set useUniquePath=false return StringUtils.format( "./%s/%d_%s", this.getStorageDir(dataSegment, false), dataSegment.getShardSpec().getPartitionNum(), indexName ); } }
public static SegmentIdWithShardSpec fromDataSegment(final DataSegment segment) { return new SegmentIdWithShardSpec( segment.getDataSource(), segment.getInterval(), segment.getVersion(), segment.getShardSpec() ); } }
public Builder withDataSources(Collection<ImmutableDruidDataSource> dataSourcesCollection) { dataSourcesCollection.forEach( dataSource -> { VersionedIntervalTimeline<String, DataSegment> timeline = dataSources.computeIfAbsent( dataSource.getName(), k -> new VersionedIntervalTimeline<>(String.CASE_INSENSITIVE_ORDER) ); dataSource.getSegments().forEach( segment -> timeline.add( segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment) ) ); } ); return this; }
@Override public void publishSegment(final DataSegment segment) throws IOException { publishSegment( segment.getId().toString(), segment.getDataSource(), DateTimes.nowUtc().toString(), segment.getInterval().getStart().toString(), segment.getInterval().getEnd().toString(), (segment.getShardSpec() instanceof NoneShardSpec) ? false : true, segment.getVersion(), true, jsonMapper.writeValueAsBytes(segment) ); }
private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) { SegmentId segmentId = segment.getId(); synchronized (lock) { log.debug("Adding segment[%s] for server[%s]", segment, server); SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); if (segmentLoadInfo == null) { // servers escape the scope of this object so use ConcurrentSet segmentLoadInfo = new SegmentLoadInfo(segment); VersionedIntervalTimeline<String, SegmentLoadInfo> timeline = timelines.get(segment.getDataSource()); if (timeline == null) { timeline = new VersionedIntervalTimeline<>(Ordering.natural()); timelines.put(segment.getDataSource(), timeline); } timeline.add( segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segmentLoadInfo) ); segmentLoadInfos.put(segmentId, segmentLoadInfo); } segmentLoadInfo.addServer(server); } }
@Override public String getStorageDir(DataSegment dataSegment, boolean useUniquePath) { String seg = JOINER.join( dataSegment.getDataSource(), StringUtils.format( "%s_%s", // Use ISODateTimeFormat.basicDateTime() format, to avoid using colons in file path. dataSegment.getInterval().getStart().toString(ISODateTimeFormat.basicDateTime()), dataSegment.getInterval().getEnd().toString(ISODateTimeFormat.basicDateTime()) ), dataSegment.getVersion().replace(':', '_'), dataSegment.getShardSpec().getPartitionNum(), useUniquePath ? DataSegmentPusher.generateUniquePath() : null ); log.info("DataSegment: [%s]", seg); // Replace colons with underscores, since they are not supported through wasb:// prefix return seg; }
static String getDefaultStorageDir(DataSegment segment, boolean useUniquePath) { return JOINER.join( segment.getDataSource(), StringUtils.format("%s_%s", segment.getInterval().getStart(), segment.getInterval().getEnd()), segment.getVersion(), segment.getShardSpec().getPartitionNum(), useUniquePath ? generateUniquePath() : null ); }
static boolean isSegmentLoaded(Iterable<ImmutableSegmentLoadInfo> serverView, SegmentDescriptor descriptor) { for (ImmutableSegmentLoadInfo segmentLoadInfo : serverView) { if (segmentLoadInfo.getSegment().getInterval().contains(descriptor.getInterval()) && segmentLoadInfo.getSegment().getShardSpec().getPartitionNum() == descriptor.getPartitionNumber() && segmentLoadInfo.getSegment().getVersion().compareTo(descriptor.getVersion()) >= 0 && Iterables.any( segmentLoadInfo.getServers(), DruidServerMetadata::segmentReplicatable )) { return true; } } return false; } }
@Override public String apply(DataSegment x) { return StringUtils.format( "%s_%s_%s_%s", x.getInterval().getStart(), x.getInterval().getEnd(), x.getVersion(), x.getShardSpec().getPartitionNum() ); } }
private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) { SegmentId segmentId = segment.getId(); synchronized (lock) { log.debug("Adding segment[%s] for server[%s]", segment, server); ServerSelector selector = selectors.get(segmentId); if (selector == null) { selector = new ServerSelector(segment, tierSelectorStrategy); VersionedIntervalTimeline<String, ServerSelector> timeline = timelines.get(segment.getDataSource()); if (timeline == null) { timeline = new VersionedIntervalTimeline<>(Ordering.natural()); timelines.put(segment.getDataSource(), timeline); } timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); selectors.put(segmentId, selector); } QueryableDruidServer queryableDruidServer = clients.get(server.getName()); if (queryableDruidServer == null) { queryableDruidServer = addServer(baseView.getInventoryValue(server.getName())); } selector.addServerAndUpdateSegment(queryableDruidServer, segment); runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); } }
public Builder(DataSegment segment) { this.dataSource = segment.getDataSource(); this.interval = segment.getInterval(); this.version = segment.getVersion(); this.loadSpec = segment.getLoadSpec(); this.dimensions = segment.getDimensions(); this.metrics = segment.getMetrics(); this.shardSpec = segment.getShardSpec(); this.binaryVersion = segment.getBinaryVersion(); this.size = segment.getSize(); }
public SpecificSegmentsQuerySegmentWalker add( final DataSegment descriptor, final QueryableIndex index ) { final Segment segment = new QueryableIndexSegment(index, descriptor.getId()); if (!timelines.containsKey(descriptor.getDataSource())) { timelines.put(descriptor.getDataSource(), new VersionedIntervalTimeline<>(Ordering.natural())); } final VersionedIntervalTimeline<String, Segment> timeline = timelines.get(descriptor.getDataSource()); timeline.add(descriptor.getInterval(), descriptor.getVersion(), descriptor.getShardSpec().createChunk(segment)); segments.add(descriptor); closeables.add(index); return this; }
private void addMoreData(String dataSource, int day) { for (int i = 0; i < 2; i++) { DataSegment newSegment = createSegment(dataSource, day, true, i); dataSources.get(dataSource).add( newSegment.getInterval(), newSegment.getVersion(), newSegment.getShardSpec().createChunk(newSegment) ); newSegment = createSegment(dataSource, day, false, i); dataSources.get(dataSource).add( newSegment.getInterval(), newSegment.getVersion(), newSegment.getShardSpec().createChunk(newSegment) ); } }
private ServerSelector makeMockSingleDimensionSelector( DruidServer server, String dimension, String start, String end, int partitionNum ) { DataSegment segment = EasyMock.createNiceMock(DataSegment.class); EasyMock.expect(segment.getId()).andReturn(SegmentId.dummy(DATA_SOURCE)).anyTimes(); EasyMock.expect(segment.getShardSpec()).andReturn(new SingleDimensionShardSpec(dimension, start, end, partitionNum)) .anyTimes(); EasyMock.replay(segment); ServerSelector selector = new ServerSelector( segment, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()) ); selector.addServerAndUpdateSegment(new QueryableDruidServer(server, null), segment); return selector; }
@Override public Set<DataSegment> findUsedSegments(Set<SegmentIdWithShardSpec> identifiers) { final VersionedIntervalTimeline<String, DataSegment> timeline = new VersionedIntervalTimeline<>(Ordering.natural()); for (DataSegment dataSegment : appenderatorTester.getPushedSegments()) { timeline.add( dataSegment.getInterval(), dataSegment.getVersion(), dataSegment.getShardSpec().createChunk(dataSegment) ); } final Set<DataSegment> retVal = new HashSet<>(); for (SegmentIdWithShardSpec identifier : identifiers) { for (TimelineObjectHolder<String, DataSegment> holder : timeline.lookup(identifier.getInterval())) { for (PartitionChunk<DataSegment> chunk : holder.getObject()) { if (identifiers.contains(SegmentIdWithShardSpec.fromDataSegment(chunk.getObject()))) { retVal.add(chunk.getObject()); } } } } return retVal; } }
Assert.assertTrue(mergeSegment.getShardSpec() instanceof NoneShardSpec);
@Test public void testRun() throws Exception { runIndexTask(); final Builder builder = new Builder( DATA_SOURCE, getObjectMapper(), AuthTestUtils.TEST_AUTHORIZER_MAPPER, null, rowIngestionMetersFactory ); final CompactionTask compactionTask = builder .interval(Intervals.of("2014-01-01/2014-01-02")) .build(); final Pair<TaskStatus, List<DataSegment>> resultPair = runTask(compactionTask); Assert.assertTrue(resultPair.lhs.isSuccess()); final List<DataSegment> segments = resultPair.rhs; Assert.assertEquals(3, segments.size()); for (int i = 0; i < 3; i++) { Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), segments.get(i).getInterval()); Assert.assertEquals(new NumberedShardSpec(0, 0), segments.get(i).getShardSpec()); } }