@Override public ServerView.CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) { CachingClusteredClient.this.cache.close(segment.getId().toString()); return ServerView.CallbackAction.CONTINUE; } }
@Override public SegmentId getId() { return SegmentId.dummy(version); }
@JsonProperty public String getVersion() { return id.getVersion(); }
public static Function<String, Interval> makeIntervalExtractor(final String dataSource) { return identifier -> { SegmentId segmentIdentifierParts = tryParse(dataSource, identifier); if (segmentIdentifierParts == null) { throw new IAE("Invalid identifier [%s]", identifier); } return segmentIdentifierParts.getInterval(); }; }
@JsonCreator public SegmentIdWithShardSpec( @JsonProperty("dataSource") String dataSource, @JsonProperty("interval") Interval interval, @JsonProperty("version") String version, @JsonProperty("shardSpec") ShardSpec shardSpec ) { this.id = SegmentId.of(dataSource, interval, version, shardSpec.getPartitionNum()); this.shardSpec = Preconditions.checkNotNull(shardSpec, "shardSpec"); this.asString = id.toString(); }
String probableDataSource = tryExtractMostProbableDataSource(segmentId); List<SegmentId> probableParsings = iteratePossibleParsingsWithDataSource(probableDataSource, segmentId); Iterable<SegmentId> otherPossibleParsings = () -> IntStream .range(1, splits.size() - 3) .mapToObj(dataSourceDelimiterOrder -> DELIMITER_JOINER.join(splits.subList(0, dataSourceDelimiterOrder))) .filter(dataSource -> dataSource.length() != probableDataSource.length()) .flatMap(dataSource -> iteratePossibleParsingsWithDataSource(dataSource, segmentId).stream()) .iterator(); return Iterables.concat(probableParsings, otherPossibleParsings); .mapToObj(dataSourceDelimiterOrder -> { String dataSource = DELIMITER_JOINER.join(splits.subList(0, dataSourceDelimiterOrder)); return iteratePossibleParsingsWithDataSource(dataSource, segmentId); }) .flatMap(List::stream)
@Test public void testBasic() { String datasource = "datasource"; SegmentId desc = SegmentId.of(datasource, Intervals.of("2015-01-02/2015-01-03"), "ver_0", 1); Assert.assertEquals("datasource_2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z_ver_0_1", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); desc = desc.withInterval(Intervals.of("2014-10-20T00:00:00Z/P1D")); Assert.assertEquals("datasource_2014-10-20T00:00:00.000Z_2014-10-21T00:00:00.000Z_ver_0_1", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); desc = SegmentId.of(datasource, Intervals.of("2015-01-02/2015-01-03"), "ver", 0); Assert.assertEquals("datasource_2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z_ver", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); desc = desc.withInterval(Intervals.of("2014-10-20T00:00:00Z/P1D")); Assert.assertEquals("datasource_2014-10-20T00:00:00.000Z_2014-10-21T00:00:00.000Z_ver", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); }
private void verifyRow( Object[] row, String segmentId, long size, long partitionNum, long numReplicas, long numRows, long isPublished, long isAvailable, long isRealtime) { Assert.assertEquals(segmentId, row[0].toString()); SegmentId id = Iterables.get(SegmentId.iterateAllPossibleParsings(segmentId), 0); Assert.assertEquals(id.getDataSource(), row[1]); Assert.assertEquals(id.getIntervalStart().toString(), row[2]); Assert.assertEquals(id.getIntervalEnd().toString(), row[3]); Assert.assertEquals(size, row[4]); Assert.assertEquals(id.getVersion(), row[5]); Assert.assertEquals(partitionNum, row[6]); Assert.assertEquals(numReplicas, row[7]); Assert.assertEquals(numRows, row[8]); Assert.assertEquals(isPublished, row[9]); Assert.assertEquals(isAvailable, row[10]); Assert.assertEquals(isRealtime, row[11]); }
@Override public boolean removeSegment(SegmentId segmentId) { try { final boolean removed = removeSegmentFromTable(segmentId.toString()); dataSources.computeIfPresent( segmentId.getDataSource(), (dsName, dataSource) -> { dataSource.removeSegment(segmentId); // Returning null from the lambda here makes the ConcurrentHashMap to remove the current entry. //noinspection ReturnOfNull return dataSource.isEmpty() ? null : dataSource; } ); return removed; } catch (Exception e) { log.error(e, e.toString()); return false; } }
@Test public void testIterateAllPossibleParsingsWithEmptyVersion() { String segmentId = "datasource_2015-01-01T00:00:00.000Z_2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z__1"; List<SegmentId> possibleParsings = ImmutableList.copyOf(SegmentId.iterateAllPossibleParsings(segmentId)); DateTime dt1 = DateTimes.of("2015-01-01T00:00:00.000Z"); DateTime dt2 = DateTimes.of("2015-01-02T00:00:00.000Z"); DateTime dt3 = DateTimes.of("2015-01-03T00:00:00.000Z"); Set<SegmentId> expected = ImmutableSet.of( SegmentId.of("datasource", new Interval(dt1, dt2), "2015-01-03T00:00:00.000Z_", 1), SegmentId.of("datasource", new Interval(dt1, dt2), "2015-01-03T00:00:00.000Z__1", 0), SegmentId.of("datasource_2015-01-01T00:00:00.000Z", new Interval(dt2, dt3), "", 1), SegmentId.of("datasource_2015-01-01T00:00:00.000Z", new Interval(dt2, dt3), "_1", 0) ); Assert.assertEquals(4, possibleParsings.size()); Assert.assertEquals(expected, ImmutableSet.copyOf(possibleParsings)); }
/** * Creates a dummy SegmentId with the given data source. This method is useful in benchmark and test code. */ public static SegmentId dummy(String dataSource) { return of(dataSource, Intervals.ETERNITY, "dummy_version", 0); }
@Test public void testTryParseVersionAmbiguity() { SegmentId segmentId = SegmentId.tryParse("datasource", "datasource_2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z_ver_0"); Assert.assertNotNull(segmentId); Assert.assertEquals("ver_0", segmentId.getVersion()); Assert.assertEquals(0, segmentId.getPartitionNum()); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof SegmentIdWithShardSpec)) { return false; } SegmentIdWithShardSpec that = (SegmentIdWithShardSpec) o; return id.equals(that.id); }
@JsonProperty public String getDataSource() { return id.getDataSource(); }
/** * Tries to parse a segment id from the given String representation, or returns null on failure. If returns a non-null * {@code SegmentId} object, calling {@link #toString()} on the latter is guaranteed to return a string equal to the * argument string of the {@code tryParse()} call. * * It is possible that this method may incorrectly parse a segment id, for example if the dataSource name in the * segment id contains a DateTime parseable string such as 'datasource_2000-01-01T00:00:00.000Z' and dataSource was * provided as 'datasource'. The desired behavior in this case would be to return null since the identifier does not * actually belong to the provided dataSource but a non-null result would be returned. This is an edge case that would * currently only affect paged select queries with a union dataSource of two similarly-named dataSources as in the * given example. * * Another source of ambiguity is the end of a segment id like '_123' - it could always be interpreted either as the * partitionNum of the segment id, or as the end of the version, with the implicit partitionNum of 0. This method * prefers the first iterpretation. To iterate all possible parsings of a segment id, use {@link * #iteratePossibleParsingsWithDataSource}. * * @param dataSource the dataSource corresponding to this segment id * @param segmentId segment id * @return a {@link SegmentId} object if the segment id could be parsed, null otherwise */ @Nullable public static SegmentId tryParse(String dataSource, String segmentId) { List<SegmentId> possibleParsings = iteratePossibleParsingsWithDataSource(dataSource, segmentId); return possibleParsings.isEmpty() ? null : possibleParsings.get(0); }
/** * The interval start is later than the end */ @Test public void testInvalidFormat0() { Assert.assertNull( SegmentId.tryParse("datasource", "datasource_2015-01-02T00:00:00.000Z_2014-10-20T00:00:00.000Z_version") ); }
@Override public int compareTo(SegmentIdWithShardSpec o) { return id.compareTo(o.id); }
.keySet() .stream() .filter(identifier -> SegmentId.tryParse(dataSource, identifier) != null) .map(SegmentId.makeIntervalExtractor(dataSource)) .sorted(query.isDescending() ? Comparators.intervalsByEndThenStart() : Comparators.intervalsByStartThenEnd())
@GET @Path("/{serverName}/segments/{segmentId}") @Produces(MediaType.APPLICATION_JSON) public Response getServerSegment(@PathParam("serverName") String serverName, @PathParam("segmentId") String segmentId) { DruidServer server = serverInventoryView.getInventoryValue(serverName); if (server == null) { return Response.status(Response.Status.NOT_FOUND).build(); } for (SegmentId possibleSegmentId : SegmentId.iterateAllPossibleParsings(segmentId)) { DataSegment segment = server.getSegment(possibleSegmentId); if (segment != null) { return Response.status(Response.Status.OK).entity(segment).build(); } } return Response.status(Response.Status.NOT_FOUND).build(); } }
@Test public void testDataSourceWithUnderscore() { String datasource = "datasource_1"; SegmentId desc = SegmentId.of(datasource, Intervals.of("2015-01-02/2015-01-03"), "ver_0", 1); Assert.assertEquals("datasource_1_2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z_ver_0_1", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); desc = desc.withInterval(Intervals.of("2014-10-20T00:00:00Z/P1D")); Assert.assertEquals("datasource_1_2014-10-20T00:00:00.000Z_2014-10-21T00:00:00.000Z_ver_0_1", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); desc = SegmentId.of(datasource, Intervals.of("2015-01-02/2015-01-03"), "ver", 0); Assert.assertEquals("datasource_1_2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z_ver", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); desc = desc.withInterval(Intervals.of("2014-10-20T00:00:00Z/P1D")); Assert.assertEquals("datasource_1_2014-10-20T00:00:00.000Z_2014-10-21T00:00:00.000Z_ver", desc.toString()); Assert.assertEquals(desc, SegmentId.tryParse(datasource, desc.toString())); }