@Override public long getSegmentId() { return segment.getSegmentId(); }
public static final Map<Long, Long> toSegmentOffsetMap(final PositionInternal position) { Preconditions.checkNotNull(position, "position"); return position.getOwnedSegmentsWithOffsets() .entrySet() .stream() .map(e -> new SimpleEntry<>(e.getKey().getSegmentId(), e.getValue())) .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); }
private Map<Long, Long> getStreamCutMap(StreamCut streamCut) { if (streamCut.equals(StreamCut.UNBOUNDED)) { return Collections.emptyMap(); } return streamCut.asImpl().getPositions().entrySet() .stream().collect(Collectors.toMap(x -> x.getKey().getSegmentId(), Map.Entry::getValue)); }
private Map<Long, Long> getStreamCutMap(StreamCut streamCut) { if (streamCut.equals(StreamCut.UNBOUNDED)) { return Collections.emptyMap(); } return streamCut.asImpl().getPositions().entrySet() .stream().collect(Collectors.toMap(x -> x.getKey().getSegmentId(), Map.Entry::getValue)); } }
@Override public String toString() { return stream.getScopedName() + ":" + ToStringUtils.mapToString(positions.entrySet() .stream() .collect(Collectors.toMap(e -> e.getKey().getSegmentId(), e -> e.getValue()))); }
@Override public CompletableFuture<Boolean> truncateStream(final String scope, final String stream, final StreamCut streamCut) { final Map<Long, Long> segmentToOffsetMap = streamCut.asImpl().getPositions().entrySet().stream() .collect(Collectors.toMap(e -> e.getKey().getSegmentId(), Map.Entry::getValue)); return truncateStream(scope, stream, segmentToOffsetMap); }
private String getText() { StringBuilder builder = new StringBuilder(Integer.toString(TO_STRING_VERSION)).append(":"); builder.append(stream.getScopedName()).append(":"); // append Stream name. //split segmentNumbers, epochs and offsets into separate lists. List<Integer> segmentNumbers = new ArrayList<>(); List<Integer> epochs = new ArrayList<>(); List<Long> offsets = new ArrayList<>(); positions.forEach((segmentId, offset) -> { segmentNumbers.add(StreamSegmentNameUtils.getSegmentNumber(segmentId.getSegmentId())); epochs.add(StreamSegmentNameUtils.getEpoch(segmentId.getSegmentId())); offsets.add(offset); }); // append segmentsNumbers, epochs and offsets. builder.append(listToString(segmentNumbers)).append(":"); builder.append(listToString(epochs)).append(":"); builder.append(listToString(offsets)); return builder.toString(); }
/** * Decodes segment and returns an instance of SegmentId. * * @param segment The segment. * @return Instance of SegmentId. */ public static final SegmentId decode(final Segment segment) { Preconditions.checkNotNull(segment, "segment"); return createSegmentId(segment.getScope(), segment.getStreamName(), segment.getSegmentId()); }
private void triggerScaleUp(String streamSegmentName, int numOfSplits) { if (initialized.get()) { Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName); long lastRequestTs = 0; if (pair != null && pair.getKey() != null) { lastRequestTs = pair.getKey(); } long timestamp = System.currentTimeMillis(); long requestId = requestIdGenerator.get(); if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) { log.info(requestId, "sending request for scale up for {}", streamSegmentName); Segment segment = Segment.fromScopedName(streamSegmentName); AutoScaleEvent event = new AutoScaleEvent(segment.getScope(), segment.getStreamName(), segment.getSegmentId(), AutoScaleEvent.UP, timestamp, numOfSplits, false, requestId); // Mute scale for timestamp for both scale up and down writeRequest(event).thenAccept(x -> cache.put(streamSegmentName, new ImmutablePair<>(timestamp, timestamp))); } } }
@Override public CompletableFuture<Boolean> isSegmentOpen(Segment segment) { return controller.isSegmentValid(segment.getScope(), segment.getStreamName(), segment.getSegmentId()); }
private CompletableFuture<StreamSegmentsWithPredecessors> getReplacement(Segment old, Segment repacement) { Map<SegmentWithRange, List<Long>> segments = new HashMap<>(); segments.put(new SegmentWithRange(repacement, 0, 1), Collections.singletonList(old.getSegmentId())); return CompletableFuture.completedFuture(new StreamSegmentsWithPredecessors(segments, "")); }
@Override public CompletableFuture<PravegaNodeUri> getEndpointForSegment(String qualifiedSegmentName) { Segment segment = Segment.fromScopedName(qualifiedSegmentName); return controller.getURI(ModelHelper.createSegmentId(segment.getScope(), segment.getStreamName(), segment.getSegmentId())).thenApply(ModelHelper::encode); }
private HashMap<Segment, StreamSegmentsWithPredecessors> mergeSegments(HashMap<Segment, Range<Double>> ranges, Entry<Segment, Segment> pair, Segment combined) { Range<Double> lowerRange = ranges.get(pair.getKey()); Range<Double> upperRange = ranges.get(pair.getValue()); assertEquals(lowerRange.upperEndpoint(), upperRange.lowerEndpoint()); Map<SegmentWithRange, List<Long>> newSegments = new HashMap<>(); newSegments.put(new SegmentWithRange(combined, lowerRange.lowerEndpoint(), upperRange.upperEndpoint()), ImmutableList.of(pair.getKey().getSegmentId(), pair.getValue().getSegmentId())); StreamSegmentsWithPredecessors replacementRanges = new StreamSegmentsWithPredecessors(newSegments, ""); HashMap<Segment, StreamSegmentsWithPredecessors> replacements = new HashMap<>(); replacements.put(pair.getKey(), replacementRanges); replacements.put(pair.getValue(), replacementRanges); return replacements; }
private HashMap<Segment, StreamSegmentsWithPredecessors> splitSegment(HashMap<Segment, Range<Double>> ranges, Segment oldSegment, Segment lower, Segment upper) { Range<Double> range = ranges.get(oldSegment); double midpoint = (range.upperEndpoint() + range.lowerEndpoint()) / 2.0; Map<SegmentWithRange, List<Long>> newSegments = new HashMap<>(); newSegments.put(new SegmentWithRange(lower, range.lowerEndpoint(), midpoint), Collections.singletonList(oldSegment.getSegmentId())); newSegments.put(new SegmentWithRange(upper, midpoint, range.upperEndpoint()), Collections.singletonList(oldSegment.getSegmentId())); StreamSegmentsWithPredecessors replacementRanges = new StreamSegmentsWithPredecessors(newSegments, ""); HashMap<Segment, StreamSegmentsWithPredecessors> replacements = new HashMap<>(); replacements.put(oldSegment, replacementRanges); return replacements; }
private void write00(StreamCutInternal cut, RevisionDataOutput revisionDataOutput) throws IOException { revisionDataOutput.writeUTF(cut.getStream().getScopedName()); Map<Segment, Long> map = cut.getPositions(); revisionDataOutput.writeMap(map, (out, s) -> out.writeCompactLong(s.getSegmentId()), (out, offset) -> out.writeCompactLong(offset)); } }
@Test public void testSameRoutingKey() { StreamSegments streamSegments = initStreamSegments(4); int[] counts = new int[4]; Arrays.fill(counts, 0); for (int i = 0; i < 20; i++) { Segment segment = streamSegments.getSegmentForKey("Foo"); assertNotNull(segment); counts[StreamSegmentNameUtils.getSegmentNumber(segment.getSegmentId())]++; } assertArrayEquals(new int[] { 20, 0, 0, 0 }, counts); }
@Test(timeout = 5000) public void testGetSegmentsWithUnboundedStreamCut() throws Exception { PravegaNodeUri location = new PravegaNodeUri("localhost", 0); MockConnectionFactoryImpl connectionFactory = getMockConnectionFactory(location); MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory); Stream stream = createStream(SCOPE, STREAM, 3, mockController); BatchClientFactoryImpl client = new BatchClientFactoryImpl(mockController, connectionFactory); Iterator<SegmentRange> unBoundedSegments = client.getSegments(stream, StreamCut.UNBOUNDED, StreamCut.UNBOUNDED).getIterator(); assertTrue(unBoundedSegments.hasNext()); assertEquals(0L, unBoundedSegments.next().asImpl().getSegment().getSegmentId()); assertTrue(unBoundedSegments.hasNext()); assertEquals(1L, unBoundedSegments.next().asImpl().getSegment().getSegmentId()); assertTrue(unBoundedSegments.hasNext()); assertEquals(2L, unBoundedSegments.next().asImpl().getSegment().getSegmentId()); assertFalse(unBoundedSegments.hasNext()); }
@Test(timeout = 5000) public void testGetSegmentsWithNullStreamCut() throws Exception { PravegaNodeUri location = new PravegaNodeUri("localhost", 0); MockConnectionFactoryImpl connectionFactory = getMockConnectionFactory(location); MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory); Stream stream = createStream(SCOPE, STREAM, 3, mockController); BatchClientFactoryImpl client = new BatchClientFactoryImpl(mockController, connectionFactory); Iterator<SegmentRange> segments = client.getSegments(stream, null, null).getIterator(); assertTrue(segments.hasNext()); assertEquals(0L, segments.next().asImpl().getSegment().getSegmentId()); assertTrue(segments.hasNext()); assertEquals(1L, segments.next().asImpl().getSegment().getSegmentId()); assertTrue(segments.hasNext()); assertEquals(2L, segments.next().asImpl().getSegment().getSegmentId()); assertFalse(segments.hasNext()); }
@Test(timeout = 5000) public void testGetSegmentsWithStreamCut() throws Exception { PravegaNodeUri location = new PravegaNodeUri("localhost", 0); MockConnectionFactoryImpl connectionFactory = getMockConnectionFactory(location); MockController mockController = new MockController(location.getEndpoint(), location.getPort(), connectionFactory); Stream stream = createStream(SCOPE, STREAM, 3, mockController); BatchClientFactoryImpl client = new BatchClientFactoryImpl(mockController, connectionFactory); Iterator<SegmentRange> boundedSegments = client.getSegments(stream, getStreamCut(5L, 0, 1, 2), getStreamCut(15L, 0, 1, 2)).getIterator(); assertTrue(boundedSegments.hasNext()); assertEquals(0L, boundedSegments.next().asImpl().getSegment().getSegmentId()); assertTrue(boundedSegments.hasNext()); assertEquals(1L, boundedSegments.next().asImpl().getSegment().getSegmentId()); assertTrue(boundedSegments.hasNext()); assertEquals(2L, boundedSegments.next().asImpl().getSegment().getSegmentId()); assertFalse(boundedSegments.hasNext()); }
@Test public void encodeSegmentId() { Segment segment = ModelHelper.encode(ModelHelper.decode(createSegmentId("stream1", 2L))); assertEquals("stream1", segment.getStreamName()); assertEquals("scope", segment.getScope()); assertEquals(2L, segment.getSegmentId()); }