private StreamCut computeEndStreamCut(Stream stream, Map<Segment, Long> endSegments) { final Map<Segment, Long> toPositions = endSegments.entrySet().stream() .filter(e -> e.getKey().getStream().equals(stream)) .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); return toPositions.isEmpty() ? StreamCut.UNBOUNDED : new StreamCutImpl(stream, toPositions); }
@Synchronized Map<Stream, Map<Segment, Long>> getPositions() { Map<Stream, Map<Segment, Long>> result = new HashMap<>(); for (Entry<Segment, Long> entry : unassignedSegments.entrySet()) { result.computeIfAbsent(entry.getKey().getStream(), s -> new HashMap<>()).put(entry.getKey(), entry.getValue()); } for (Map<Segment, Long> assigned : assignedSegments.values()) { for (Entry<Segment, Long> entry : assigned.entrySet()) { result.computeIfAbsent(entry.getKey().getStream(), s -> new HashMap<>()).put(entry.getKey(), entry.getValue()); } } return result; }
@Synchronized Optional<Map<Stream, StreamCut>> getStreamCutsForCompletedCheckpoint(final String checkpointId) { final Optional<Map<Segment, Long>> positionMap = Optional.ofNullable(checkpointState.getPositionsForCompletedCheckpoint(checkpointId)); return positionMap.map(map -> map.entrySet().stream() .collect(groupingBy(o -> o.getKey().getStream(), collectingAndThen(toMap(Entry::getKey, Entry::getValue), x -> new StreamCutImpl(x.keySet().stream().findAny().get().getStream(), x))))); }
CheckpointImpl(String name, Map<Segment, Long> segmentPositions) { this.name = name; Map<Stream, ImmutableMap.Builder<Segment, Long>> streamPositions = new HashMap<>(); for (Entry<Segment, Long> position : segmentPositions.entrySet()) { streamPositions.computeIfAbsent(position.getKey().getStream(), k -> new ImmutableMap.Builder<Segment, Long>()) .put(position); } ImmutableMap.Builder<Stream, StreamCut> positionBuilder = ImmutableMap.builder(); for (Entry<Stream, Builder<Segment, Long>> streamPosition : streamPositions.entrySet()) { positionBuilder.put(streamPosition.getKey(), new StreamCutImpl(streamPosition.getKey(), streamPosition.getValue().build())); } this.positions = positionBuilder.build(); }
@Synchronized Optional<Map<Stream, Map<Segment, Long>>> getPositionsForLastCompletedCheckpoint() { Optional<Map<Segment, Long>> positions = checkpointState.getPositionsForLatestCompletedCheckpoint(); if (positions.isPresent()) { Map<Stream, Map<Segment, Long>> result = new HashMap<>(); for (Entry<Segment, Long> entry : positions.get().entrySet()) { result.computeIfAbsent(entry.getKey().getStream(), s -> new HashMap<>()).put(entry.getKey(), entry.getValue()); } return Optional.of(result); } else { return Optional.empty(); } }
private EventSegmentReader getEventSegmentReader(Segment segment, long endOffset, int bufferSize) { String delegationToken = Futures.getAndHandleExceptions(controller.getOrRefreshDelegationTokenFor(segment.getScope(), segment.getStream() .getStreamName()), RuntimeException::new); AsyncSegmentInputStreamImpl async = new AsyncSegmentInputStreamImpl(controller, cf, segment, delegationToken); async.getConnection(); return getEventSegmentReader(async, 0, endOffset, bufferSize); }
private ByteStreamReader createByteStreamReaders(Segment segment) { String delegationToken = Futures.getAndHandleExceptions(controller.getOrRefreshDelegationTokenFor(segment.getScope(), segment.getStream() .getStreamName()), RuntimeException::new); return new ByteStreamReaderImpl(inputStreamFactory.createInputStreamForSegment(segment, delegationToken), metaStreamFactory.createSegmentMetadataClient(segment, delegationToken)); }
@Test public void testReaderGroupInit() throws Exception { ReaderGroupInitSerializer initSerializer = new ReaderGroupInitSerializer(); ReaderGroupConfig config = ReaderGroupConfig.builder() .disableAutomaticCheckpoints() .groupRefreshTimeMillis(r.nextInt(1000)) .stream(createSegment().getStream()) .build(); verify(initSerializer, new ReaderGroupStateInit(config, createSegmentToLongMap(), createSegmentToLongMap())); CompactReaderGroupStateBuilder builder = new CompactReaderGroupState.CompactReaderGroupStateBuilder(); builder.assignedSegments(createMap(this::createString, this::createSegmentToLongMap)); builder.checkpointState(new CheckpointState.CheckpointStateBuilder().checkpoints(createList(this::createString)) .lastCheckpointPosition(createSegmentToLongMap()) .checkpointPositions(createMap(this::createString, this::createSegmentToLongMap)) .uncheckpointedHosts(createMap(this::createString, this::createStringList)) .build()); builder.config(config); builder.distanceToTail(createMap(this::createString, r::nextLong)); builder.endSegments(createSegmentToLongMap()); builder.unassignedSegments(createSegmentToLongMap()); builder.futureSegments(createMap(this::createSegment, () -> new HashSet<>(createLongList()))); verify(initSerializer, builder.build()); }