/** * Constructing the end-of-stream states for a task * @param ssps all the ssps assigned to this task * @param producerTaskCounts mapping from a stream to the number of upstream tasks that produce to it */ EndOfStreamStates(Set<SystemStreamPartition> ssps, Map<SystemStream, Integer> producerTaskCounts) { Map<SystemStreamPartition, EndOfStreamState> states = new HashMap<>(); ssps.forEach(ssp -> { states.put(ssp, new EndOfStreamState(producerTaskCounts.getOrDefault(ssp.getSystemStream(), 0))); }); this.eosStates = Collections.unmodifiableMap(states); }
/** * Constructing the end-of-stream states for a task * @param ssps all the ssps assigned to this task * @param producerTaskCounts mapping from a stream to the number of upstream tasks that produce to it */ EndOfStreamStates(Set<SystemStreamPartition> ssps, Map<SystemStream, Integer> producerTaskCounts) { Map<SystemStreamPartition, EndOfStreamState> states = new HashMap<>(); ssps.forEach(ssp -> { states.put(ssp, new EndOfStreamState(producerTaskCounts.getOrDefault(ssp.getSystemStream(), 0))); }); this.eosStates = Collections.unmodifiableMap(states); }
@Override protected void put(SystemStreamPartition ssp, IncomingMessageEnvelope envelope) { try { super.put(ssp, envelope); } catch (Exception e) { LOG.error("Exception while putting record. Shutting down SystemStream {}", ssp.getSystemStream(), e); Thread.currentThread().interrupt(); } }
/** * Constructing the end-of-stream states for a task * @param ssps all the ssps assigned to this task * @param producerTaskCounts mapping from a stream to the number of upstream tasks that produce to it */ EndOfStreamStates(Set<SystemStreamPartition> ssps, Map<SystemStream, Integer> producerTaskCounts) { Map<SystemStreamPartition, EndOfStreamState> states = new HashMap<>(); ssps.forEach(ssp -> { states.put(ssp, new EndOfStreamState(producerTaskCounts.getOrDefault(ssp.getSystemStream(), 0))); }); this.eosStates = Collections.unmodifiableMap(states); }
/** * Constructing the end-of-stream states for a task * @param ssps all the ssps assigned to this task * @param producerTaskCounts mapping from a stream to the number of upstream tasks that produce to it */ EndOfStreamStates(Set<SystemStreamPartition> ssps, Map<SystemStream, Integer> producerTaskCounts) { Map<SystemStreamPartition, EndOfStreamState> states = new HashMap<>(); ssps.forEach(ssp -> { states.put(ssp, new EndOfStreamState(producerTaskCounts.getOrDefault(ssp.getSystemStream(), 0))); }); this.eosStates = Collections.unmodifiableMap(states); }
long getWatermark(SystemStream systemStream) { return watermarkStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .map(entry -> entry.getValue().getWatermarkTime()) .min(Long::compare) .orElse(WATERMARK_NOT_EXIST); }
long getWatermark(SystemStream systemStream) { return watermarkStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .map(entry -> entry.getValue().getWatermarkTime()) .min(Long::compare) .orElse(WATERMARK_NOT_EXIST); }
boolean isEndOfStream(SystemStream systemStream) { return eosStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .allMatch(entry -> entry.getValue().isEndOfStream()); }
long getWatermark(SystemStream systemStream) { return watermarkStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .map(entry -> entry.getValue().getWatermarkTime()) .min(Long::compare) .orElse(WATERMARK_NOT_EXIST); }
boolean isEndOfStream(SystemStream systemStream) { return eosStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .allMatch(entry -> entry.getValue().isEndOfStream()); }
boolean isEndOfStream(SystemStream systemStream) { return eosStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .allMatch(entry -> entry.getValue().isEndOfStream()); }
void registerInputStream(SystemStream input) { this.inputStreams.add(input); usedInCurrentTask = usedInCurrentTask || taskModel.getSystemStreamPartitions().stream().anyMatch(ssp -> ssp.getSystemStream().equals(input)); }
void registerInputStream(SystemStream input) { this.inputStreams.add(input); usedInCurrentTask = usedInCurrentTask || taskModel.getSystemStreamPartitions().stream().anyMatch(ssp -> ssp.getSystemStream().equals(input)); }
void registerInputStream(SystemStream input) { this.inputStreams.add(input); usedInCurrentTask = usedInCurrentTask || taskModel.getSystemStreamPartitions().stream().anyMatch(ssp -> ssp.getSystemStream().equals(input)); }
synchronized void free(SystemStreamPartition ssp) { boolean success = availableSsps.computeIfAbsent(ssp.getStream(), p -> new HashSet<>()).add(ssp); Validate.isTrue(success, String.format("Ssp %s is already in free pool.", ssp)); LOG.info("Number of unassigned partitions for system-stream {} is {}.", ssp.getSystemStream(), availableSsps.get(ssp.getStream()).size()); } }
@Override public SystemStreamPartition getPreviousSSP(SystemStreamPartition currentSystemStreamPartition, int previousPartitionCount, int currentPartitionCount) { Preconditions.checkNotNull(currentSystemStreamPartition); Preconditions.checkArgument(currentPartitionCount % previousPartitionCount == 0, String.format("New partition count: %d should be a multiple of previous partition count: %d.", currentPartitionCount, previousPartitionCount)); Partition partition = currentSystemStreamPartition.getPartition(); Preconditions.checkNotNull(partition, String.format("SystemStreamPartition: %s cannot have null partition", currentSystemStreamPartition)); int currentPartitionId = partition.getPartitionId(); int previousPartitionId = currentPartitionId % previousPartitionCount; return new SystemStreamPartition(currentSystemStreamPartition.getSystemStream(), new Partition(previousPartitionId)); } }
void broadcastToOtherPartitions(ControlMessage message, SystemStreamPartition ssp, MessageCollector collector) { SystemStream systemStream = ssp.getSystemStream(); int partitionCount = getPartitionCount(systemStream); int currentPartition = ssp.getPartition().getPartitionId(); for (int i = 0; i < partitionCount; i++) { if (i != currentPartition) { OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, i, null, message); collector.send(envelopeOut); } } }
void broadcastToOtherPartitions(ControlMessage message, SystemStreamPartition ssp, MessageCollector collector) { SystemStream systemStream = ssp.getSystemStream(); int partitionCount = getPartitionCount(systemStream); int currentPartition = ssp.getPartition().getPartitionId(); for (int i = 0; i < partitionCount; i++) { if (i != currentPartition) { OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, i, null, message); collector.send(envelopeOut); } } }
void broadcastToOtherPartitions(ControlMessage message, SystemStreamPartition ssp, MessageCollector collector) { SystemStream systemStream = ssp.getSystemStream(); int partitionCount = getPartitionCount(systemStream); int currentPartition = ssp.getPartition().getPartitionId(); for (int i = 0; i < partitionCount; i++) { if (i != currentPartition) { OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, i, null, message); collector.send(envelopeOut); } } }
void broadcastToOtherPartitions(ControlMessage message, SystemStreamPartition ssp, MessageCollector collector) { SystemStream systemStream = ssp.getSystemStream(); int partitionCount = getPartitionCount(systemStream); int currentPartition = ssp.getPartition().getPartitionId(); for (int i = 0; i < partitionCount; i++) { if (i != currentPartition) { OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, i, null, message); collector.send(envelopeOut); } } }